From 7bbc43442b3d49354110677cdaf16e537e956851 Mon Sep 17 00:00:00 2001 From: Matt Brock Date: Tue, 17 Dec 2024 16:06:54 -0600 Subject: [PATCH 01/12] Protobuf and configuration for Access Graph Azure Discovery --- api/types/types.pb.go | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/api/types/types.pb.go b/api/types/types.pb.go index b8eec25929456..db2b270523755 100644 --- a/api/types/types.pb.go +++ b/api/types/types.pb.go @@ -50522,6 +50522,47 @@ func (m *AccessGraphAzureSync) MarshalTo(dAtA []byte) (int, error) { return m.MarshalToSizedBuffer(dAtA[:size]) } +func (m *AccessGraphAzureSync) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if len(m.Integration) > 0 { + i -= len(m.Integration) + copy(dAtA[i:], m.Integration) + i = encodeVarintTypes(dAtA, i, uint64(len(m.Integration))) + i-- + dAtA[i] = 0x1a + } + if len(m.SubscriptionID) > 0 { + i -= len(m.SubscriptionID) + copy(dAtA[i:], m.SubscriptionID) + i = encodeVarintTypes(dAtA, i, uint64(len(m.SubscriptionID))) + i-- + dAtA[i] = 0x12 + } + return len(dAtA) - i, nil +} + +func (m *AccessGraphAzureSync) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *AccessGraphAzureSync) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + func (m *AccessGraphAzureSync) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i From 0e5e04f5d12247c9d5b724c07d0cbf2fce5b96ce Mon Sep 17 00:00:00 2001 From: Matt Brock Date: Tue, 17 Dec 2024 16:49:05 -0600 Subject: [PATCH 02/12] Adding the Azure sync module functions along with new cloud client functionality --- go.mod | 1 + go.sum | 2 + lib/cloud/azure/roleassignments.go | 57 +++++ lib/cloud/azure/roledefinitions.go | 57 +++++ lib/cloud/clients.go | 28 +- .../fetchers/azure-sync/msggraphclient.go | 240 ++++++++++++++++++ .../fetchers/azure-sync/principals.go | 82 ++++++ .../fetchers/azure-sync/reconcile.go | 165 ++++++++++++ .../fetchers/azure-sync/reconcile_test.go | 191 ++++++++++++++ .../fetchers/azure-sync/roleassignments.go | 63 +++++ .../fetchers/azure-sync/roledefinitions.go | 77 ++++++ .../fetchers/azure-sync/virtualmachines.go | 56 ++++ 12 files changed, 1018 insertions(+), 1 deletion(-) create mode 100644 lib/cloud/azure/roleassignments.go create mode 100644 lib/cloud/azure/roledefinitions.go create mode 100644 lib/srv/discovery/fetchers/azure-sync/msggraphclient.go create mode 100644 lib/srv/discovery/fetchers/azure-sync/principals.go create mode 100644 lib/srv/discovery/fetchers/azure-sync/reconcile.go create mode 100644 lib/srv/discovery/fetchers/azure-sync/reconcile_test.go create mode 100644 lib/srv/discovery/fetchers/azure-sync/roleassignments.go create mode 100644 lib/srv/discovery/fetchers/azure-sync/roledefinitions.go create mode 100644 lib/srv/discovery/fetchers/azure-sync/virtualmachines.go diff --git a/go.mod b/go.mod index eb16212086763..3e0e465c36757 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( connectrpc.com/connect v1.17.0 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0 + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v2 v2.2.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v6 v6.1.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v6 v6.3.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi v1.2.0 diff --git a/go.sum b/go.sum index 82132a3ee7268..e210b7c537d2d 100644 --- a/go.sum +++ b/go.sum @@ -668,6 +668,8 @@ github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0/go.mod h1:eWRD7oawr1Mu1sLC github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.1/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 h1:ywEEhmNahHBihViHepv3xPBn1663uRv2t2q/ESv9seY= github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0/go.mod h1:iZDifYGJTIgIIkYRNWPENUnqx6bJ2xnSDFI2tjwZNuY= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v2 v2.2.0 h1:Hp+EScFOu9HeCbeW8WU2yQPJd4gGwhMgKxWe+G6jNzw= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v2 v2.2.0/go.mod h1:/pz8dyNQe+Ey3yBp/XuYz7oqX8YDNWVpPB0hH3XWfbc= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v6 v6.1.0 h1:zDeQI/PaWztI2tcrGO/9RIMey9NvqYbnyttf/0P3QWM= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v6 v6.1.0/go.mod h1:zflC9v4VfViJrSvcvplqws/yGXVbUEMZi/iHpZdSPWA= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v5 v5.0.0 h1:5n7dPVqsWfVKw+ZiEKSd3Kzu7gwBkbEBkeXb8rgaE9Q= diff --git a/lib/cloud/azure/roleassignments.go b/lib/cloud/azure/roleassignments.go new file mode 100644 index 0000000000000..114bceef88b96 --- /dev/null +++ b/lib/cloud/azure/roleassignments.go @@ -0,0 +1,57 @@ +/* + * Teleport + * Copyright (C) 2024 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package azure + +import ( + "context" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v2" + "github.com/gravitational/trace" +) + +// RoleAssignmentsClient wraps the Azure API to provide a high level subset of functionality +type RoleAssignmentsClient struct { + cli *armauthorization.RoleAssignmentsClient +} + +// NewRoleAssignmentsClient creates a new client for a given subscription and credentials +func NewRoleAssignmentsClient(subscription string, cred azcore.TokenCredential, options *arm.ClientOptions) (*RoleAssignmentsClient, error) { + clientFactory, err := armauthorization.NewClientFactory(subscription, cred, options) + if err != nil { + return nil, trace.Wrap(err) + } + roleDefCli := clientFactory.NewRoleAssignmentsClient() + return &RoleAssignmentsClient{cli: roleDefCli}, nil +} + +// ListRoleAssignments returns role assignments for a given scope +func (c *RoleAssignmentsClient) ListRoleAssignments(ctx context.Context, scope string) ([]*armauthorization.RoleAssignment, error) { + pager := c.cli.NewListForScopePager(scope, nil) + var roleDefs []*armauthorization.RoleAssignment + for pager.More() { + page, err := pager.NextPage(ctx) + if err != nil { + return nil, trace.Wrap(err) + } + roleDefs = append(roleDefs, page.Value...) + } + return roleDefs, nil +} diff --git a/lib/cloud/azure/roledefinitions.go b/lib/cloud/azure/roledefinitions.go new file mode 100644 index 0000000000000..cdc46196aa530 --- /dev/null +++ b/lib/cloud/azure/roledefinitions.go @@ -0,0 +1,57 @@ +/* + * Teleport + * Copyright (C) 2024 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package azure + +import ( + "context" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v2" + "github.com/gravitational/trace" +) + +// RoleDefinitionsClient wraps the Azure API to provide a high level subset of functionality +type RoleDefinitionsClient struct { + cli *armauthorization.RoleDefinitionsClient +} + +// NewRoleDefinitionsClient creates a new client for a given subscription and credentials +func NewRoleDefinitionsClient(subscription string, cred azcore.TokenCredential, options *arm.ClientOptions) (*RoleDefinitionsClient, error) { + clientFactory, err := armauthorization.NewClientFactory(subscription, cred, options) + if err != nil { + return nil, trace.Wrap(err) + } + roleDefCli := clientFactory.NewRoleDefinitionsClient() + return &RoleDefinitionsClient{cli: roleDefCli}, nil +} + +// ListRoleDefinitions returns role definitions for a given scope +func (c *RoleDefinitionsClient) ListRoleDefinitions(ctx context.Context, scope string) ([]*armauthorization.RoleDefinition, error) { + pager := c.cli.NewListPager(scope, nil) + var roleDefs []*armauthorization.RoleDefinition + for pager.More() { + page, err := pager.NextPage(ctx) + if err != nil { + return nil, trace.Wrap(err) + } + roleDefs = append(roleDefs, page.Value...) + } + return roleDefs, nil +} diff --git a/lib/cloud/clients.go b/lib/cloud/clients.go index 54b02d84dc400..1f6c04d5ca7c4 100644 --- a/lib/cloud/clients.go +++ b/lib/cloud/clients.go @@ -356,6 +356,10 @@ type azureClients struct { azurePostgresFlexServersClients azure.ClientMap[azure.PostgresFlexServersClient] // azureRunCommandClients contains the cached Azure Run Command clients. azureRunCommandClients azure.ClientMap[azure.RunCommandClient] + // azureRoleDefinitionsClients contains the cached Azure Role Definitions clients. + azureRoleDefinitionsClients azure.ClientMap[azure.RoleDefinitionsClient] + // azureRoleAssignmentsClients contains the cached Azure Role Assignments clients. + azureRoleAssignmentsClients azure.ClientMap[azure.RoleAssignmentsClient] } // credentialsSource defines where the credentials must come from. @@ -756,6 +760,16 @@ func (c *cloudClients) GetAzureRunCommandClient(subscription string) (azure.RunC return c.azureRunCommandClients.Get(subscription, c.GetAzureCredential) } +// GetAzureRoleDefinitionsClient returns an Azure Role Definitions client +func (c *cloudClients) GetAzureRoleDefinitionsClient(subscription string) (azure.RoleDefinitionsClient, error) { + return c.azureRoleDefinitionsClients.Get(subscription, c.GetAzureCredential) +} + +// GetAzureRoleAssignmentsClient returns an Azure Role Assignments client +func (c *cloudClients) GetAzureRoleAssignmentsClient(subscription string) (azure.RoleAssignmentsClient, error) { + return c.azureRoleAssignmentsClients.Get(subscription, c.GetAzureCredential) +} + // Close closes all initialized clients. func (c *cloudClients) Close() (err error) { c.mtx.Lock() @@ -1064,6 +1078,8 @@ type TestCloudClients struct { AzureMySQLFlex azure.MySQLFlexServersClient AzurePostgresFlex azure.PostgresFlexServersClient AzureRunCommand azure.RunCommandClient + AzureRoleDefinitions azure.RoleDefinitionsClient + AzureRoleAssignments azure.RoleAssignmentsClient } // GetAWSSession returns AWS session for the specified region, optionally @@ -1317,11 +1333,21 @@ func (c *TestCloudClients) GetAzurePostgresFlexServersClient(subscription string return c.AzurePostgresFlex, nil } -// GetAzureRunCommand returns an Azure Run Command client for the given subscription. +// GetAzureRunCommandClient returns an Azure Run Command client for the given subscription. func (c *TestCloudClients) GetAzureRunCommandClient(subscription string) (azure.RunCommandClient, error) { return c.AzureRunCommand, nil } +// GetAzureRoleDefinitionsClient returns an Azure Role Definitions client for the given subscription. +func (c *TestCloudClients) GetAzureRoleDefinitionsClient(subscription string) (azure.RoleDefinitionsClient, error) { + return c.AzureRoleDefinitions, nil +} + +// GetAzureRoleAssignmentsClient returns an Azure Role Assignments client for the given subscription. +func (c *TestCloudClients) GetAzureRoleAssignmentsClient(subscription string) (azure.RoleAssignmentsClient, error) { + return c.AzureRoleAssignments, nil +} + // Close closes all initialized clients. func (c *TestCloudClients) Close() error { return nil diff --git a/lib/srv/discovery/fetchers/azure-sync/msggraphclient.go b/lib/srv/discovery/fetchers/azure-sync/msggraphclient.go new file mode 100644 index 0000000000000..75d2960d7fa55 --- /dev/null +++ b/lib/srv/discovery/fetchers/azure-sync/msggraphclient.go @@ -0,0 +1,240 @@ +/* + * Teleport + * Copyright (C) 2024 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package azure_sync + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" +) + +// GraphClient represents generic MS API client +type GraphClient struct { + token azcore.AccessToken +} + +const ( + usersSuffix = "users" + groupsSuffix = "groups" + servicePrincipalsSuffix = "servicePrincipals" + graphBaseURL = "https://graph.microsoft.com/v1.0" + httpTimeout = time.Second * 30 +) + +// graphError represents MS Graph error +type graphError struct { + E struct { + Code string `json:"code"` + Message string `json:"message"` + } `json:"error"` +} + +// genericGraphResponse represents the utility struct for parsing MS Graph API response +type genericGraphResponse struct { + Context string `json:"@odata.context"` + Count int `json:"@odata.count"` + NextLink string `json:"@odata.nextLink"` + Value json.RawMessage `json:"value"` +} + +// User represents user resource +type User struct { + ID string `json:"id"` + Name string `json:"displayName"` + MemberOf []Membership `json:"memberOf"` +} + +type Membership struct { + Type string `json:"@odata.type"` + ID string `json:"id"` +} + +// request represents generic request structure +type request struct { + // Method HTTP method + Method string + // URL which overrides URL construction + URL *string + // Path to a resource + Path string + // Expand $expand value + Expand []string + // Filter $filter value + Filter string + // Body request body + Body string + // Response represents template structure for a response + Response interface{} + // Err represents template structure for an error + Err error + // SuccessCode http code representing success + SuccessCode int +} + +// GetURL builds the request URL +func (r *request) GetURL() (string, error) { + if r.URL != nil { + return *r.URL, nil + } + u, err := url.Parse(graphBaseURL) + if err != nil { + return "", err + } + + data := url.Values{} + if len(r.Expand) > 0 { + data.Set("$expand", strings.Join(r.Expand, ",")) + } + if r.Filter != "" { + data.Set("$filter", r.Filter) + } + + u.Path = u.Path + "/" + r.Path + u.RawQuery = data.Encode() + + return u.String(), nil +} + +// NewGraphClient creates MS Graph API client +func NewGraphClient(token azcore.AccessToken) *GraphClient { + return &GraphClient{ + token: token, + } +} + +// Error returns error string +func (e graphError) Error() string { + return e.E.Code + " " + e.E.Message +} + +func (c *GraphClient) ListUsers(ctx context.Context) ([]User, error) { + return c.listIdentities(ctx, usersSuffix, []string{"memberOf"}) +} + +func (c *GraphClient) ListGroups(ctx context.Context) ([]User, error) { + return c.listIdentities(ctx, groupsSuffix, []string{"memberOf"}) +} + +func (c *GraphClient) ListServicePrincipals(ctx context.Context) ([]User, error) { + return c.listIdentities(ctx, servicePrincipalsSuffix, []string{"memberOf"}) +} + +func (c *GraphClient) listIdentities(ctx context.Context, idType string, expand []string) ([]User, error) { + var users []User + var nextLink *string + for { + g := &genericGraphResponse{} + req := request{ + Method: http.MethodGet, + Path: idType, + Expand: expand, + Response: &g, + Err: &graphError{}, + URL: nextLink, + } + err := c.request(ctx, req) + if err != nil { + return nil, err + } + var newUsers []User + err = json.NewDecoder(bytes.NewReader(g.Value)).Decode(&newUsers) + if err != nil { + return nil, err + } + users = append(users, newUsers...) + if g.NextLink == "" { + break + } + nextLink = &g.NextLink + } + + return users, nil +} + +// request sends the request to the graph/bot service and returns response body as bytes slice +func (c *GraphClient) request(ctx context.Context, req request) error { + reqUrl, err := req.GetURL() + if err != nil { + return err + } + + r, err := http.NewRequestWithContext(ctx, req.Method, reqUrl, strings.NewReader(req.Body)) + if err != nil { + return err + } + + r.Header.Set("Authorization", "Bearer "+c.token.Token) + r.Header.Set("Content-Type", "application/json") + + client := http.Client{Timeout: httpTimeout} + resp, err := client.Do(r) + if err != nil { + return err + } + + defer func(r *http.Response) { + _ = r.Body.Close() + }(resp) + + b, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + + expectedCode := req.SuccessCode + if expectedCode == 0 { + expectedCode = http.StatusOK + } + + if expectedCode == resp.StatusCode { + if req.Response == nil { + return nil + } + + err := json.NewDecoder(bytes.NewReader(b)).Decode(req.Response) + if err != nil { + return err + } + } else { + if req.Err == nil { + return fmt.Errorf("Error requesting MS Graph API: %v", string(b)) + } + + err := json.NewDecoder(bytes.NewReader(b)).Decode(req.Err) + if err != nil { + return err + } + + if req.Err.Error() == "" { + return fmt.Errorf("Error requesting MS Graph API. Expected response code was %v, but is %v", expectedCode, resp.StatusCode) + } + + return req.Err + } + + return nil +} diff --git a/lib/srv/discovery/fetchers/azure-sync/principals.go b/lib/srv/discovery/fetchers/azure-sync/principals.go new file mode 100644 index 0000000000000..850e0cb389f71 --- /dev/null +++ b/lib/srv/discovery/fetchers/azure-sync/principals.go @@ -0,0 +1,82 @@ +/* + * Teleport + * Copyright (C) 2024 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package azure_sync + +import ( + "context" + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "slices" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/gravitational/trace" + "google.golang.org/protobuf/types/known/timestamppb" + + accessgraphv1alpha "github.com/gravitational/teleport/gen/proto/go/accessgraph/v1alpha" +) + +const groupType = "#microsoft.graph.group" +const defaultGraphScope = "https://graph.microsoft.com/.default" + +// fetchPrincipals fetches the Azure principals (users, groups, and service principals) using the Graph API +func fetchPrincipals(ctx context.Context, subscriptionID string, cred azcore.TokenCredential) ([]*accessgraphv1alpha.AzurePrincipal, error) { + // Get the graph client + scopes := []string{defaultGraphScope} + token, err := cred.GetToken(ctx, policy.TokenRequestOptions{Scopes: scopes}) + if err != nil { + return nil, trace.Wrap(err) + } + cli := NewGraphClient(token) + + // Fetch the users, groups, and managed identities + users, err := cli.ListUsers(ctx) + if err != nil { + return nil, trace.Wrap(err) + } + groups, err := cli.ListGroups(ctx) + if err != nil { + return nil, trace.Wrap(err) + } + svcPrincipals, err := cli.ListServicePrincipals(ctx) + if err != nil { + return nil, trace.Wrap(err) + } + principals := slices.Concat(users, groups, svcPrincipals) + + // Return the users as protobuf messages + pbPrincipals := make([]*accessgraphv1alpha.AzurePrincipal, 0, len(principals)) + for _, principal := range principals { + // Extract group membership + memberOf := make([]string, 0) + for _, member := range principal.MemberOf { + if member.Type == groupType { + memberOf = append(memberOf, member.ID) + } + } + // Create the protobuf principal and append it to the list + pbPrincipal := &accessgraphv1alpha.AzurePrincipal{ + Id: principal.ID, + SubscriptionId: subscriptionID, + LastSyncTime: timestamppb.Now(), + DisplayName: principal.Name, + MemberOf: memberOf, + } + pbPrincipals = append(pbPrincipals, pbPrincipal) + } + return pbPrincipals, nil +} diff --git a/lib/srv/discovery/fetchers/azure-sync/reconcile.go b/lib/srv/discovery/fetchers/azure-sync/reconcile.go new file mode 100644 index 0000000000000..2b54c8cfac911 --- /dev/null +++ b/lib/srv/discovery/fetchers/azure-sync/reconcile.go @@ -0,0 +1,165 @@ +/* + * Teleport + * Copyright (C) 2024 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package azure_sync + +import ( + "fmt" + + "google.golang.org/protobuf/proto" + + accessgraphv1alpha "github.com/gravitational/teleport/gen/proto/go/accessgraph/v1alpha" + "github.com/gravitational/teleport/lib/srv/discovery/common" +) + +// MergeResources merges Azure resources fetched from multiple configured Azure fetchers +func MergeResources(results ...*Resources) *Resources { + if len(results) == 0 { + return &Resources{} + } + if len(results) == 1 { + return results[0] + } + result := &Resources{} + for _, r := range results { + result.Principals = append(result.Principals, r.Principals...) + result.RoleAssignments = append(result.RoleAssignments, r.RoleAssignments...) + result.RoleDefinitions = append(result.RoleDefinitions, r.RoleDefinitions...) + result.VirtualMachines = append(result.VirtualMachines, r.VirtualMachines...) + } + result.Principals = common.DeduplicateSlice(result.Principals, azurePrincipalsKey) + result.RoleAssignments = common.DeduplicateSlice(result.RoleAssignments, azureRoleAssignKey) + result.RoleDefinitions = common.DeduplicateSlice(result.RoleDefinitions, azureRoleDefKey) + result.VirtualMachines = common.DeduplicateSlice(result.VirtualMachines, azureVmKey) + return result +} + +// newResourceList creates a new resource list message +func newResourceList() *accessgraphv1alpha.AzureResourceList { + return &accessgraphv1alpha.AzureResourceList{ + Resources: make([]*accessgraphv1alpha.AzureResource, 0), + } +} + +// ReconcileResults compares previously and currently fetched results and determines which resources to upsert and +// which to delete. +func ReconcileResults(old *Resources, new *Resources) (upsert, delete *accessgraphv1alpha.AzureResourceList) { + upsert, delete = newResourceList(), newResourceList() + reconciledResources := []*reconcilePair{ + reconcile(old.Principals, new.Principals, azurePrincipalsKey, azurePrincipalsWrap), + reconcile(old.RoleAssignments, new.RoleAssignments, azureRoleAssignKey, azureRoleAssignWrap), + reconcile(old.RoleDefinitions, new.RoleDefinitions, azureRoleDefKey, azureRoleDefWrap), + reconcile(old.VirtualMachines, new.VirtualMachines, azureVmKey, azureVmWrap), + } + for _, res := range reconciledResources { + upsert.Resources = append(upsert.Resources, res.upsert.Resources...) + delete.Resources = append(delete.Resources, res.delete.Resources...) + } + return upsert, delete +} + +// reconcilePair contains the Azure resources to upsert and delete +type reconcilePair struct { + upsert, delete *accessgraphv1alpha.AzureResourceList +} + +// reconcile compares old and new items to build a list of resources to upsert and delete in the Access Graph +func reconcile[T proto.Message]( + oldItems []T, + newItems []T, + keyFn func(T) string, + wrapFn func(T) *accessgraphv1alpha.AzureResource, +) *reconcilePair { + // Remove duplicates from the new items + newItems = common.DeduplicateSlice(newItems, keyFn) + upsertRes := newResourceList() + deleteRes := newResourceList() + + // Delete all old items if there are no new items + if len(newItems) == 0 { + for _, item := range oldItems { + deleteRes.Resources = append(deleteRes.Resources, wrapFn(item)) + } + return &reconcilePair{upsertRes, deleteRes} + } + + // Create all new items if there are no old items + if len(oldItems) == 0 { + for _, item := range newItems { + upsertRes.Resources = append(upsertRes.Resources, wrapFn(item)) + } + return &reconcilePair{upsertRes, deleteRes} + } + + // Map old and new items by their key + oldMap := make(map[string]T, len(oldItems)) + for _, item := range oldItems { + oldMap[keyFn(item)] = item + } + newMap := make(map[string]T, len(newItems)) + for _, item := range newItems { + newMap[keyFn(item)] = item + } + + // Append new or modified items to the upsert list + for _, item := range newItems { + if oldItem, ok := oldMap[keyFn(item)]; !ok || !proto.Equal(oldItem, item) { + upsertRes.Resources = append(upsertRes.Resources, wrapFn(item)) + } + } + + // Append removed items to the delete list + for _, item := range oldItems { + if _, ok := newMap[keyFn(item)]; !ok { + deleteRes.Resources = append(deleteRes.Resources, wrapFn(item)) + } + } + return &reconcilePair{upsertRes, deleteRes} +} + +func azurePrincipalsKey(user *accessgraphv1alpha.AzurePrincipal) string { + return fmt.Sprintf("%s:%s", user.SubscriptionId, user.Id) +} + +func azurePrincipalsWrap(principal *accessgraphv1alpha.AzurePrincipal) *accessgraphv1alpha.AzureResource { + return &accessgraphv1alpha.AzureResource{Resource: &accessgraphv1alpha.AzureResource_Principal{Principal: principal}} +} + +func azureRoleAssignKey(roleAssign *accessgraphv1alpha.AzureRoleAssignment) string { + return fmt.Sprintf("%s:%s", roleAssign.SubscriptionId, roleAssign.Id) +} + +func azureRoleAssignWrap(roleAssign *accessgraphv1alpha.AzureRoleAssignment) *accessgraphv1alpha.AzureResource { + return &accessgraphv1alpha.AzureResource{Resource: &accessgraphv1alpha.AzureResource_RoleAssignment{RoleAssignment: roleAssign}} +} + +func azureRoleDefKey(roleDef *accessgraphv1alpha.AzureRoleDefinition) string { + return fmt.Sprintf("%s:%s", roleDef.SubscriptionId, roleDef.Id) +} + +func azureRoleDefWrap(roleDef *accessgraphv1alpha.AzureRoleDefinition) *accessgraphv1alpha.AzureResource { + return &accessgraphv1alpha.AzureResource{Resource: &accessgraphv1alpha.AzureResource_RoleDefinition{RoleDefinition: roleDef}} +} + +func azureVmKey(vm *accessgraphv1alpha.AzureVirtualMachine) string { + return fmt.Sprintf("%s:%s", vm.SubscriptionId, vm.Id) +} + +func azureVmWrap(vm *accessgraphv1alpha.AzureVirtualMachine) *accessgraphv1alpha.AzureResource { + return &accessgraphv1alpha.AzureResource{Resource: &accessgraphv1alpha.AzureResource_VirtualMachine{VirtualMachine: vm}} +} diff --git a/lib/srv/discovery/fetchers/azure-sync/reconcile_test.go b/lib/srv/discovery/fetchers/azure-sync/reconcile_test.go new file mode 100644 index 0000000000000..28b293bcf1f8d --- /dev/null +++ b/lib/srv/discovery/fetchers/azure-sync/reconcile_test.go @@ -0,0 +1,191 @@ +/* + * Teleport + * Copyright (C) 2024 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package azure_sync + +import ( + "testing" + + "github.com/stretchr/testify/require" + + accessgraphv1alpha "github.com/gravitational/teleport/gen/proto/go/accessgraph/v1alpha" +) + +func TestReconcileResults(t *testing.T) { + principals := generatePrincipals() + roleDefs := generateRoleDefs() + roleAssigns := generateRoleAssigns() + vms := generateVms() + + tests := []struct { + oldResults *Resources + newResults *Resources + expectedUpserts *accessgraphv1alpha.AzureResourceList + expectedDeletes *accessgraphv1alpha.AzureResourceList + }{ + // Overlapping old and new results + { + oldResults: &Resources{ + Principals: principals[0:2], + RoleDefinitions: roleDefs[0:2], + RoleAssignments: roleAssigns[0:2], + VirtualMachines: vms[0:2], + }, + newResults: &Resources{ + Principals: principals[1:3], + RoleDefinitions: roleDefs[1:3], + RoleAssignments: roleAssigns[1:3], + VirtualMachines: vms[1:3], + }, + expectedUpserts: generateExpected(principals[2:3], roleDefs[2:3], roleAssigns[2:3], vms[2:3]), + expectedDeletes: generateExpected(principals[0:1], roleDefs[0:1], roleAssigns[0:1], vms[0:1]), + }, + // Completely new results + { + oldResults: &Resources{ + Principals: nil, + RoleDefinitions: nil, + RoleAssignments: nil, + VirtualMachines: nil, + }, + newResults: &Resources{ + Principals: principals[1:3], + RoleDefinitions: roleDefs[1:3], + RoleAssignments: roleAssigns[1:3], + VirtualMachines: vms[1:3], + }, + expectedUpserts: generateExpected(principals[1:3], roleDefs[1:3], roleAssigns[1:3], vms[1:3]), + expectedDeletes: generateExpected(nil, nil, nil, nil), + }, + // No new results + { + oldResults: &Resources{ + Principals: principals[1:3], + RoleDefinitions: roleDefs[1:3], + RoleAssignments: roleAssigns[1:3], + VirtualMachines: vms[1:3], + }, + newResults: &Resources{ + Principals: nil, + RoleDefinitions: nil, + RoleAssignments: nil, + VirtualMachines: nil, + }, + expectedUpserts: generateExpected(nil, nil, nil, nil), + expectedDeletes: generateExpected(principals[1:3], roleDefs[1:3], roleAssigns[1:3], vms[1:3]), + }, + } + + for _, tt := range tests { + upserts, deletes := ReconcileResults(tt.oldResults, tt.newResults) + require.ElementsMatch(t, upserts.Resources, tt.expectedUpserts.Resources) + require.ElementsMatch(t, deletes.Resources, tt.expectedDeletes.Resources) + } + +} + +func generateExpected( + principals []*accessgraphv1alpha.AzurePrincipal, + roleDefs []*accessgraphv1alpha.AzureRoleDefinition, + roleAssigns []*accessgraphv1alpha.AzureRoleAssignment, + vms []*accessgraphv1alpha.AzureVirtualMachine, +) *accessgraphv1alpha.AzureResourceList { + resList := &accessgraphv1alpha.AzureResourceList{ + Resources: make([]*accessgraphv1alpha.AzureResource, 0), + } + for _, principal := range principals { + resList.Resources = append(resList.Resources, azurePrincipalsWrap(principal)) + } + for _, roleDef := range roleDefs { + resList.Resources = append(resList.Resources, azureRoleDefWrap(roleDef)) + } + for _, roleAssign := range roleAssigns { + resList.Resources = append(resList.Resources, azureRoleAssignWrap(roleAssign)) + } + for _, vm := range vms { + resList.Resources = append(resList.Resources, azureVmWrap(vm)) + } + return resList +} + +func generatePrincipals() []*accessgraphv1alpha.AzurePrincipal { + return []*accessgraphv1alpha.AzurePrincipal{ + { + Id: "/principals/foo", + DisplayName: "userFoo", + }, + { + Id: "/principals/bar", + DisplayName: "userBar", + }, + { + Id: "/principals/charles", + DisplayName: "userCharles", + }, + } +} + +func generateRoleDefs() []*accessgraphv1alpha.AzureRoleDefinition { + return []*accessgraphv1alpha.AzureRoleDefinition{ + { + Id: "/roledefinitions/foo", + Name: "roleFoo", + }, + { + Id: "/roledefinitions/bar", + Name: "roleBar", + }, + { + Id: "/roledefinitions/charles", + Name: "roleCharles", + }, + } +} + +func generateRoleAssigns() []*accessgraphv1alpha.AzureRoleAssignment { + return []*accessgraphv1alpha.AzureRoleAssignment{ + { + Id: "/roleassignments/foo", + PrincipalId: "userFoo", + }, + { + Id: "/roleassignments/bar", + PrincipalId: "userBar", + }, + { + Id: "/roleassignments/charles", + PrincipalId: "userCharles", + }, + } +} + +func generateVms() []*accessgraphv1alpha.AzureVirtualMachine { + return []*accessgraphv1alpha.AzureVirtualMachine{ + { + Id: "/vms/foo", + Name: "userFoo", + }, + { + Id: "/vms/bar", + Name: "userBar", + }, + { + Id: "/vms/charles", + Name: "userCharles", + }, + } +} diff --git a/lib/srv/discovery/fetchers/azure-sync/roleassignments.go b/lib/srv/discovery/fetchers/azure-sync/roleassignments.go new file mode 100644 index 0000000000000..58cfa89c8ae3e --- /dev/null +++ b/lib/srv/discovery/fetchers/azure-sync/roleassignments.go @@ -0,0 +1,63 @@ +/* + * Teleport + * Copyright (C) 2024 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package azure_sync + +import ( + "context" + "fmt" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v2" + + "github.com/gravitational/trace" + "google.golang.org/protobuf/types/known/timestamppb" + + accessgraphv1alpha "github.com/gravitational/teleport/gen/proto/go/accessgraph/v1alpha" +) + +// RoleAssignmentsClient specifies the methods used to fetch role assignments from Azure +type RoleAssignmentsClient interface { + ListRoleAssignments(ctx context.Context, scope string) ([]*armauthorization.RoleAssignment, error) +} + +// fetchRoleAssignments fetches Azure role assignments using the Azure role assignments API +func fetchRoleAssignments( + ctx context.Context, + subscriptionID string, + cli RoleAssignmentsClient, +) ([]*accessgraphv1alpha.AzureRoleAssignment, error) { + // List the role definitions + roleAssigns, err := cli.ListRoleAssignments(ctx, fmt.Sprintf("/subscriptions/%s", subscriptionID)) + if err != nil { + return nil, trace.Wrap(err) + } + + // Convert to protobuf format + pbRoleAssigns := make([]*accessgraphv1alpha.AzureRoleAssignment, 0, len(roleAssigns)) + for _, roleAssign := range roleAssigns { + pbRoleAssign := &accessgraphv1alpha.AzureRoleAssignment{ + Id: *roleAssign.ID, + SubscriptionId: subscriptionID, + LastSyncTime: timestamppb.Now(), + PrincipalId: *roleAssign.Properties.PrincipalID, + RoleDefinitionId: *roleAssign.Properties.RoleDefinitionID, + Scope: *roleAssign.Properties.Scope, + } + pbRoleAssigns = append(pbRoleAssigns, pbRoleAssign) + } + return pbRoleAssigns, nil +} diff --git a/lib/srv/discovery/fetchers/azure-sync/roledefinitions.go b/lib/srv/discovery/fetchers/azure-sync/roledefinitions.go new file mode 100644 index 0000000000000..3af31524f47b0 --- /dev/null +++ b/lib/srv/discovery/fetchers/azure-sync/roledefinitions.go @@ -0,0 +1,77 @@ +/* + * Teleport + * Copyright (C) 2024 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package azure_sync + +import ( + "context" + "fmt" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v2" + + "github.com/gravitational/trace" + "google.golang.org/protobuf/types/known/timestamppb" + + accessgraphv1alpha "github.com/gravitational/teleport/gen/proto/go/accessgraph/v1alpha" +) + +// RoleDefinitionsClient specifies the methods used to fetch roles from Azure +type RoleDefinitionsClient interface { + ListRoleDefinitions(ctx context.Context, scope string) ([]*armauthorization.RoleDefinition, error) +} + +func (a *Fetcher) fetchRoleDefinitions( + ctx context.Context, + subscriptionID string, + cli RoleDefinitionsClient, +) ([]*accessgraphv1alpha.AzureRoleDefinition, error) { + // List the role definitions + roleDefs, err := cli.ListRoleDefinitions(ctx, fmt.Sprintf("/subscriptions/%s", subscriptionID)) + if err != nil { + return nil, trace.Wrap(err) + } + + // Convert to protobuf format + pbRoleDefs := make([]*accessgraphv1alpha.AzureRoleDefinition, 0, len(roleDefs)) + for _, roleDef := range roleDefs { + pbPerms := make([]*accessgraphv1alpha.AzureRBACPermission, 0, len(roleDef.Properties.Permissions)) + for _, perm := range roleDef.Properties.Permissions { + pbPerm := accessgraphv1alpha.AzureRBACPermission{ + Actions: ptrsToList(perm.Actions), + NotActions: ptrsToList(perm.NotActions), + } + pbPerms = append(pbPerms, &pbPerm) + } + pbRoleDef := &accessgraphv1alpha.AzureRoleDefinition{ + Id: *roleDef.ID, + Name: *roleDef.Properties.RoleName, + SubscriptionId: a.SubscriptionID, + LastSyncTime: timestamppb.Now(), + Permissions: pbPerms, + } + pbRoleDefs = append(pbRoleDefs, pbRoleDef) + } + return pbRoleDefs, nil +} + +func ptrsToList(ptrs []*string) []string { + strList := make([]string, 0, len(ptrs)) + for _, ptr := range ptrs { + strList = append(strList, *ptr) + } + return strList +} diff --git a/lib/srv/discovery/fetchers/azure-sync/virtualmachines.go b/lib/srv/discovery/fetchers/azure-sync/virtualmachines.go new file mode 100644 index 0000000000000..39477cf096ade --- /dev/null +++ b/lib/srv/discovery/fetchers/azure-sync/virtualmachines.go @@ -0,0 +1,56 @@ +/* + * Teleport + * Copyright (C) 2024 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package azure_sync + +import ( + "context" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v6" + + "github.com/gravitational/trace" + "google.golang.org/protobuf/types/known/timestamppb" + + accessgraphv1alpha "github.com/gravitational/teleport/gen/proto/go/accessgraph/v1alpha" +) + +const allResourceGroups = "*" + +// VirtualMachinesClient specifies the methods used to fetch virtual machines from Azure +type VirtualMachinesClient interface { + ListVirtualMachines(ctx context.Context, resourceGroup string) ([]*armcompute.VirtualMachine, error) +} + +func fetchVirtualMachines(ctx context.Context, subscriptionID string, cli VirtualMachinesClient) ([]*accessgraphv1alpha.AzureVirtualMachine, error) { + vms, err := cli.ListVirtualMachines(ctx, allResourceGroups) + if err != nil { + return nil, trace.Wrap(err) + } + + // Return the VMs as protobuf messages + pbVms := make([]*accessgraphv1alpha.AzureVirtualMachine, 0, len(vms)) + for _, vm := range vms { + pbVm := accessgraphv1alpha.AzureVirtualMachine{ + Id: *vm.ID, + SubscriptionId: subscriptionID, + LastSyncTime: timestamppb.Now(), + Name: *vm.Name, + } + pbVms = append(pbVms, &pbVm) + } + return pbVms, nil +} From 646b1c17c3b244d3e642f002d45de06ea9d6c9e2 Mon Sep 17 00:00:00 2001 From: Matt Brock Date: Tue, 17 Dec 2024 16:58:20 -0600 Subject: [PATCH 03/12] Forgot to decouple role definitions fetching function from the fetcher --- lib/srv/discovery/fetchers/azure-sync/roledefinitions.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/srv/discovery/fetchers/azure-sync/roledefinitions.go b/lib/srv/discovery/fetchers/azure-sync/roledefinitions.go index 3af31524f47b0..35dfce444188c 100644 --- a/lib/srv/discovery/fetchers/azure-sync/roledefinitions.go +++ b/lib/srv/discovery/fetchers/azure-sync/roledefinitions.go @@ -34,7 +34,7 @@ type RoleDefinitionsClient interface { ListRoleDefinitions(ctx context.Context, scope string) ([]*armauthorization.RoleDefinition, error) } -func (a *Fetcher) fetchRoleDefinitions( +func fetchRoleDefinitions( ctx context.Context, subscriptionID string, cli RoleDefinitionsClient, @@ -59,7 +59,7 @@ func (a *Fetcher) fetchRoleDefinitions( pbRoleDef := &accessgraphv1alpha.AzureRoleDefinition{ Id: *roleDef.ID, Name: *roleDef.Properties.RoleName, - SubscriptionId: a.SubscriptionID, + SubscriptionId: subscriptionID, LastSyncTime: timestamppb.Now(), Permissions: pbPerms, } From 6d841aa3daa4c61921c67edb82c65f00663fb04d Mon Sep 17 00:00:00 2001 From: Matt Brock Date: Tue, 17 Dec 2024 17:12:53 -0600 Subject: [PATCH 04/12] Moving reconciliation to the upstream azure sync PR --- .../fetchers/azure-sync/reconcile.go | 165 ------------------ 1 file changed, 165 deletions(-) delete mode 100644 lib/srv/discovery/fetchers/azure-sync/reconcile.go diff --git a/lib/srv/discovery/fetchers/azure-sync/reconcile.go b/lib/srv/discovery/fetchers/azure-sync/reconcile.go deleted file mode 100644 index 2b54c8cfac911..0000000000000 --- a/lib/srv/discovery/fetchers/azure-sync/reconcile.go +++ /dev/null @@ -1,165 +0,0 @@ -/* - * Teleport - * Copyright (C) 2024 Gravitational, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package azure_sync - -import ( - "fmt" - - "google.golang.org/protobuf/proto" - - accessgraphv1alpha "github.com/gravitational/teleport/gen/proto/go/accessgraph/v1alpha" - "github.com/gravitational/teleport/lib/srv/discovery/common" -) - -// MergeResources merges Azure resources fetched from multiple configured Azure fetchers -func MergeResources(results ...*Resources) *Resources { - if len(results) == 0 { - return &Resources{} - } - if len(results) == 1 { - return results[0] - } - result := &Resources{} - for _, r := range results { - result.Principals = append(result.Principals, r.Principals...) - result.RoleAssignments = append(result.RoleAssignments, r.RoleAssignments...) - result.RoleDefinitions = append(result.RoleDefinitions, r.RoleDefinitions...) - result.VirtualMachines = append(result.VirtualMachines, r.VirtualMachines...) - } - result.Principals = common.DeduplicateSlice(result.Principals, azurePrincipalsKey) - result.RoleAssignments = common.DeduplicateSlice(result.RoleAssignments, azureRoleAssignKey) - result.RoleDefinitions = common.DeduplicateSlice(result.RoleDefinitions, azureRoleDefKey) - result.VirtualMachines = common.DeduplicateSlice(result.VirtualMachines, azureVmKey) - return result -} - -// newResourceList creates a new resource list message -func newResourceList() *accessgraphv1alpha.AzureResourceList { - return &accessgraphv1alpha.AzureResourceList{ - Resources: make([]*accessgraphv1alpha.AzureResource, 0), - } -} - -// ReconcileResults compares previously and currently fetched results and determines which resources to upsert and -// which to delete. -func ReconcileResults(old *Resources, new *Resources) (upsert, delete *accessgraphv1alpha.AzureResourceList) { - upsert, delete = newResourceList(), newResourceList() - reconciledResources := []*reconcilePair{ - reconcile(old.Principals, new.Principals, azurePrincipalsKey, azurePrincipalsWrap), - reconcile(old.RoleAssignments, new.RoleAssignments, azureRoleAssignKey, azureRoleAssignWrap), - reconcile(old.RoleDefinitions, new.RoleDefinitions, azureRoleDefKey, azureRoleDefWrap), - reconcile(old.VirtualMachines, new.VirtualMachines, azureVmKey, azureVmWrap), - } - for _, res := range reconciledResources { - upsert.Resources = append(upsert.Resources, res.upsert.Resources...) - delete.Resources = append(delete.Resources, res.delete.Resources...) - } - return upsert, delete -} - -// reconcilePair contains the Azure resources to upsert and delete -type reconcilePair struct { - upsert, delete *accessgraphv1alpha.AzureResourceList -} - -// reconcile compares old and new items to build a list of resources to upsert and delete in the Access Graph -func reconcile[T proto.Message]( - oldItems []T, - newItems []T, - keyFn func(T) string, - wrapFn func(T) *accessgraphv1alpha.AzureResource, -) *reconcilePair { - // Remove duplicates from the new items - newItems = common.DeduplicateSlice(newItems, keyFn) - upsertRes := newResourceList() - deleteRes := newResourceList() - - // Delete all old items if there are no new items - if len(newItems) == 0 { - for _, item := range oldItems { - deleteRes.Resources = append(deleteRes.Resources, wrapFn(item)) - } - return &reconcilePair{upsertRes, deleteRes} - } - - // Create all new items if there are no old items - if len(oldItems) == 0 { - for _, item := range newItems { - upsertRes.Resources = append(upsertRes.Resources, wrapFn(item)) - } - return &reconcilePair{upsertRes, deleteRes} - } - - // Map old and new items by their key - oldMap := make(map[string]T, len(oldItems)) - for _, item := range oldItems { - oldMap[keyFn(item)] = item - } - newMap := make(map[string]T, len(newItems)) - for _, item := range newItems { - newMap[keyFn(item)] = item - } - - // Append new or modified items to the upsert list - for _, item := range newItems { - if oldItem, ok := oldMap[keyFn(item)]; !ok || !proto.Equal(oldItem, item) { - upsertRes.Resources = append(upsertRes.Resources, wrapFn(item)) - } - } - - // Append removed items to the delete list - for _, item := range oldItems { - if _, ok := newMap[keyFn(item)]; !ok { - deleteRes.Resources = append(deleteRes.Resources, wrapFn(item)) - } - } - return &reconcilePair{upsertRes, deleteRes} -} - -func azurePrincipalsKey(user *accessgraphv1alpha.AzurePrincipal) string { - return fmt.Sprintf("%s:%s", user.SubscriptionId, user.Id) -} - -func azurePrincipalsWrap(principal *accessgraphv1alpha.AzurePrincipal) *accessgraphv1alpha.AzureResource { - return &accessgraphv1alpha.AzureResource{Resource: &accessgraphv1alpha.AzureResource_Principal{Principal: principal}} -} - -func azureRoleAssignKey(roleAssign *accessgraphv1alpha.AzureRoleAssignment) string { - return fmt.Sprintf("%s:%s", roleAssign.SubscriptionId, roleAssign.Id) -} - -func azureRoleAssignWrap(roleAssign *accessgraphv1alpha.AzureRoleAssignment) *accessgraphv1alpha.AzureResource { - return &accessgraphv1alpha.AzureResource{Resource: &accessgraphv1alpha.AzureResource_RoleAssignment{RoleAssignment: roleAssign}} -} - -func azureRoleDefKey(roleDef *accessgraphv1alpha.AzureRoleDefinition) string { - return fmt.Sprintf("%s:%s", roleDef.SubscriptionId, roleDef.Id) -} - -func azureRoleDefWrap(roleDef *accessgraphv1alpha.AzureRoleDefinition) *accessgraphv1alpha.AzureResource { - return &accessgraphv1alpha.AzureResource{Resource: &accessgraphv1alpha.AzureResource_RoleDefinition{RoleDefinition: roleDef}} -} - -func azureVmKey(vm *accessgraphv1alpha.AzureVirtualMachine) string { - return fmt.Sprintf("%s:%s", vm.SubscriptionId, vm.Id) -} - -func azureVmWrap(vm *accessgraphv1alpha.AzureVirtualMachine) *accessgraphv1alpha.AzureResource { - return &accessgraphv1alpha.AzureResource{Resource: &accessgraphv1alpha.AzureResource_VirtualMachine{VirtualMachine: vm}} -} From 47cbbc9e0d0a54d39e1ba9414b4444123de34e85 Mon Sep 17 00:00:00 2001 From: Matt Brock Date: Tue, 17 Dec 2024 17:15:42 -0600 Subject: [PATCH 05/12] Moving reconciliation test to the upstream azure sync PR --- .../fetchers/azure-sync/reconcile_test.go | 191 ------------------ 1 file changed, 191 deletions(-) delete mode 100644 lib/srv/discovery/fetchers/azure-sync/reconcile_test.go diff --git a/lib/srv/discovery/fetchers/azure-sync/reconcile_test.go b/lib/srv/discovery/fetchers/azure-sync/reconcile_test.go deleted file mode 100644 index 28b293bcf1f8d..0000000000000 --- a/lib/srv/discovery/fetchers/azure-sync/reconcile_test.go +++ /dev/null @@ -1,191 +0,0 @@ -/* - * Teleport - * Copyright (C) 2024 Gravitational, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package azure_sync - -import ( - "testing" - - "github.com/stretchr/testify/require" - - accessgraphv1alpha "github.com/gravitational/teleport/gen/proto/go/accessgraph/v1alpha" -) - -func TestReconcileResults(t *testing.T) { - principals := generatePrincipals() - roleDefs := generateRoleDefs() - roleAssigns := generateRoleAssigns() - vms := generateVms() - - tests := []struct { - oldResults *Resources - newResults *Resources - expectedUpserts *accessgraphv1alpha.AzureResourceList - expectedDeletes *accessgraphv1alpha.AzureResourceList - }{ - // Overlapping old and new results - { - oldResults: &Resources{ - Principals: principals[0:2], - RoleDefinitions: roleDefs[0:2], - RoleAssignments: roleAssigns[0:2], - VirtualMachines: vms[0:2], - }, - newResults: &Resources{ - Principals: principals[1:3], - RoleDefinitions: roleDefs[1:3], - RoleAssignments: roleAssigns[1:3], - VirtualMachines: vms[1:3], - }, - expectedUpserts: generateExpected(principals[2:3], roleDefs[2:3], roleAssigns[2:3], vms[2:3]), - expectedDeletes: generateExpected(principals[0:1], roleDefs[0:1], roleAssigns[0:1], vms[0:1]), - }, - // Completely new results - { - oldResults: &Resources{ - Principals: nil, - RoleDefinitions: nil, - RoleAssignments: nil, - VirtualMachines: nil, - }, - newResults: &Resources{ - Principals: principals[1:3], - RoleDefinitions: roleDefs[1:3], - RoleAssignments: roleAssigns[1:3], - VirtualMachines: vms[1:3], - }, - expectedUpserts: generateExpected(principals[1:3], roleDefs[1:3], roleAssigns[1:3], vms[1:3]), - expectedDeletes: generateExpected(nil, nil, nil, nil), - }, - // No new results - { - oldResults: &Resources{ - Principals: principals[1:3], - RoleDefinitions: roleDefs[1:3], - RoleAssignments: roleAssigns[1:3], - VirtualMachines: vms[1:3], - }, - newResults: &Resources{ - Principals: nil, - RoleDefinitions: nil, - RoleAssignments: nil, - VirtualMachines: nil, - }, - expectedUpserts: generateExpected(nil, nil, nil, nil), - expectedDeletes: generateExpected(principals[1:3], roleDefs[1:3], roleAssigns[1:3], vms[1:3]), - }, - } - - for _, tt := range tests { - upserts, deletes := ReconcileResults(tt.oldResults, tt.newResults) - require.ElementsMatch(t, upserts.Resources, tt.expectedUpserts.Resources) - require.ElementsMatch(t, deletes.Resources, tt.expectedDeletes.Resources) - } - -} - -func generateExpected( - principals []*accessgraphv1alpha.AzurePrincipal, - roleDefs []*accessgraphv1alpha.AzureRoleDefinition, - roleAssigns []*accessgraphv1alpha.AzureRoleAssignment, - vms []*accessgraphv1alpha.AzureVirtualMachine, -) *accessgraphv1alpha.AzureResourceList { - resList := &accessgraphv1alpha.AzureResourceList{ - Resources: make([]*accessgraphv1alpha.AzureResource, 0), - } - for _, principal := range principals { - resList.Resources = append(resList.Resources, azurePrincipalsWrap(principal)) - } - for _, roleDef := range roleDefs { - resList.Resources = append(resList.Resources, azureRoleDefWrap(roleDef)) - } - for _, roleAssign := range roleAssigns { - resList.Resources = append(resList.Resources, azureRoleAssignWrap(roleAssign)) - } - for _, vm := range vms { - resList.Resources = append(resList.Resources, azureVmWrap(vm)) - } - return resList -} - -func generatePrincipals() []*accessgraphv1alpha.AzurePrincipal { - return []*accessgraphv1alpha.AzurePrincipal{ - { - Id: "/principals/foo", - DisplayName: "userFoo", - }, - { - Id: "/principals/bar", - DisplayName: "userBar", - }, - { - Id: "/principals/charles", - DisplayName: "userCharles", - }, - } -} - -func generateRoleDefs() []*accessgraphv1alpha.AzureRoleDefinition { - return []*accessgraphv1alpha.AzureRoleDefinition{ - { - Id: "/roledefinitions/foo", - Name: "roleFoo", - }, - { - Id: "/roledefinitions/bar", - Name: "roleBar", - }, - { - Id: "/roledefinitions/charles", - Name: "roleCharles", - }, - } -} - -func generateRoleAssigns() []*accessgraphv1alpha.AzureRoleAssignment { - return []*accessgraphv1alpha.AzureRoleAssignment{ - { - Id: "/roleassignments/foo", - PrincipalId: "userFoo", - }, - { - Id: "/roleassignments/bar", - PrincipalId: "userBar", - }, - { - Id: "/roleassignments/charles", - PrincipalId: "userCharles", - }, - } -} - -func generateVms() []*accessgraphv1alpha.AzureVirtualMachine { - return []*accessgraphv1alpha.AzureVirtualMachine{ - { - Id: "/vms/foo", - Name: "userFoo", - }, - { - Id: "/vms/bar", - Name: "userBar", - }, - { - Id: "/vms/charles", - Name: "userCharles", - }, - } -} From eb6b32d7d3d4008a90b82e1f8ff75833c4e53e91 Mon Sep 17 00:00:00 2001 From: Matt Brock Date: Tue, 17 Dec 2024 17:29:20 -0600 Subject: [PATCH 06/12] Updating go.sum --- integrations/event-handler/go.mod | 1 + integrations/event-handler/go.sum | 2 ++ integrations/terraform/go.mod | 1 + integrations/terraform/go.sum | 2 ++ 4 files changed, 6 insertions(+) diff --git a/integrations/event-handler/go.mod b/integrations/event-handler/go.mod index 56ac4bd95bd73..a59328b43c21c 100644 --- a/integrations/event-handler/go.mod +++ b/integrations/event-handler/go.mod @@ -37,6 +37,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v2 v2.2.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v6 v6.1.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v6 v6.3.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi v1.2.0 // indirect diff --git a/integrations/event-handler/go.sum b/integrations/event-handler/go.sum index 2b0f134829fcd..c31d10c51aa2c 100644 --- a/integrations/event-handler/go.sum +++ b/integrations/event-handler/go.sum @@ -631,6 +631,8 @@ github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.0 h1:+m0M/LFxN43KvUL github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.0/go.mod h1:PwOyop78lveYMRs6oCxjiVyBdyCgIYH6XHIVZO9/SFQ= github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 h1:ywEEhmNahHBihViHepv3xPBn1663uRv2t2q/ESv9seY= github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0/go.mod h1:iZDifYGJTIgIIkYRNWPENUnqx6bJ2xnSDFI2tjwZNuY= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v2 v2.2.0 h1:Hp+EScFOu9HeCbeW8WU2yQPJd4gGwhMgKxWe+G6jNzw= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v2 v2.2.0/go.mod h1:/pz8dyNQe+Ey3yBp/XuYz7oqX8YDNWVpPB0hH3XWfbc= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v6 v6.1.0 h1:zDeQI/PaWztI2tcrGO/9RIMey9NvqYbnyttf/0P3QWM= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v6 v6.1.0/go.mod h1:zflC9v4VfViJrSvcvplqws/yGXVbUEMZi/iHpZdSPWA= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v5 v5.0.0 h1:5n7dPVqsWfVKw+ZiEKSd3Kzu7gwBkbEBkeXb8rgaE9Q= diff --git a/integrations/terraform/go.mod b/integrations/terraform/go.mod index e6e5624117945..9660a61254edb 100644 --- a/integrations/terraform/go.mod +++ b/integrations/terraform/go.mod @@ -44,6 +44,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v2 v2.2.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v6 v6.1.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v6 v6.3.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi v1.2.0 // indirect diff --git a/integrations/terraform/go.sum b/integrations/terraform/go.sum index e2d3ece037acc..7617dd825ad12 100644 --- a/integrations/terraform/go.sum +++ b/integrations/terraform/go.sum @@ -644,6 +644,8 @@ github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.0 h1:+m0M/LFxN43KvUL github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.0/go.mod h1:PwOyop78lveYMRs6oCxjiVyBdyCgIYH6XHIVZO9/SFQ= github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 h1:ywEEhmNahHBihViHepv3xPBn1663uRv2t2q/ESv9seY= github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0/go.mod h1:iZDifYGJTIgIIkYRNWPENUnqx6bJ2xnSDFI2tjwZNuY= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v2 v2.2.0 h1:Hp+EScFOu9HeCbeW8WU2yQPJd4gGwhMgKxWe+G6jNzw= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v2 v2.2.0/go.mod h1:/pz8dyNQe+Ey3yBp/XuYz7oqX8YDNWVpPB0hH3XWfbc= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v6 v6.1.0 h1:zDeQI/PaWztI2tcrGO/9RIMey9NvqYbnyttf/0P3QWM= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v6 v6.1.0/go.mod h1:zflC9v4VfViJrSvcvplqws/yGXVbUEMZi/iHpZdSPWA= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v5 v5.0.0 h1:5n7dPVqsWfVKw+ZiEKSd3Kzu7gwBkbEBkeXb8rgaE9Q= From 5c89fc7fff9741b2186e7b209bf0739989fffa16 Mon Sep 17 00:00:00 2001 From: Matt Brock Date: Wed, 18 Dec 2024 15:14:34 -0600 Subject: [PATCH 07/12] Fixing rebase after protobuf gen --- api/types/types.pb.go | 41 ----------------------------------------- 1 file changed, 41 deletions(-) diff --git a/api/types/types.pb.go b/api/types/types.pb.go index db2b270523755..b8eec25929456 100644 --- a/api/types/types.pb.go +++ b/api/types/types.pb.go @@ -50522,47 +50522,6 @@ func (m *AccessGraphAzureSync) MarshalTo(dAtA []byte) (int, error) { return m.MarshalToSizedBuffer(dAtA[:size]) } -func (m *AccessGraphAzureSync) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i - var l int - _ = l - if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) - } - if len(m.Integration) > 0 { - i -= len(m.Integration) - copy(dAtA[i:], m.Integration) - i = encodeVarintTypes(dAtA, i, uint64(len(m.Integration))) - i-- - dAtA[i] = 0x1a - } - if len(m.SubscriptionID) > 0 { - i -= len(m.SubscriptionID) - copy(dAtA[i:], m.SubscriptionID) - i = encodeVarintTypes(dAtA, i, uint64(len(m.SubscriptionID))) - i-- - dAtA[i] = 0x12 - } - return len(dAtA) - i, nil -} - -func (m *AccessGraphAzureSync) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *AccessGraphAzureSync) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - func (m *AccessGraphAzureSync) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i From 7c8c247ae1411754fc5e3bbbf956c3736df88451 Mon Sep 17 00:00:00 2001 From: Matt Brock Date: Thu, 19 Dec 2024 01:06:20 -0600 Subject: [PATCH 08/12] Nolinting until upstream PRs --- lib/srv/discovery/fetchers/azure-sync/principals.go | 8 ++++---- .../discovery/fetchers/azure-sync/roleassignments.go | 10 +++------- .../discovery/fetchers/azure-sync/roledefinitions.go | 12 ++++-------- .../discovery/fetchers/azure-sync/virtualmachines.go | 6 +++--- 4 files changed, 14 insertions(+), 22 deletions(-) diff --git a/lib/srv/discovery/fetchers/azure-sync/principals.go b/lib/srv/discovery/fetchers/azure-sync/principals.go index 850e0cb389f71..f20878e7e3a61 100644 --- a/lib/srv/discovery/fetchers/azure-sync/principals.go +++ b/lib/srv/discovery/fetchers/azure-sync/principals.go @@ -20,9 +20,9 @@ package azure_sync import ( "context" - "github.com/Azure/azure-sdk-for-go/sdk/azcore" "slices" + "github.com/Azure/azure-sdk-for-go/sdk/azcore" //nolint:unused // used in a dependent PR "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" "github.com/gravitational/trace" "google.golang.org/protobuf/types/known/timestamppb" @@ -30,11 +30,11 @@ import ( accessgraphv1alpha "github.com/gravitational/teleport/gen/proto/go/accessgraph/v1alpha" ) -const groupType = "#microsoft.graph.group" -const defaultGraphScope = "https://graph.microsoft.com/.default" +const groupType = "#microsoft.graph.group" //nolint:unused // used in a dependent PR +const defaultGraphScope = "https://graph.microsoft.com/.default" //nolint:unused // used in a dependent PR // fetchPrincipals fetches the Azure principals (users, groups, and service principals) using the Graph API -func fetchPrincipals(ctx context.Context, subscriptionID string, cred azcore.TokenCredential) ([]*accessgraphv1alpha.AzurePrincipal, error) { +func fetchPrincipals(ctx context.Context, subscriptionID string, cred azcore.TokenCredential) ([]*accessgraphv1alpha.AzurePrincipal, error) { //nolint:unused // used in a dependent PR // Get the graph client scopes := []string{defaultGraphScope} token, err := cred.GetToken(ctx, policy.TokenRequestOptions{Scopes: scopes}) diff --git a/lib/srv/discovery/fetchers/azure-sync/roleassignments.go b/lib/srv/discovery/fetchers/azure-sync/roleassignments.go index 58cfa89c8ae3e..0844b22a4fc94 100644 --- a/lib/srv/discovery/fetchers/azure-sync/roleassignments.go +++ b/lib/srv/discovery/fetchers/azure-sync/roleassignments.go @@ -20,9 +20,9 @@ package azure_sync import ( "context" - "fmt" - "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v2" + "fmt" //nolint:unused // used in a dependent PR + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v2" "github.com/gravitational/trace" "google.golang.org/protobuf/types/known/timestamppb" @@ -35,11 +35,7 @@ type RoleAssignmentsClient interface { } // fetchRoleAssignments fetches Azure role assignments using the Azure role assignments API -func fetchRoleAssignments( - ctx context.Context, - subscriptionID string, - cli RoleAssignmentsClient, -) ([]*accessgraphv1alpha.AzureRoleAssignment, error) { +func fetchRoleAssignments(ctx context.Context, subscriptionID string, cli RoleAssignmentsClient) ([]*accessgraphv1alpha.AzureRoleAssignment, error) { //nolint:unused // invoked in a dependent PR // List the role definitions roleAssigns, err := cli.ListRoleAssignments(ctx, fmt.Sprintf("/subscriptions/%s", subscriptionID)) if err != nil { diff --git a/lib/srv/discovery/fetchers/azure-sync/roledefinitions.go b/lib/srv/discovery/fetchers/azure-sync/roledefinitions.go index 35dfce444188c..03b5c38f9a056 100644 --- a/lib/srv/discovery/fetchers/azure-sync/roledefinitions.go +++ b/lib/srv/discovery/fetchers/azure-sync/roledefinitions.go @@ -20,9 +20,9 @@ package azure_sync import ( "context" - "fmt" - "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v2" + "fmt" //nolint:golint // used in a dependent PR + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v2" "github.com/gravitational/trace" "google.golang.org/protobuf/types/known/timestamppb" @@ -34,11 +34,7 @@ type RoleDefinitionsClient interface { ListRoleDefinitions(ctx context.Context, scope string) ([]*armauthorization.RoleDefinition, error) } -func fetchRoleDefinitions( - ctx context.Context, - subscriptionID string, - cli RoleDefinitionsClient, -) ([]*accessgraphv1alpha.AzureRoleDefinition, error) { +func fetchRoleDefinitions(ctx context.Context, subscriptionID string, cli RoleDefinitionsClient) ([]*accessgraphv1alpha.AzureRoleDefinition, error) { //nolint:unused // used in a dependent PR // List the role definitions roleDefs, err := cli.ListRoleDefinitions(ctx, fmt.Sprintf("/subscriptions/%s", subscriptionID)) if err != nil { @@ -68,7 +64,7 @@ func fetchRoleDefinitions( return pbRoleDefs, nil } -func ptrsToList(ptrs []*string) []string { +func ptrsToList(ptrs []*string) []string { //nolint:unused // used in a dependent PR strList := make([]string, 0, len(ptrs)) for _, ptr := range ptrs { strList = append(strList, *ptr) diff --git a/lib/srv/discovery/fetchers/azure-sync/virtualmachines.go b/lib/srv/discovery/fetchers/azure-sync/virtualmachines.go index 39477cf096ade..743061b5d0f9a 100644 --- a/lib/srv/discovery/fetchers/azure-sync/virtualmachines.go +++ b/lib/srv/discovery/fetchers/azure-sync/virtualmachines.go @@ -20,22 +20,22 @@ package azure_sync import ( "context" - "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v6" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v6" "github.com/gravitational/trace" "google.golang.org/protobuf/types/known/timestamppb" accessgraphv1alpha "github.com/gravitational/teleport/gen/proto/go/accessgraph/v1alpha" ) -const allResourceGroups = "*" +const allResourceGroups = "*" //nolint:unused // used in a dependent PR // VirtualMachinesClient specifies the methods used to fetch virtual machines from Azure type VirtualMachinesClient interface { ListVirtualMachines(ctx context.Context, resourceGroup string) ([]*armcompute.VirtualMachine, error) } -func fetchVirtualMachines(ctx context.Context, subscriptionID string, cli VirtualMachinesClient) ([]*accessgraphv1alpha.AzureVirtualMachine, error) { +func fetchVirtualMachines(ctx context.Context, subscriptionID string, cli VirtualMachinesClient) ([]*accessgraphv1alpha.AzureVirtualMachine, error) { //nolint:unused // invoked in a dependent PR vms, err := cli.ListVirtualMachines(ctx, allResourceGroups) if err != nil { return nil, trace.Wrap(err) From 672ae8d1f56a1fba7016c19e85dfe10b510eb711 Mon Sep 17 00:00:00 2001 From: Matt Brock Date: Thu, 19 Dec 2024 17:47:38 -0600 Subject: [PATCH 09/12] Updating to use existing msgraph client --- lib/msgraph/models.go | 10 +- lib/msgraph/paginated.go | 17 +- .../fetchers/azure-sync/msggraphclient.go | 240 ------------------ .../fetchers/azure-sync/principals.go | 92 ++++--- 4 files changed, 82 insertions(+), 277 deletions(-) delete mode 100644 lib/srv/discovery/fetchers/azure-sync/msggraphclient.go diff --git a/lib/msgraph/models.go b/lib/msgraph/models.go index 52c3e97cfec7b..4f2181f81055d 100644 --- a/lib/msgraph/models.go +++ b/lib/msgraph/models.go @@ -28,9 +28,15 @@ type GroupMember interface { isGroupMember() } +type Membership struct { + Type string `json:"@odata.type"` + ID string `json:"id"` +} + type DirectoryObject struct { - ID *string `json:"id,omitempty"` - DisplayName *string `json:"displayName,omitempty"` + ID *string `json:"id,omitempty"` + DisplayName *string `json:"displayName,omitempty"` + MemberOf []Membership `json:"memberOf,omitempty"` } type Group struct { diff --git a/lib/msgraph/paginated.go b/lib/msgraph/paginated.go index 51c587f19d074..cc25162ef849f 100644 --- a/lib/msgraph/paginated.go +++ b/lib/msgraph/paginated.go @@ -54,7 +54,14 @@ func iterateSimple[T any](c *Client, ctx context.Context, endpoint string, f fun func (c *Client) iterate(ctx context.Context, endpoint string, f func(json.RawMessage) bool) error { uri := *c.baseURL uri.Path = path.Join(uri.Path, endpoint) - uri.RawQuery = url.Values{"$top": {strconv.Itoa(c.pageSize)}}.Encode() + uri.RawQuery = url.Values{ + "$top": { + strconv.Itoa(c.pageSize), + }, + "$expand": { + "memberOf", + }, + }.Encode() uriString := uri.String() for uriString != "" { resp, err := c.request(ctx, http.MethodGet, uriString, nil) @@ -101,6 +108,14 @@ func (c *Client) IterateUsers(ctx context.Context, f func(*User) bool) error { return iterateSimple(c, ctx, "users", f) } +// IterateServicePrincipals lists all service principals in the Entra ID directory using pagination. +// `f` will be called for each object in the result set. +// if `f` returns `false`, the iteration is stopped (equivalent to `break` in a normal loop). +// Ref: [https://learn.microsoft.com/en-us/graph/api/user-list]. +func (c *Client) IterateServicePrincipals(ctx context.Context, f func(principal *ServicePrincipal) bool) error { + return iterateSimple(c, ctx, "servicePrincipals", f) +} + // IterateGroupMembers lists all members for the given Entra ID group using pagination. // `f` will be called for each object in the result set. // if `f` returns `false`, the iteration is stopped (equivalent to `break` in a normal loop). diff --git a/lib/srv/discovery/fetchers/azure-sync/msggraphclient.go b/lib/srv/discovery/fetchers/azure-sync/msggraphclient.go deleted file mode 100644 index 75d2960d7fa55..0000000000000 --- a/lib/srv/discovery/fetchers/azure-sync/msggraphclient.go +++ /dev/null @@ -1,240 +0,0 @@ -/* - * Teleport - * Copyright (C) 2024 Gravitational, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package azure_sync - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "net/url" - "strings" - "time" - - "github.com/Azure/azure-sdk-for-go/sdk/azcore" -) - -// GraphClient represents generic MS API client -type GraphClient struct { - token azcore.AccessToken -} - -const ( - usersSuffix = "users" - groupsSuffix = "groups" - servicePrincipalsSuffix = "servicePrincipals" - graphBaseURL = "https://graph.microsoft.com/v1.0" - httpTimeout = time.Second * 30 -) - -// graphError represents MS Graph error -type graphError struct { - E struct { - Code string `json:"code"` - Message string `json:"message"` - } `json:"error"` -} - -// genericGraphResponse represents the utility struct for parsing MS Graph API response -type genericGraphResponse struct { - Context string `json:"@odata.context"` - Count int `json:"@odata.count"` - NextLink string `json:"@odata.nextLink"` - Value json.RawMessage `json:"value"` -} - -// User represents user resource -type User struct { - ID string `json:"id"` - Name string `json:"displayName"` - MemberOf []Membership `json:"memberOf"` -} - -type Membership struct { - Type string `json:"@odata.type"` - ID string `json:"id"` -} - -// request represents generic request structure -type request struct { - // Method HTTP method - Method string - // URL which overrides URL construction - URL *string - // Path to a resource - Path string - // Expand $expand value - Expand []string - // Filter $filter value - Filter string - // Body request body - Body string - // Response represents template structure for a response - Response interface{} - // Err represents template structure for an error - Err error - // SuccessCode http code representing success - SuccessCode int -} - -// GetURL builds the request URL -func (r *request) GetURL() (string, error) { - if r.URL != nil { - return *r.URL, nil - } - u, err := url.Parse(graphBaseURL) - if err != nil { - return "", err - } - - data := url.Values{} - if len(r.Expand) > 0 { - data.Set("$expand", strings.Join(r.Expand, ",")) - } - if r.Filter != "" { - data.Set("$filter", r.Filter) - } - - u.Path = u.Path + "/" + r.Path - u.RawQuery = data.Encode() - - return u.String(), nil -} - -// NewGraphClient creates MS Graph API client -func NewGraphClient(token azcore.AccessToken) *GraphClient { - return &GraphClient{ - token: token, - } -} - -// Error returns error string -func (e graphError) Error() string { - return e.E.Code + " " + e.E.Message -} - -func (c *GraphClient) ListUsers(ctx context.Context) ([]User, error) { - return c.listIdentities(ctx, usersSuffix, []string{"memberOf"}) -} - -func (c *GraphClient) ListGroups(ctx context.Context) ([]User, error) { - return c.listIdentities(ctx, groupsSuffix, []string{"memberOf"}) -} - -func (c *GraphClient) ListServicePrincipals(ctx context.Context) ([]User, error) { - return c.listIdentities(ctx, servicePrincipalsSuffix, []string{"memberOf"}) -} - -func (c *GraphClient) listIdentities(ctx context.Context, idType string, expand []string) ([]User, error) { - var users []User - var nextLink *string - for { - g := &genericGraphResponse{} - req := request{ - Method: http.MethodGet, - Path: idType, - Expand: expand, - Response: &g, - Err: &graphError{}, - URL: nextLink, - } - err := c.request(ctx, req) - if err != nil { - return nil, err - } - var newUsers []User - err = json.NewDecoder(bytes.NewReader(g.Value)).Decode(&newUsers) - if err != nil { - return nil, err - } - users = append(users, newUsers...) - if g.NextLink == "" { - break - } - nextLink = &g.NextLink - } - - return users, nil -} - -// request sends the request to the graph/bot service and returns response body as bytes slice -func (c *GraphClient) request(ctx context.Context, req request) error { - reqUrl, err := req.GetURL() - if err != nil { - return err - } - - r, err := http.NewRequestWithContext(ctx, req.Method, reqUrl, strings.NewReader(req.Body)) - if err != nil { - return err - } - - r.Header.Set("Authorization", "Bearer "+c.token.Token) - r.Header.Set("Content-Type", "application/json") - - client := http.Client{Timeout: httpTimeout} - resp, err := client.Do(r) - if err != nil { - return err - } - - defer func(r *http.Response) { - _ = r.Body.Close() - }(resp) - - b, err := io.ReadAll(resp.Body) - if err != nil { - return err - } - - expectedCode := req.SuccessCode - if expectedCode == 0 { - expectedCode = http.StatusOK - } - - if expectedCode == resp.StatusCode { - if req.Response == nil { - return nil - } - - err := json.NewDecoder(bytes.NewReader(b)).Decode(req.Response) - if err != nil { - return err - } - } else { - if req.Err == nil { - return fmt.Errorf("Error requesting MS Graph API: %v", string(b)) - } - - err := json.NewDecoder(bytes.NewReader(b)).Decode(req.Err) - if err != nil { - return err - } - - if req.Err.Error() == "" { - return fmt.Errorf("Error requesting MS Graph API. Expected response code was %v, but is %v", expectedCode, resp.StatusCode) - } - - return req.Err - } - - return nil -} diff --git a/lib/srv/discovery/fetchers/azure-sync/principals.go b/lib/srv/discovery/fetchers/azure-sync/principals.go index f20878e7e3a61..757c78255ed46 100644 --- a/lib/srv/discovery/fetchers/azure-sync/principals.go +++ b/lib/srv/discovery/fetchers/azure-sync/principals.go @@ -20,63 +20,87 @@ package azure_sync import ( "context" - "slices" - - "github.com/Azure/azure-sdk-for-go/sdk/azcore" //nolint:unused // used in a dependent PR - "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/gravitational/teleport/lib/msgraph" "github.com/gravitational/trace" "google.golang.org/protobuf/types/known/timestamppb" accessgraphv1alpha "github.com/gravitational/teleport/gen/proto/go/accessgraph/v1alpha" ) -const groupType = "#microsoft.graph.group" //nolint:unused // used in a dependent PR -const defaultGraphScope = "https://graph.microsoft.com/.default" //nolint:unused // used in a dependent PR - // fetchPrincipals fetches the Azure principals (users, groups, and service principals) using the Graph API -func fetchPrincipals(ctx context.Context, subscriptionID string, cred azcore.TokenCredential) ([]*accessgraphv1alpha.AzurePrincipal, error) { //nolint:unused // used in a dependent PR - // Get the graph client - scopes := []string{defaultGraphScope} - token, err := cred.GetToken(ctx, policy.TokenRequestOptions{Scopes: scopes}) +func fetchPrincipals(ctx context.Context, subscriptionID string, cli *msgraph.Client) ([]*accessgraphv1alpha.AzurePrincipal, error) { + // Fetch the users, groups, and service principals + var users []*msgraph.User + err := cli.IterateUsers(ctx, func(user *msgraph.User) bool { + users = append(users, user) + return true + }) if err != nil { return nil, trace.Wrap(err) } - cli := NewGraphClient(token) - // Fetch the users, groups, and managed identities - users, err := cli.ListUsers(ctx) - if err != nil { - return nil, trace.Wrap(err) - } - groups, err := cli.ListGroups(ctx) + var groups []*msgraph.Group + err = cli.IterateGroups(ctx, func(group *msgraph.Group) bool { + groups = append(groups, group) + return true + }) if err != nil { return nil, trace.Wrap(err) } - svcPrincipals, err := cli.ListServicePrincipals(ctx) + + var servicePrincipals []*msgraph.ServicePrincipal + err = cli.IterateServicePrincipals(ctx, func(servicePrincipal *msgraph.ServicePrincipal) bool { + servicePrincipals = append(servicePrincipals, servicePrincipal) + return true + }) if err != nil { return nil, trace.Wrap(err) } - principals := slices.Concat(users, groups, svcPrincipals) - // Return the users as protobuf messages - pbPrincipals := make([]*accessgraphv1alpha.AzurePrincipal, 0, len(principals)) - for _, principal := range principals { - // Extract group membership - memberOf := make([]string, 0) - for _, member := range principal.MemberOf { - if member.Type == groupType { - memberOf = append(memberOf, member.ID) - } + // Return the users, groups, and service principals as protobuf messages + var pbPrincipals []*accessgraphv1alpha.AzurePrincipal + for _, user := range users { + var memberOf []string + for _, member := range user.MemberOf { + memberOf = append(memberOf, member.ID) + } + pbPrincipals = append(pbPrincipals, &accessgraphv1alpha.AzurePrincipal{ + Id: *user.ID, + SubscriptionId: subscriptionID, + LastSyncTime: timestamppb.Now(), + DisplayName: *user.DisplayName, + MemberOf: memberOf, + ObjectType: "user", + }) + } + for _, group := range groups { + var memberOf []string + for _, member := range group.MemberOf { + memberOf = append(memberOf, member.ID) } - // Create the protobuf principal and append it to the list - pbPrincipal := &accessgraphv1alpha.AzurePrincipal{ - Id: principal.ID, + pbPrincipals = append(pbPrincipals, &accessgraphv1alpha.AzurePrincipal{ + Id: *group.ID, SubscriptionId: subscriptionID, LastSyncTime: timestamppb.Now(), - DisplayName: principal.Name, + DisplayName: *group.DisplayName, MemberOf: memberOf, + ObjectType: "group", + }) + } + for _, sp := range servicePrincipals { + var memberOf []string + for _, member := range sp.MemberOf { + memberOf = append(memberOf, member.ID) } - pbPrincipals = append(pbPrincipals, pbPrincipal) + pbPrincipals = append(pbPrincipals, &accessgraphv1alpha.AzurePrincipal{ + Id: *sp.ID, + SubscriptionId: subscriptionID, + LastSyncTime: timestamppb.Now(), + DisplayName: *sp.DisplayName, + MemberOf: memberOf, + ObjectType: "servicePrincipal", + }) } + return pbPrincipals, nil } From 9a96e07d62a1dd0bd2fea2d4b18b94b89b2c36d9 Mon Sep 17 00:00:00 2001 From: Matt Brock Date: Thu, 19 Dec 2024 20:54:10 -0600 Subject: [PATCH 10/12] Adding protection around nil values --- .../fetchers/azure-sync/principals.go | 21 +++++++++++++++---- .../fetchers/azure-sync/roleassignments.go | 13 ++++++++++-- .../fetchers/azure-sync/roledefinitions.go | 18 ++++++++++++++-- .../fetchers/azure-sync/virtualmachines.go | 7 ++++++- 4 files changed, 50 insertions(+), 9 deletions(-) diff --git a/lib/srv/discovery/fetchers/azure-sync/principals.go b/lib/srv/discovery/fetchers/azure-sync/principals.go index 757c78255ed46..0cd9572f06212 100644 --- a/lib/srv/discovery/fetchers/azure-sync/principals.go +++ b/lib/srv/discovery/fetchers/azure-sync/principals.go @@ -20,15 +20,16 @@ package azure_sync import ( "context" - "github.com/gravitational/teleport/lib/msgraph" + "github.com/gravitational/trace" "google.golang.org/protobuf/types/known/timestamppb" accessgraphv1alpha "github.com/gravitational/teleport/gen/proto/go/accessgraph/v1alpha" + "github.com/gravitational/teleport/lib/msgraph" ) // fetchPrincipals fetches the Azure principals (users, groups, and service principals) using the Graph API -func fetchPrincipals(ctx context.Context, subscriptionID string, cli *msgraph.Client) ([]*accessgraphv1alpha.AzurePrincipal, error) { +func fetchPrincipals(ctx context.Context, subscriptionID string, cli *msgraph.Client) ([]*accessgraphv1alpha.AzurePrincipal, error) { //nolint: unused // invoked in a dependent PR // Fetch the users, groups, and service principals var users []*msgraph.User err := cli.IterateUsers(ctx, func(user *msgraph.User) bool { @@ -58,8 +59,13 @@ func fetchPrincipals(ctx context.Context, subscriptionID string, cli *msgraph.Cl } // Return the users, groups, and service principals as protobuf messages + var fetchErrs []error var pbPrincipals []*accessgraphv1alpha.AzurePrincipal for _, user := range users { + if user.ID == nil || user.DisplayName == nil { + fetchErrs = append(fetchErrs, trace.BadParameter("nil values on msgraph User object: %v", user)) + continue + } var memberOf []string for _, member := range user.MemberOf { memberOf = append(memberOf, member.ID) @@ -74,6 +80,10 @@ func fetchPrincipals(ctx context.Context, subscriptionID string, cli *msgraph.Cl }) } for _, group := range groups { + if group.ID == nil || group.DisplayName == nil { + fetchErrs = append(fetchErrs, trace.BadParameter("nil values on msgraph User object: %v", group)) + continue + } var memberOf []string for _, member := range group.MemberOf { memberOf = append(memberOf, member.ID) @@ -88,6 +98,10 @@ func fetchPrincipals(ctx context.Context, subscriptionID string, cli *msgraph.Cl }) } for _, sp := range servicePrincipals { + if sp.ID == nil || sp.DisplayName == nil { + fetchErrs = append(fetchErrs, trace.BadParameter("nil values on msgraph User object: %v", sp)) + continue + } var memberOf []string for _, member := range sp.MemberOf { memberOf = append(memberOf, member.ID) @@ -101,6 +115,5 @@ func fetchPrincipals(ctx context.Context, subscriptionID string, cli *msgraph.Cl ObjectType: "servicePrincipal", }) } - - return pbPrincipals, nil + return pbPrincipals, trace.NewAggregate(fetchErrs...) } diff --git a/lib/srv/discovery/fetchers/azure-sync/roleassignments.go b/lib/srv/discovery/fetchers/azure-sync/roleassignments.go index 0844b22a4fc94..4490346e15047 100644 --- a/lib/srv/discovery/fetchers/azure-sync/roleassignments.go +++ b/lib/srv/discovery/fetchers/azure-sync/roleassignments.go @@ -20,7 +20,7 @@ package azure_sync import ( "context" - "fmt" //nolint:unused // used in a dependent PR + "fmt" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v2" "github.com/gravitational/trace" @@ -44,7 +44,16 @@ func fetchRoleAssignments(ctx context.Context, subscriptionID string, cli RoleAs // Convert to protobuf format pbRoleAssigns := make([]*accessgraphv1alpha.AzureRoleAssignment, 0, len(roleAssigns)) + var fetchErrs []error for _, roleAssign := range roleAssigns { + if roleAssign.ID == nil || + roleAssign.Properties == nil || + roleAssign.Properties.PrincipalID == nil || + roleAssign.Properties.Scope == nil { + fetchErrs = append(fetchErrs, + trace.BadParameter("nil values on AzureRoleAssignment object: %v", roleAssign)) + continue + } pbRoleAssign := &accessgraphv1alpha.AzureRoleAssignment{ Id: *roleAssign.ID, SubscriptionId: subscriptionID, @@ -55,5 +64,5 @@ func fetchRoleAssignments(ctx context.Context, subscriptionID string, cli RoleAs } pbRoleAssigns = append(pbRoleAssigns, pbRoleAssign) } - return pbRoleAssigns, nil + return pbRoleAssigns, trace.NewAggregate(fetchErrs...) } diff --git a/lib/srv/discovery/fetchers/azure-sync/roledefinitions.go b/lib/srv/discovery/fetchers/azure-sync/roledefinitions.go index 03b5c38f9a056..889c9664f00bc 100644 --- a/lib/srv/discovery/fetchers/azure-sync/roledefinitions.go +++ b/lib/srv/discovery/fetchers/azure-sync/roledefinitions.go @@ -43,9 +43,21 @@ func fetchRoleDefinitions(ctx context.Context, subscriptionID string, cli RoleDe // Convert to protobuf format pbRoleDefs := make([]*accessgraphv1alpha.AzureRoleDefinition, 0, len(roleDefs)) + var fetchErrs []error for _, roleDef := range roleDefs { + if roleDef.ID == nil || + roleDef.Properties == nil || + roleDef.Properties.Permissions == nil || + roleDef.Properties.RoleName == nil { + fetchErrs = append(fetchErrs, trace.BadParameter("nil values on AzureRoleDefinition object: %v", roleDef)) + continue + } pbPerms := make([]*accessgraphv1alpha.AzureRBACPermission, 0, len(roleDef.Properties.Permissions)) for _, perm := range roleDef.Properties.Permissions { + if perm.Actions == nil || perm.NotActions == nil { + fetchErrs = append(fetchErrs, trace.BadParameter("nil values on Permission object: %v", perm)) + continue + } pbPerm := accessgraphv1alpha.AzureRBACPermission{ Actions: ptrsToList(perm.Actions), NotActions: ptrsToList(perm.NotActions), @@ -61,13 +73,15 @@ func fetchRoleDefinitions(ctx context.Context, subscriptionID string, cli RoleDe } pbRoleDefs = append(pbRoleDefs, pbRoleDef) } - return pbRoleDefs, nil + return pbRoleDefs, trace.NewAggregate(fetchErrs...) } func ptrsToList(ptrs []*string) []string { //nolint:unused // used in a dependent PR strList := make([]string, 0, len(ptrs)) for _, ptr := range ptrs { - strList = append(strList, *ptr) + if ptr != nil { + strList = append(strList, *ptr) + } } return strList } diff --git a/lib/srv/discovery/fetchers/azure-sync/virtualmachines.go b/lib/srv/discovery/fetchers/azure-sync/virtualmachines.go index 743061b5d0f9a..8e4de7a7c2971 100644 --- a/lib/srv/discovery/fetchers/azure-sync/virtualmachines.go +++ b/lib/srv/discovery/fetchers/azure-sync/virtualmachines.go @@ -43,7 +43,12 @@ func fetchVirtualMachines(ctx context.Context, subscriptionID string, cli Virtua // Return the VMs as protobuf messages pbVms := make([]*accessgraphv1alpha.AzureVirtualMachine, 0, len(vms)) + var fetchErrs []error for _, vm := range vms { + if vm.ID == nil || vm.Name == nil { + fetchErrs = append(fetchErrs, trace.BadParameter("nil values on AzureVirtualMachine object: %v", vm)) + continue + } pbVm := accessgraphv1alpha.AzureVirtualMachine{ Id: *vm.ID, SubscriptionId: subscriptionID, @@ -52,5 +57,5 @@ func fetchVirtualMachines(ctx context.Context, subscriptionID string, cli Virtua } pbVms = append(pbVms, &pbVm) } - return pbVms, nil + return pbVms, trace.NewAggregate(fetchErrs...) } From 0b12f0d81421e404f147da5bdc8e782e89b9aba1 Mon Sep 17 00:00:00 2001 From: Matt Brock Date: Fri, 20 Dec 2024 11:28:30 -0600 Subject: [PATCH 11/12] PR feedback --- lib/integrations/azureoidc/accessgraph.go | 2 +- lib/msgraph/client_test.go | 4 +- lib/msgraph/paginated.go | 39 +++++----- .../fetchers/azure-sync/principals.go | 73 +++++-------------- 4 files changed, 44 insertions(+), 74 deletions(-) diff --git a/lib/integrations/azureoidc/accessgraph.go b/lib/integrations/azureoidc/accessgraph.go index 6cf43a41c3291..d7462f32aca58 100644 --- a/lib/integrations/azureoidc/accessgraph.go +++ b/lib/integrations/azureoidc/accessgraph.go @@ -116,7 +116,7 @@ func CreateTAGCacheFile(ctx context.Context) error { } cache := &TAGInfoCache{} - err = graphClient.IterateApplications(ctx, func(app *msgraph.Application) bool { + err = graphClient.IterateApplications(ctx, nil, func(app *msgraph.Application) bool { appID := app.AppID if appID == nil { slog.WarnContext(ctx, "app ID is nil", "app", app) diff --git a/lib/msgraph/client_test.go b/lib/msgraph/client_test.go index 174b8f924ce14..c302d91cc2f6f 100644 --- a/lib/msgraph/client_test.go +++ b/lib/msgraph/client_test.go @@ -186,7 +186,7 @@ func TestIterateUsers_Empty(t *testing.T) { baseURL: uri, pageSize: defaultPageSize, } - err = client.IterateUsers(context.Background(), func(*User) bool { + err = client.IterateUsers(context.Background(), nil, func(*User) bool { assert.Fail(t, "should never get called") return true }) @@ -215,7 +215,7 @@ func TestIterateUsers(t *testing.T) { } var users []*User - err = client.IterateUsers(context.Background(), func(u *User) bool { + err = client.IterateUsers(context.Background(), nil, func(u *User) bool { users = append(users, u) return true }) diff --git a/lib/msgraph/paginated.go b/lib/msgraph/paginated.go index cc25162ef849f..89140b1879c5f 100644 --- a/lib/msgraph/paginated.go +++ b/lib/msgraph/paginated.go @@ -30,9 +30,9 @@ import ( ) // iterateSimple implements pagination for "simple" object lists, where additional logic isn't needed -func iterateSimple[T any](c *Client, ctx context.Context, endpoint string, f func(*T) bool) error { +func iterateSimple[T any](c *Client, ctx context.Context, endpoint string, params *url.Values, f func(*T) bool) error { var err error - itErr := c.iterate(ctx, endpoint, func(msg json.RawMessage) bool { + itErr := c.iterate(ctx, endpoint, params, func(msg json.RawMessage) bool { var page []T if err = json.Unmarshal(msg, &page); err != nil { return false @@ -51,17 +51,22 @@ func iterateSimple[T any](c *Client, ctx context.Context, endpoint string, f fun } // iterate implements pagination for "list" endpoints. -func (c *Client) iterate(ctx context.Context, endpoint string, f func(json.RawMessage) bool) error { +func (c *Client) iterate(ctx context.Context, endpoint string, params *url.Values, f func(json.RawMessage) bool) error { uri := *c.baseURL uri.Path = path.Join(uri.Path, endpoint) - uri.RawQuery = url.Values{ + rawQuery := url.Values{ "$top": { strconv.Itoa(c.pageSize), }, - "$expand": { - "memberOf", - }, - }.Encode() + } + if params != nil { + for key, values := range *params { + for _, value := range values { + rawQuery.Add(key, value) + } + } + } + uri.RawQuery = rawQuery.Encode() uriString := uri.String() for uriString != "" { resp, err := c.request(ctx, http.MethodGet, uriString, nil) @@ -88,32 +93,32 @@ func (c *Client) iterate(ctx context.Context, endpoint string, f func(json.RawMe // `f` will be called for each object in the result set. // if `f` returns `false`, the iteration is stopped (equivalent to `break` in a normal loop). // Ref: [https://learn.microsoft.com/en-us/graph/api/application-list]. -func (c *Client) IterateApplications(ctx context.Context, f func(*Application) bool) error { - return iterateSimple(c, ctx, "applications", f) +func (c *Client) IterateApplications(ctx context.Context, params *url.Values, f func(*Application) bool) error { + return iterateSimple(c, ctx, "applications", params, f) } // IterateGroups lists all groups in the Entra ID directory using pagination. // `f` will be called for each object in the result set. // if `f` returns `false`, the iteration is stopped (equivalent to `break` in a normal loop). // Ref: [https://learn.microsoft.com/en-us/graph/api/group-list]. -func (c *Client) IterateGroups(ctx context.Context, f func(*Group) bool) error { - return iterateSimple(c, ctx, "groups", f) +func (c *Client) IterateGroups(ctx context.Context, params *url.Values, f func(*Group) bool) error { + return iterateSimple(c, ctx, "groups", params, f) } // IterateUsers lists all users in the Entra ID directory using pagination. // `f` will be called for each object in the result set. // if `f` returns `false`, the iteration is stopped (equivalent to `break` in a normal loop). // Ref: [https://learn.microsoft.com/en-us/graph/api/user-list]. -func (c *Client) IterateUsers(ctx context.Context, f func(*User) bool) error { - return iterateSimple(c, ctx, "users", f) +func (c *Client) IterateUsers(ctx context.Context, params *url.Values, f func(*User) bool) error { + return iterateSimple(c, ctx, "users", params, f) } // IterateServicePrincipals lists all service principals in the Entra ID directory using pagination. // `f` will be called for each object in the result set. // if `f` returns `false`, the iteration is stopped (equivalent to `break` in a normal loop). // Ref: [https://learn.microsoft.com/en-us/graph/api/user-list]. -func (c *Client) IterateServicePrincipals(ctx context.Context, f func(principal *ServicePrincipal) bool) error { - return iterateSimple(c, ctx, "servicePrincipals", f) +func (c *Client) IterateServicePrincipals(ctx context.Context, params *url.Values, f func(principal *ServicePrincipal) bool) error { + return iterateSimple(c, ctx, "servicePrincipals", params, f) } // IterateGroupMembers lists all members for the given Entra ID group using pagination. @@ -122,7 +127,7 @@ func (c *Client) IterateServicePrincipals(ctx context.Context, f func(principal // Ref: [https://learn.microsoft.com/en-us/graph/api/group-list-members]. func (c *Client) IterateGroupMembers(ctx context.Context, groupID string, f func(GroupMember) bool) error { var err error - itErr := c.iterate(ctx, path.Join("groups", groupID, "members"), func(msg json.RawMessage) bool { + itErr := c.iterate(ctx, path.Join("groups", groupID, "members"), nil, func(msg json.RawMessage) bool { var page []json.RawMessage if err = json.Unmarshal(msg, &page); err != nil { return false diff --git a/lib/srv/discovery/fetchers/azure-sync/principals.go b/lib/srv/discovery/fetchers/azure-sync/principals.go index 0cd9572f06212..ab8ae3e48bb64 100644 --- a/lib/srv/discovery/fetchers/azure-sync/principals.go +++ b/lib/srv/discovery/fetchers/azure-sync/principals.go @@ -20,6 +20,7 @@ package azure_sync import ( "context" + "net/url" "github.com/gravitational/trace" "google.golang.org/protobuf/types/known/timestamppb" @@ -30,28 +31,28 @@ import ( // fetchPrincipals fetches the Azure principals (users, groups, and service principals) using the Graph API func fetchPrincipals(ctx context.Context, subscriptionID string, cli *msgraph.Client) ([]*accessgraphv1alpha.AzurePrincipal, error) { //nolint: unused // invoked in a dependent PR - // Fetch the users, groups, and service principals - var users []*msgraph.User - err := cli.IterateUsers(ctx, func(user *msgraph.User) bool { - users = append(users, user) + var params = &url.Values{ + "$expand": []string{"memberOf"}, + } + + // Fetch the users, groups, and service principals as directory objects + var dirObjs []msgraph.DirectoryObject + err := cli.IterateUsers(ctx, params, func(user *msgraph.User) bool { + dirObjs = append(dirObjs, user.DirectoryObject) return true }) if err != nil { return nil, trace.Wrap(err) } - - var groups []*msgraph.Group - err = cli.IterateGroups(ctx, func(group *msgraph.Group) bool { - groups = append(groups, group) + err = cli.IterateGroups(ctx, params, func(group *msgraph.Group) bool { + dirObjs = append(dirObjs, group.DirectoryObject) return true }) if err != nil { return nil, trace.Wrap(err) } - - var servicePrincipals []*msgraph.ServicePrincipal - err = cli.IterateServicePrincipals(ctx, func(servicePrincipal *msgraph.ServicePrincipal) bool { - servicePrincipals = append(servicePrincipals, servicePrincipal) + err = cli.IterateServicePrincipals(ctx, params, func(servicePrincipal *msgraph.ServicePrincipal) bool { + dirObjs = append(dirObjs, servicePrincipal.DirectoryObject) return true }) if err != nil { @@ -61,59 +62,23 @@ func fetchPrincipals(ctx context.Context, subscriptionID string, cli *msgraph.Cl // Return the users, groups, and service principals as protobuf messages var fetchErrs []error var pbPrincipals []*accessgraphv1alpha.AzurePrincipal - for _, user := range users { - if user.ID == nil || user.DisplayName == nil { - fetchErrs = append(fetchErrs, trace.BadParameter("nil values on msgraph User object: %v", user)) + for _, dirObj := range dirObjs { + if dirObj.ID == nil || dirObj.DisplayName == nil { + fetchErrs = append(fetchErrs, trace.BadParameter("nil values on msgraph directory object: %v", dirObj)) continue } var memberOf []string - for _, member := range user.MemberOf { + for _, member := range dirObj.MemberOf { memberOf = append(memberOf, member.ID) } pbPrincipals = append(pbPrincipals, &accessgraphv1alpha.AzurePrincipal{ - Id: *user.ID, + Id: *dirObj.ID, SubscriptionId: subscriptionID, LastSyncTime: timestamppb.Now(), - DisplayName: *user.DisplayName, + DisplayName: *dirObj.DisplayName, MemberOf: memberOf, ObjectType: "user", }) } - for _, group := range groups { - if group.ID == nil || group.DisplayName == nil { - fetchErrs = append(fetchErrs, trace.BadParameter("nil values on msgraph User object: %v", group)) - continue - } - var memberOf []string - for _, member := range group.MemberOf { - memberOf = append(memberOf, member.ID) - } - pbPrincipals = append(pbPrincipals, &accessgraphv1alpha.AzurePrincipal{ - Id: *group.ID, - SubscriptionId: subscriptionID, - LastSyncTime: timestamppb.Now(), - DisplayName: *group.DisplayName, - MemberOf: memberOf, - ObjectType: "group", - }) - } - for _, sp := range servicePrincipals { - if sp.ID == nil || sp.DisplayName == nil { - fetchErrs = append(fetchErrs, trace.BadParameter("nil values on msgraph User object: %v", sp)) - continue - } - var memberOf []string - for _, member := range sp.MemberOf { - memberOf = append(memberOf, member.ID) - } - pbPrincipals = append(pbPrincipals, &accessgraphv1alpha.AzurePrincipal{ - Id: *sp.ID, - SubscriptionId: subscriptionID, - LastSyncTime: timestamppb.Now(), - DisplayName: *sp.DisplayName, - MemberOf: memberOf, - ObjectType: "servicePrincipal", - }) - } return pbPrincipals, trace.NewAggregate(fetchErrs...) } From 4b21960db1f2688d197d2576a055b25d0c7c1b03 Mon Sep 17 00:00:00 2001 From: Matt Brock Date: Fri, 20 Dec 2024 16:28:48 -0600 Subject: [PATCH 12/12] Updating principal fetching to incorporate metadata from principal subtypes --- .../fetchers/azure-sync/principals.go | 35 +++++++++++++------ 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/lib/srv/discovery/fetchers/azure-sync/principals.go b/lib/srv/discovery/fetchers/azure-sync/principals.go index ab8ae3e48bb64..f372ee8de756e 100644 --- a/lib/srv/discovery/fetchers/azure-sync/principals.go +++ b/lib/srv/discovery/fetchers/azure-sync/principals.go @@ -29,6 +29,15 @@ import ( "github.com/gravitational/teleport/lib/msgraph" ) +type dirObjMetadata struct { + objectType string +} + +type queryResult struct { + metadata dirObjMetadata + dirObj msgraph.DirectoryObject +} + // fetchPrincipals fetches the Azure principals (users, groups, and service principals) using the Graph API func fetchPrincipals(ctx context.Context, subscriptionID string, cli *msgraph.Client) ([]*accessgraphv1alpha.AzurePrincipal, error) { //nolint: unused // invoked in a dependent PR var params = &url.Values{ @@ -36,23 +45,26 @@ func fetchPrincipals(ctx context.Context, subscriptionID string, cli *msgraph.Cl } // Fetch the users, groups, and service principals as directory objects - var dirObjs []msgraph.DirectoryObject + var queryResults []queryResult err := cli.IterateUsers(ctx, params, func(user *msgraph.User) bool { - dirObjs = append(dirObjs, user.DirectoryObject) + res := queryResult{metadata: dirObjMetadata{objectType: "user"}, dirObj: user.DirectoryObject} + queryResults = append(queryResults, res) return true }) if err != nil { return nil, trace.Wrap(err) } err = cli.IterateGroups(ctx, params, func(group *msgraph.Group) bool { - dirObjs = append(dirObjs, group.DirectoryObject) + res := queryResult{metadata: dirObjMetadata{objectType: "group"}, dirObj: group.DirectoryObject} + queryResults = append(queryResults, res) return true }) if err != nil { return nil, trace.Wrap(err) } err = cli.IterateServicePrincipals(ctx, params, func(servicePrincipal *msgraph.ServicePrincipal) bool { - dirObjs = append(dirObjs, servicePrincipal.DirectoryObject) + res := queryResult{metadata: dirObjMetadata{objectType: "servicePrincipal"}, dirObj: servicePrincipal.DirectoryObject} + queryResults = append(queryResults, res) return true }) if err != nil { @@ -62,22 +74,23 @@ func fetchPrincipals(ctx context.Context, subscriptionID string, cli *msgraph.Cl // Return the users, groups, and service principals as protobuf messages var fetchErrs []error var pbPrincipals []*accessgraphv1alpha.AzurePrincipal - for _, dirObj := range dirObjs { - if dirObj.ID == nil || dirObj.DisplayName == nil { - fetchErrs = append(fetchErrs, trace.BadParameter("nil values on msgraph directory object: %v", dirObj)) + for _, res := range queryResults { + if res.dirObj.ID == nil || res.dirObj.DisplayName == nil { + fetchErrs = append(fetchErrs, + trace.BadParameter("nil values on msgraph directory object: %v", res.dirObj)) continue } var memberOf []string - for _, member := range dirObj.MemberOf { + for _, member := range res.dirObj.MemberOf { memberOf = append(memberOf, member.ID) } pbPrincipals = append(pbPrincipals, &accessgraphv1alpha.AzurePrincipal{ - Id: *dirObj.ID, + Id: *res.dirObj.ID, SubscriptionId: subscriptionID, LastSyncTime: timestamppb.Now(), - DisplayName: *dirObj.DisplayName, + DisplayName: *res.dirObj.DisplayName, MemberOf: memberOf, - ObjectType: "user", + ObjectType: res.metadata.objectType, }) } return pbPrincipals, trace.NewAggregate(fetchErrs...)