diff --git a/deepfence_server/apiDocs/docs.go b/deepfence_server/apiDocs/docs.go index 6faea73e17..a4e69b1c87 100644 --- a/deepfence_server/apiDocs/docs.go +++ b/deepfence_server/apiDocs/docs.go @@ -36,6 +36,7 @@ const ( tagSettings = "Settings" tagDiffAdd = "Diff Add" tagCompletion = "Completion" + tagNotification = "Notification" securityName = "bearer_token" ) diff --git a/deepfence_server/apiDocs/operation.go b/deepfence_server/apiDocs/operation.go index 12ebc8da94..7b92bfcb69 100644 --- a/deepfence_server/apiDocs/operation.go +++ b/deepfence_server/apiDocs/operation.go @@ -921,3 +921,18 @@ func (d *OpenAPIDocs) AddCompletionOperations() { "Get Completion for Container fields", "Complete Container info", http.StatusOK, []string{tagCompletion}, bearerToken, new(CompletionNodeFieldReq), new(CompletionNodeFieldRes)) } + +func (d *OpenAPIDocs) AddNotificationOperations() { + d.AddOperation("getNotificationScans", http.MethodGet, "/deepfence/notification/scans", + "Get Notification", "Get Scans for Notification", + http.StatusOK, []string{tagNotification}, bearerToken, new(NotificationGetScanRequest), new(NotificationGetScanResponse)) + d.AddOperation("markNotificationScansRead", http.MethodPost, "/deepfence/notification/scans/mark-read", + "Mark Notification Scans Read", "Mark Notification Scans Read", + http.StatusNoContent, []string{tagNotification}, bearerToken, new(NotificationMarkScanReadRequest), nil) + d.AddOperation("getNotificationRegistrySync", http.MethodGet, "/deepfence/notification/registry-sync", + "Get Notification", "Get Registry Sync for Notification", + http.StatusOK, []string{tagNotification}, bearerToken, nil, new([]RegistryAccount)) + d.AddOperation("getNotificationIntegrationFailures", http.MethodGet, "/deepfence/notification/integration", + "Get Notification", "Get Integration Failures for Notification", + http.StatusOK, []string{tagNotification}, bearerToken, nil, new([]IntegrationListResp)) +} diff --git a/deepfence_server/auth/policy.csv b/deepfence_server/auth/policy.csv index c834d94503..8187a8820d 100644 --- a/deepfence_server/auth/policy.csv +++ b/deepfence_server/auth/policy.csv @@ -92,3 +92,10 @@ p, admin, license, read p, admin, license, write p, admin, license, delete p, standard-user, license, read + +p, admin, notification, read +p, admin, notification, write +p, admin, notification, delete +p, standard-user, notification, read +p, standard-user, notification, write +p, read-only-user, notification, read diff --git a/deepfence_server/handler/integration.go b/deepfence_server/handler/integration.go index 1368d83cea..432f0a2732 100644 --- a/deepfence_server/handler/integration.go +++ b/deepfence_server/handler/integration.go @@ -176,6 +176,11 @@ func (h *Handler) GetIntegrations(w http.ResponseWriter, r *http.Request) { integrationStatus = integration.ErrorMsg.String } + var lastSentTime string + if integration.LastSentTime.Valid { + lastSentTime = integration.LastSentTime.Time.String() + } + newIntegration := model.IntegrationListResp{ ID: integration.ID, IntegrationType: integration.IntegrationType, @@ -183,6 +188,7 @@ func (h *Handler) GetIntegrations(w http.ResponseWriter, r *http.Request) { Config: config, Filters: filters, LastErrorMsg: integrationStatus, + LastSentTime: lastSentTime, } newIntegration.RedactSensitiveFieldsInConfig() diff --git a/deepfence_server/handler/notification.go b/deepfence_server/handler/notification.go new file mode 100644 index 0000000000..75681a2fff --- /dev/null +++ b/deepfence_server/handler/notification.go @@ -0,0 +1,100 @@ +package handler + +import ( + "net/http" + + "github.com/deepfence/ThreatMapper/deepfence_server/model" + "github.com/deepfence/ThreatMapper/deepfence_server/reporters/notification" + "github.com/deepfence/ThreatMapper/deepfence_utils/log" + httpext "github.com/go-playground/pkg/v5/net/http" +) + +func (h *Handler) GetScansHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + var req model.NotificationGetScanRequest + + // parse request body + err := httpext.DecodeJSON(r, httpext.NoQueryParams, MaxPostRequestSize, &req) + if err != nil { + log.Error().Msgf("Error decoding request: %v", err) + h.respondError(err, w) + return + } + + // TODO: check if status provided are valid + + // get scans from db + scans, err := notification.GetScans(ctx, req.ScanTypes, req.Statuses) + if err != nil { + log.Error().Msgf("Error getting scans: %v", err) + h.respondError(err, w) + return + } + + // respond with scans + err = httpext.JSON(w, http.StatusOK, scans) + return +} + +func (h *Handler) MarkScansReadHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + var req model.NotificationMarkScanReadRequest + + // parse request body + err := httpext.DecodeJSON(r, httpext.NoQueryParams, MaxPostRequestSize, &req) + if err != nil { + log.Error().Msgf("Error decoding request: %v", err) + h.respondError(err, w) + return + } + + // mark scans as read + err = notification.MarkScansRead(ctx, req.ScanType, req.NodeIDs) + if err != nil { + log.Error().Msgf("Error marking scans as read: %v", err) + h.respondError(err, w) + return + } + + // respond with success + err = httpext.JSON(w, http.StatusOK, nil) + return +} + +/* Registry Sync Handlers */ + +// GetRegistrySyncHandler returns the registries that are syncing +func (h *Handler) GetRegistrySyncHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + // get registries that are syncing + registries, err := notification.GetRegistrySync(ctx) + if err != nil { + log.Error().Msgf("Error getting registries that are syncing: %v", err) + h.respondError(err, w) + return + } + + // respond with registries + err = httpext.JSON(w, http.StatusOK, registries) + return +} + +/* Integration Handlers */ + +// GetIntegrationFailuresHandler returns the integrations that have failed +func (h *Handler) GetIntegrationFailuresHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + // get integrations that have failed + integrations, err := notification.GetIntegrationFailures(ctx) + if err != nil { + log.Error().Msgf("Error getting integrations that have failed: %v", err) + h.respondError(err, w) + return + } + + // respond with integrations + err = httpext.JSON(w, http.StatusOK, integrations) + return +} diff --git a/deepfence_server/main.go b/deepfence_server/main.go index 22f8848530..a9ef9af04a 100644 --- a/deepfence_server/main.go +++ b/deepfence_server/main.go @@ -320,6 +320,7 @@ func initializeOpenAPIDocs(openAPIDocs *apiDocs.OpenAPIDocs) { openAPIDocs.AddDiffAddOperations() openAPIDocs.AddCompletionOperations() openAPIDocs.AddLicenseOperations() + openAPIDocs.AddNotificationOperations() } func initializeInternalOpenAPIDocs(openAPIDocs *apiDocs.OpenAPIDocs) { diff --git a/deepfence_server/model/integration.go b/deepfence_server/model/integration.go index 5a8491b3c7..276542f741 100644 --- a/deepfence_server/model/integration.go +++ b/deepfence_server/model/integration.go @@ -103,6 +103,7 @@ type IntegrationListResp struct { Config map[string]interface{} `json:"config"` Filters IntegrationFilters `json:"filters"` LastErrorMsg string `json:"last_error_msg"` + LastSentTime string `json:"last_sent_time"` } func (i *IntegrationListReq) GetIntegrations(ctx context.Context, pgClient *postgresqlDb.Queries) ([]postgresqlDb.Integration, error) { diff --git a/deepfence_server/model/notification.go b/deepfence_server/model/notification.go new file mode 100644 index 0000000000..8625c5c5c3 --- /dev/null +++ b/deepfence_server/model/notification.go @@ -0,0 +1,48 @@ +package model + +type NotificationGetScanResponse struct { + VulnerabilityScan []Scan `json:"vulnerability_scan"` + SecretScan []Scan `json:"secret_scan"` + MalwareScan []Scan `json:"malware_scan"` + PostureScan []Scan `json:"posture_scan"` +} + +type Scan struct { + CreatedAt int64 `json:"created_at"` + UpdatedAt int64 `json:"updated_at"` + NodeID string `json:"node_id"` + IsPriority bool `json:"is_priority"` + Status string `json:"status"` + StatusMessage string `json:"status_message"` + TriggerAction string `json:"trigger_action"` + Retries int64 `json:"retries"` +} + +// TODO: later +type TriggerAction struct { + ID int `json:"id"` + RequestPayload string `json:"request_payload"` +} +type RequestPayload struct { + NodeID string `json:"node_id"` + NodeType int `json:"node_type"` + BinArgs struct { + NodeID string `json:"node_id"` + NodeType string `json:"node_type"` + RegistryID string `json:"registry_id"` + ScanID string `json:"scan_id"` + ScanType string `json:"scan_type"` + } `json:"bin_args"` +} + +type NotificationGetScanRequest struct { + ScanTypes []string `json:"scan_types"` + Statuses []string `json:"status"` + Page int `json:"page"` + Limit int `json:"limit"` +} + +type NotificationMarkScanReadRequest struct { + ScanType string `json:"scan_type"` + NodeIDs []string `json:"node_ids"` +} diff --git a/deepfence_server/reporters/notification/integration.go b/deepfence_server/reporters/notification/integration.go new file mode 100644 index 0000000000..c7517ba41c --- /dev/null +++ b/deepfence_server/reporters/notification/integration.go @@ -0,0 +1,69 @@ +package notification + +import ( + "context" + "encoding/json" + + "github.com/deepfence/ThreatMapper/deepfence_server/model" + "github.com/deepfence/ThreatMapper/deepfence_utils/directory" + "github.com/deepfence/ThreatMapper/deepfence_utils/log" +) + +// GetIntegrationFailures returns the integrations that have failed +func GetIntegrationFailures(ctx context.Context) ([]model.IntegrationListResp, error) { + var failedIntegrations []model.IntegrationListResp + pgClient, err := directory.PostgresClient(ctx) + if err != nil { + return failedIntegrations, nil + } + + integrations, err := pgClient.GetIntegrations(ctx) + if err != nil { + log.Error().Msgf("Error getting postgresCtx: %v", err) + return failedIntegrations, err + } + + // filter out integrations that have errorMsg + for _, integration := range integrations { + if integration.ErrorMsg.Valid { + var config map[string]interface{} + var filters model.IntegrationFilters + + err = json.Unmarshal(integration.Config, &config) + if err != nil { + log.Error().Msgf(err.Error()) + continue + } + err = json.Unmarshal(integration.Filters, &filters) + if err != nil { + log.Error().Msgf(err.Error()) + continue + } + + var integrationStatus string + if integration.ErrorMsg.Valid { + integrationStatus = integration.ErrorMsg.String + } + + var lastSentTime string + if integration.LastSentTime.Valid { + lastSentTime = integration.LastSentTime.Time.String() + } + + newIntegration := model.IntegrationListResp{ + ID: integration.ID, + IntegrationType: integration.IntegrationType, + NotificationType: integration.Resource, + Config: config, + Filters: filters, + LastErrorMsg: integrationStatus, + LastSentTime: lastSentTime, + } + + newIntegration.RedactSensitiveFieldsInConfig() + failedIntegrations = append(failedIntegrations, newIntegration) + } + } + + return failedIntegrations, nil +} diff --git a/deepfence_server/reporters/notification/registry.go b/deepfence_server/reporters/notification/registry.go new file mode 100644 index 0000000000..c1786e8306 --- /dev/null +++ b/deepfence_server/reporters/notification/registry.go @@ -0,0 +1,62 @@ +package notification + +import ( + "context" + "time" + + "github.com/deepfence/ThreatMapper/deepfence_server/model" + "github.com/deepfence/ThreatMapper/deepfence_utils/directory" + "github.com/deepfence/ThreatMapper/deepfence_utils/log" + "github.com/neo4j/neo4j-go-driver/v5/neo4j" +) + +// GetRegistrySync returns the registries that are syncing +func GetRegistrySync(ctx context.Context) ([]model.RegistryAccount, error) { + registries := []model.RegistryAccount{} + + driver, err := directory.Neo4jClient(ctx) + if err != nil { + return registries, err + } + + log.Info().Msgf("Getting registries that are syncing") + + session := driver.NewSession(ctx, neo4j.SessionConfig{AccessMode: neo4j.AccessModeRead}) + defer session.Close(ctx) + + tx, err := session.BeginTransaction(ctx, neo4j.WithTxTimeout(30*time.Second)) + if err != nil { + return registries, err + } + defer tx.Close(ctx) + query := ` + MATCH (r:RegistryAccount) + WHERE r.syncing = true + RETURN r.name, r.node_id, r.registry_type, r.syncing + ` + log.Debug().Msgf("Query: %s", query) + result, err := tx.Run(ctx, query, map[string]interface{}{}) + if err != nil { + return registries, err + } + + rec, err := result.Collect(ctx) + if err != nil { + return registries, err + } + + if len(rec) == 0 { + return registries, nil + } + + for _, record := range rec { + reg := model.RegistryAccount{} + reg.Name = record.Values[0].(string) + reg.ID = record.Values[1].(string) + reg.RegistryType = record.Values[2].(string) + reg.Syncing = record.Values[3].(bool) + registries = append(registries, reg) + } + + return registries, nil +} diff --git a/deepfence_server/reporters/notification/scan.go b/deepfence_server/reporters/notification/scan.go new file mode 100644 index 0000000000..a590df640e --- /dev/null +++ b/deepfence_server/reporters/notification/scan.go @@ -0,0 +1,155 @@ +package notification + +import ( + "context" + "fmt" + "time" + + "github.com/deepfence/ThreatMapper/deepfence_server/model" + "github.com/deepfence/ThreatMapper/deepfence_utils/directory" + "github.com/deepfence/ThreatMapper/deepfence_utils/log" + "github.com/neo4j/neo4j-go-driver/v5/neo4j" +) + +func GetScans(ctx context.Context, scanTypes []string, statues []string) (model.NotificationGetScanResponse, error) { + response := model.NotificationGetScanResponse{} + var err error + for _, scanType := range scanTypes { + switch scanType { + case "vulnerability": + response.VulnerabilityScan, err = GetScansFor(ctx, "VulnerabilityScan", statues) + if err != nil { + return response, err + } + case "secret": + response.SecretScan, err = GetScansFor(ctx, "SecretScan", statues) + if err != nil { + return response, err + } + case "malware": + response.MalwareScan, err = GetScansFor(ctx, "MalwareScan", statues) + if err != nil { + return response, err + } + case "posture": + response.PostureScan, err = GetScansFor(ctx, "PostureScan", statues) + if err != nil { + return response, err + } + case "all": + response.VulnerabilityScan, err = GetScansFor(ctx, "VulnerabilityScan", statues) + if err != nil { + return response, err + } + response.SecretScan, err = GetScansFor(ctx, "SecretScan", statues) + if err != nil { + return response, err + } + response.MalwareScan, err = GetScansFor(ctx, "MalwareScan", statues) + if err != nil { + return response, err + } + response.PostureScan, err = GetScansFor(ctx, "PostureScan", statues) + if err != nil { + return response, err + } + default: + return response, fmt.Errorf("Invalid scan type") + } + } + return response, nil +} + +func GetScansFor(ctx context.Context, scanType string, statues []string) ([]model.Scan, error) { + scans := []model.Scan{} + driver, err := directory.Neo4jClient(ctx) + if err != nil { + return scans, err + } + + log.Info().Msgf("Getting scans for %s and with statues %+v", scanType, statues) + log.Info().Msgf("len of status: %d", len(statues)) + + session := driver.NewSession(ctx, neo4j.SessionConfig{AccessMode: neo4j.AccessModeRead}) + defer session.Close(ctx) + + tx, err := session.BeginTransaction(ctx, neo4j.WithTxTimeout(30*time.Second)) + if err != nil { + return scans, err + } + defer tx.Close(ctx) + query := ` + MATCH (n:` + scanType + `) + WHERE n.status IN $statues + AND n.acknowledged_at IS NULL + RETURN n.created_at, n.updated_at, n.node_id, n.is_priority, n.status, n.status_message, n.trigger_action, n.retries` + if len(statues) == 0 { + query = ` + MATCH (n:` + scanType + `) + WHERE n.acknowledged_at IS NULL + RETURN n.created_at, n.updated_at, n.node_id, n.is_priority, n.status, n.status_message, n.trigger_action, n.retries` + } + log.Debug().Msgf("Query: %s", query) + result, err := tx.Run(ctx, query, map[string]interface{}{"statues": statues}) + if err != nil { + return scans, err + } + + rec, err := result.Collect(ctx) + if err != nil { + return scans, err + } + + if len(rec) == 0 { + return scans, nil + } + + for i := range rec { + scan := model.Scan{} + scan.CreatedAt = rec[i].Values[0].(int64) + scan.UpdatedAt = rec[i].Values[1].(int64) + scan.NodeID = rec[i].Values[2].(string) + scan.IsPriority = rec[i].Values[3].(bool) + scan.Status = rec[i].Values[4].(string) + scan.StatusMessage = rec[i].Values[5].(string) + scan.TriggerAction = rec[i].Values[6].(string) + scan.Retries = rec[i].Values[7].(int64) + scans = append(scans, scan) + } + + return scans, nil +} + +func MarkScansRead(ctx context.Context, scanType string, nodeIDs []string) error { + driver, err := directory.Neo4jClient(ctx) + if err != nil { + return err + } + + session := driver.NewSession(ctx, neo4j.SessionConfig{AccessMode: neo4j.AccessModeWrite}) + defer session.Close(ctx) + + tx, err := session.BeginTransaction(ctx, neo4j.WithTxTimeout(30*time.Second)) + if err != nil { + return err + } + defer tx.Close(ctx) + + query := ` + MATCH (n:` + scanType + `) + WHERE n.node_id IN $nodeIDs + SET n.acknowledged_at = datetime() + RETURN n` + log.Debug().Msgf("Query: %s", query) + _, err = tx.Run(ctx, query, map[string]interface{}{"nodeIDs": nodeIDs}) + if err != nil { + return err + } + + err = tx.Commit(ctx) + if err != nil { + return err + } + + return nil +} diff --git a/deepfence_server/router/router.go b/deepfence_server/router/router.go index fb1133d7b3..e2a77bbda7 100644 --- a/deepfence_server/router/router.go +++ b/deepfence_server/router/router.go @@ -39,19 +39,20 @@ const ( // API RBAC Resources - ResourceUser = "user" - ResourceSettings = "settings" - ResourceAllUsers = "all-users" - ResourceAgentReport = "agent-report" - ResourceCloudReport = "cloud-report" - ResourceScanReport = "scan-report" - ResourceScan = "scan" - ResourceDiagnosis = "diagnosis" - ResourceCloudNode = "cloud-node" - ResourceRegistry = "container-registry" - ResourceIntegration = "integration" - ResourceReport = "report" - ResourceLicense = "license" + ResourceUser = "user" + ResourceSettings = "settings" + ResourceAllUsers = "all-users" + ResourceAgentReport = "agent-report" + ResourceCloudReport = "cloud-report" + ResourceScanReport = "scan-report" + ResourceScan = "scan" + ResourceDiagnosis = "diagnosis" + ResourceCloudNode = "cloud-node" + ResourceRegistry = "container-registry" + ResourceIntegration = "integration" + ResourceReport = "report" + ResourceLicense = "license" + ResourceNotification = "notification" ) // func telemetryInjector(next http.Handler) http.Handler { @@ -576,6 +577,24 @@ func SetupRoutes(r *chi.Mux, serverPort string, serveOpenapiDocs bool, ingestC c r.Get("/", dfHandler.AuthHandler(ResourceLicense, PermissionRead, dfHandler.GetLicenseHandler)) r.Delete("/", dfHandler.AuthHandler(ResourceLicense, PermissionDelete, dfHandler.DeleteLicenseHandler)) }) + + // notification apis + r.Route("/notification", func(r chi.Router) { + r.Route("/scans", func(r chi.Router) { + r.Get("/", dfHandler.AuthHandler(ResourceNotification, PermissionRead, dfHandler.GetScansHandler)) + // todo: implement this + // r.Get("/count", dfHandler.AuthHandler(ResourceNotification, PermissionRead, dfHandler.GetScansCountHandler)) + // mark-read or acknowledge can only be done on scans which are not IN_PROGRESS + r.Post("/mark-read", dfHandler.AuthHandler(ResourceNotification, PermissionWrite, dfHandler.MarkScansReadHandler)) + }) + r.Route("/registry-sync", func(r chi.Router) { + r.Get("/", dfHandler.AuthHandler(ResourceNotification, PermissionRead, dfHandler.GetRegistrySyncHandler)) + }) + // integration failures + r.Route("/integration", func(r chi.Router) { + r.Get("/", dfHandler.AuthHandler(ResourceNotification, PermissionRead, dfHandler.GetIntegrationFailuresHandler)) + }) + }) }) })