diff --git a/backend/api/event/event.go b/backend/api/event/event.go index 0df56eb41..b0a593285 100644 --- a/backend/api/event/event.go +++ b/backend/api/event/event.go @@ -51,6 +51,9 @@ const ( maxNavigationFromChars = 128 maxNavigationSourceChars = 128 maxScreenViewNameChars = 128 + maxUserDefAttrsCount = 100 + maxUserDefAttrsKeyChars = 256 + maxUserDefAttrsValsChars = 256 ) const TypeANR = "anr" @@ -365,38 +368,39 @@ type ScreenView struct { } type EventField struct { - ID uuid.UUID `json:"id"` - IPv4 net.IP `json:"inet_ipv4"` - IPv6 net.IP `json:"inet_ipv6"` - CountryCode string `json:"inet_country_code"` - AppID uuid.UUID `json:"app_id"` - SessionID uuid.UUID `json:"session_id" binding:"required"` - Timestamp time.Time `json:"timestamp" binding:"required"` - Type string `json:"type" binding:"required"` - UserTriggered bool `json:"user_triggered" binding:"required"` - Attribute Attribute `json:"attribute" binding:"required"` - Attachments []Attachment `json:"attachments" binding:"required"` - ANR *ANR `json:"anr,omitempty"` - Exception *Exception `json:"exception,omitempty"` - AppExit *AppExit `json:"app_exit,omitempty"` - LogString *LogString `json:"string,omitempty"` - GestureLongClick *GestureLongClick `json:"gesture_long_click,omitempty"` - GestureScroll *GestureScroll `json:"gesture_scroll,omitempty"` - GestureClick *GestureClick `json:"gesture_click,omitempty"` - LifecycleActivity *LifecycleActivity `json:"lifecycle_activity,omitempty"` - LifecycleFragment *LifecycleFragment `json:"lifecycle_fragment,omitempty"` - LifecycleApp *LifecycleApp `json:"lifecycle_app,omitempty"` - ColdLaunch *ColdLaunch `json:"cold_launch,omitempty"` - WarmLaunch *WarmLaunch `json:"warm_launch,omitempty"` - HotLaunch *HotLaunch `json:"hot_launch,omitempty"` - NetworkChange *NetworkChange `json:"network_change,omitempty"` - Http *Http `json:"http,omitempty"` - MemoryUsage *MemoryUsage `json:"memory_usage,omitempty"` - LowMemory *LowMemory `json:"low_memory,omitempty"` - TrimMemory *TrimMemory `json:"trim_memory,omitempty"` - CPUUsage *CPUUsage `json:"cpu_usage,omitempty"` - Navigation *Navigation `json:"navigation,omitempty"` - ScreenView *ScreenView `json:"screen_view,omitempty"` + ID uuid.UUID `json:"id"` + IPv4 net.IP `json:"inet_ipv4"` + IPv6 net.IP `json:"inet_ipv6"` + CountryCode string `json:"inet_country_code"` + AppID uuid.UUID `json:"app_id"` + SessionID uuid.UUID `json:"session_id" binding:"required"` + Timestamp time.Time `json:"timestamp" binding:"required"` + Type string `json:"type" binding:"required"` + UserTriggered bool `json:"user_triggered" binding:"required"` + Attribute Attribute `json:"attribute" binding:"required"` + UserDefinedAttribute UDAttribute `json:"user_defined_attribute" binding:"required"` + Attachments []Attachment `json:"attachments" binding:"required"` + ANR *ANR `json:"anr,omitempty"` + Exception *Exception `json:"exception,omitempty"` + AppExit *AppExit `json:"app_exit,omitempty"` + LogString *LogString `json:"string,omitempty"` + GestureLongClick *GestureLongClick `json:"gesture_long_click,omitempty"` + GestureScroll *GestureScroll `json:"gesture_scroll,omitempty"` + GestureClick *GestureClick `json:"gesture_click,omitempty"` + LifecycleActivity *LifecycleActivity `json:"lifecycle_activity,omitempty"` + LifecycleFragment *LifecycleFragment `json:"lifecycle_fragment,omitempty"` + LifecycleApp *LifecycleApp `json:"lifecycle_app,omitempty"` + ColdLaunch *ColdLaunch `json:"cold_launch,omitempty"` + WarmLaunch *WarmLaunch `json:"warm_launch,omitempty"` + HotLaunch *HotLaunch `json:"hot_launch,omitempty"` + NetworkChange *NetworkChange `json:"network_change,omitempty"` + Http *Http `json:"http,omitempty"` + MemoryUsage *MemoryUsage `json:"memory_usage,omitempty"` + LowMemory *LowMemory `json:"low_memory,omitempty"` + TrimMemory *TrimMemory `json:"trim_memory,omitempty"` + CPUUsage *CPUUsage `json:"cpu_usage,omitempty"` + Navigation *Navigation `json:"navigation,omitempty"` + ScreenView *ScreenView `json:"screen_view,omitempty"` } // Compute computes the most accurate cold launch timing diff --git a/backend/api/event/userdefattr.go b/backend/api/event/userdefattr.go new file mode 100644 index 000000000..a1dd85df2 --- /dev/null +++ b/backend/api/event/userdefattr.go @@ -0,0 +1,630 @@ +package event + +import ( + "encoding/json" + "errors" + "fmt" + "reflect" + "regexp" + "slices" + "strconv" + "strings" + + "github.com/leporo/sqlf" +) + +// attrKeyPattern defines the regular +// expression pattern for validating +// attribute keys. +const attrKeyPattern = "^[a-z0-9_-]+$" + +// maxAllowedDegree defines the maximum +// nesting depth a recursive nested +// expression is allowed. +const maxAllowedDegree = 2 + +const ( + AttrUnknown AttrType = iota + AttrString + AttrInt64 + AttrFloat64 + AttrBool +) + +type AttrType int + +// String returns a string representation of the +// attribute type. +func (a AttrType) String() string { + switch a { + default: + return "unknown" + case AttrString: + return "string" + case AttrInt64: + return "int64" + case AttrFloat64: + return "float64" + case AttrBool: + return "bool" + } +} + +func (a *AttrType) UnmarshalJSON(b []byte) error { + var s string + if err := json.Unmarshal(b, &s); err != nil { + return err + } + + s = strings.ToLower(s) + + switch s { + case AttrString.String(): + *a = AttrString + case AttrInt64.String(): + *a = AttrInt64 + case AttrFloat64.String(): + *a = AttrFloat64 + case AttrBool.String(): + *a = AttrBool + default: + return fmt.Errorf("invalid attribute type: %s", s) + } + + return nil +} + +const ( + OpEq AttrOp = iota + OpNeq + OpContains + OpStartsWith + OpGt + OpLt + OpGte + OpLte +) + +type AttrOp int + +// String returns a string representation of the +// attribute operator. +func (o AttrOp) String() string { + switch o { + default: + return "unknown" + case OpEq: + return "eq" + case OpNeq: + return "neq" + case OpContains: + return "contains" + case OpStartsWith: + return "startsWith" + case OpGt: + return "gt" + case OpLt: + return "lt" + case OpGte: + return "gte" + case OpLte: + return "lte" + } +} + +// Sql returns the SQL compatible +// operator to be consumed directly +// in a SQL query. +func (o AttrOp) Sql() string { + switch o { + default: + return "=" + case OpEq: + return "=" + case OpNeq: + return "!=" + case OpContains: + return "ilike" + case OpStartsWith: + return "ilike" + case OpGt: + return ">" + case OpLt: + return "<" + case OpGte: + return ">=" + case OpLte: + return "<=" + } +} + +// getValidOperators provides a slice of valid attribute +// operators for the requested attribute type. +func getValidOperators(ty AttrType) (ops []AttrOp) { + switch ty { + case AttrBool: + ops = []AttrOp{OpEq, OpNeq} + case AttrString: + ops = []AttrOp{OpEq, OpNeq, OpContains, OpStartsWith} + case AttrInt64, AttrFloat64: + ops = []AttrOp{OpEq, OpNeq, OpGt, OpGte, OpLt, OpLte} + } + return +} + +func (o *AttrOp) UnmarshalJSON(b []byte) error { + // extract the value inside double quotes + var s string + if err := json.Unmarshal(b, &s); err != nil { + return err + } + + switch s { + case OpEq.String(): + *o = OpEq + case OpNeq.String(): + *o = OpNeq + case OpContains.String(): + *o = OpContains + case OpStartsWith.String(): + *o = OpStartsWith + case OpGt.String(): + *o = OpGt + case OpLt.String(): + *o = OpLt + case OpGte.String(): + *o = OpGte + case OpLte.String(): + *o = OpLte + } + + return nil +} + +// UDKeyType represents a single pair +// of user defined attribute key and +// its type. +type UDKeyType struct { + Key string `json:"key"` + Type string `json:"type"` +} + +// UDExpression represents a self-referential +// composite expression used for querying with +// user defined attributes. +type UDExpression struct { + And []UDExpression `json:"and,omitempty"` + Or []UDExpression `json:"or,omitempty"` + Cmp UDComparison `json:"cmp,omitempty"` +} + +// Degree computes the maximum nesting depth +// of a recursive user defined expression. +func (u *UDExpression) Degree() (degree int) { + type stackItem struct { + expr *UDExpression + degree int + } + + stack := []stackItem{{expr: u, degree: 1}} + maxDegree := 1 + + // process expressions until stack + // is empty + for len(stack) > 0 { + // pop last item from stack + n := len(stack) - 1 + current := stack[n] + stack = stack[:n] + + if current.degree > maxDegree { + maxDegree = current.degree + } + + // add all AND expressions to the stack + for _, andExpr := range current.expr.And { + stack = append(stack, stackItem{ + expr: &andExpr, + degree: current.degree + 1, + }) + } + + // add all OR expressions to the stack + for _, orExpr := range current.expr.Or { + stack = append(stack, stackItem{ + expr: &orExpr, + degree: current.degree + 1, + }) + } + } + + degree = maxDegree + + return +} + +// Empty returns true if the user defined +// expression does not contain any usable +// and meaningful values. +func (e *UDExpression) Empty() bool { + return len(e.And) == 0 && len(e.Or) == 0 && e.Cmp.Empty() +} + +// Left returns true if the expression does +// not contain any further `And` or `Or` +// expressions. +func (u *UDExpression) Leaf() bool { + return len(u.And) == 0 && len(u.Or) == 0 && !u.Cmp.Empty() +} + +// HasAnd returns true if expression contains +// at least 1 `And` expression. +func (u *UDExpression) HasAnd() bool { + return len(u.And) > 0 +} + +// HasOr returns true if expression contains +// at least 1 `Or` expression. +func (u *UDExpression) HasOr() bool { + return len(u.Or) > 0 +} + +// Validate validates the user defined expression +// and returns error if not valid. +func (u *UDExpression) Validate() (err error) { + // should not be empty + if u.Empty() { + err = errors.New("user defined expression cannot be empty") + } + + // should not contain `and` and `or` both + // expression at the same time + if u.HasAnd() && u.HasOr() { + err = errors.New("user defined expression cannot contain both `and` and `or` expressions") + } + + // should not exceed maximum allowed nesting + // level + if u.Degree() > maxAllowedDegree { + err = fmt.Errorf("user defined expression exceeds maximum allowed degree of %d. a degree is the maximum depth of nesting of the expression.", maxAllowedDegree) + } + + // validate each comparison expression + comparisons := u.getComparisons() + for _, cmp := range comparisons { + if err = cmp.Validate(); err != nil { + return + } + } + + return +} + +// Augment augments the sql statement with fully +// qualified `where` expressions. +func (u *UDExpression) Augment(stmt *sqlf.Stmt) { + if u.HasAnd() { + for i, andExpr := range u.And { + if i > 0 { + stmt.Clause("OR") + } + andExpr.Augment(stmt) + } + } else if u.HasOr() { + for i, orExpr := range u.Or { + if i > 0 { + stmt.Clause("OR") + } + orExpr.Augment(stmt) + } + } else if !u.Cmp.Empty() { + u.Cmp.Augment(stmt) + } +} + +// getComparisons extracts all comparison expressions +// from the a user defined expression. +func (u *UDExpression) getComparisons() (cmps []UDComparison) { + stack := []UDExpression{*u} + + // process expressions until stack is empty + for len(stack) > 0 { + n := len(stack) - 1 + current := stack[n] + stack = stack[:n] + + // if expression has a comparison, add it + if !current.Cmp.Empty() { + cmps = append(cmps, current.Cmp) + } + + // add all AND expressions + for i := len(current.And) - 1; i >= 0; i -= 1 { + stack = append(stack, current.And[i]) + } + + // add all OR expressions + for i := len(current.Or) - 1; i >= 0; i -= 1 { + stack = append(stack, current.Or[i]) + } + } + + return +} + +// UDComparison represnts comparison +// expressions. +type UDComparison struct { + Key string `json:"key"` + Type AttrType `json:"type"` + Op AttrOp `json:"op"` + Value string `json:"value"` +} + +// Empty returns true if comparison expression +// lacks a usable key. +func (c UDComparison) Empty() bool { + return c.Key == "" +} + +// EscapedValue provides the escaped string +// for use in LIKE or ILIKE SQL expressions. +func (c UDComparison) EscapedValue() string { + return strings.ReplaceAll(c.Value, "%", "\\%") +} + +// Augment augments the sql statement with fully +// qualified sql expressions. +func (c *UDComparison) Augment(stmt *sqlf.Stmt) { + if c.Empty() { + fmt.Printf("warning: not augmenting user defined comparison to statement %q because comparison expression is empty", stmt.String()) + return + } + + opSymbol := c.Op.Sql() + danglingExpr := hasExprClause(stmt) + exprFunc := stmt.Where + + if danglingExpr { + exprFunc = stmt.Expr + } + + switch c.Op { + case OpEq, OpNeq, OpGt, OpGte, OpLt, OpLte: + exprFunc(fmt.Sprintf("(key = ? AND type = ? AND value %s ?)", opSymbol), c.Key, c.Type.String(), c.Value) + case OpContains: + exprFunc(fmt.Sprintf("(key = ? AND type = ? AND value %s %%?%%)", opSymbol), c.Key, c.Type.String(), c.EscapedValue()) + case OpStartsWith: + exprFunc(fmt.Sprintf("(key = ? AND type = ? AND value %s ?%%)", opSymbol), c.Key, c.Type.String(), c.EscapedValue()) + } +} + +// endsWithAnd returns true if the statement +// ends with a dangling "AND" keyword. +func endsWithAnd(stmt *sqlf.Stmt) bool { + return strings.HasSuffix(strings.ToLower(stmt.String()), "and") +} + +// endsWithOr returns true if the statement +// ends with a dangling "OR" keyword. +func endsWithOr(stmt *sqlf.Stmt) bool { + return strings.HasSuffix(strings.ToLower(stmt.String()), "or") +} + +// hasExprClause returns true if the statement +// ends with either a dangling "AND" or "OR" +// keyword. +func hasExprClause(stmt *sqlf.Stmt) bool { + return endsWithAnd(stmt) || endsWithOr(stmt) +} + +// Validate validates the user defined comparison +// and returns error if not valid. +func (c *UDComparison) Validate() (err error) { + validOps := getValidOperators(c.Type) + if !slices.Contains(validOps, c.Op) { + err = fmt.Errorf("%q operator is not valid for type: %q", c.Op.String(), c.Type) + } + return +} + +// UDAttribute represents user defined +// attributes in a convenient & usable +// structure. +// +// User Defined Attributes are related +// to events or spans. +type UDAttribute struct { + rawAttrs map[string]any + keyTypes map[string]AttrType +} + +// Empty returns true if user defined +// attributes does not contain any keys +// or attributes. +func (u UDAttribute) Empty() bool { + return len(u.rawAttrs) == 0 && len(u.keyTypes) == 0 +} + +// MarshalJSON marshals UDAttribute type of user +// defined attributes to JSON. +func (u UDAttribute) MarshalJSON() (data []byte, err error) { + for key, keytype := range u.keyTypes { + switch keytype { + case AttrBool: + strval := u.rawAttrs[key].(string) + value, err := strconv.ParseBool(strval) + if err != nil { + return nil, err + } + u.rawAttrs[key] = value + case AttrInt64: + strval := u.rawAttrs[key].(string) + value, err := strconv.ParseInt(strval, 10, 64) + if err != nil { + return nil, err + } + u.rawAttrs[key] = value + case AttrFloat64: + strval := u.rawAttrs[key].(string) + value, err := strconv.ParseFloat(strval, 64) + if err != nil { + return nil, err + } + u.rawAttrs[key] = value + case AttrString: + u.rawAttrs[key] = u.rawAttrs[key].(string) + } + } + return json.Marshal(u.rawAttrs) +} + +// UnmarshalJSON unmarshalls bytes resembling user defined +// attributes to UDAttribute type. +func (u *UDAttribute) UnmarshalJSON(data []byte) (err error) { + return json.Unmarshal(data, &u.rawAttrs) +} + +// Validate validates user defined attributes bag. +func (u *UDAttribute) Validate() (err error) { + if u.rawAttrs == nil { + return errors.New("user defined attributes must not be empty") + } + + re := regexp.MustCompile(attrKeyPattern) + + count := len(u.rawAttrs) + + if count > maxUserDefAttrsCount { + return fmt.Errorf("user defined attributes must not exceed %d items", maxUserDefAttrsCount) + } + + if u.keyTypes == nil { + u.keyTypes = make(map[string]AttrType) + } + + for k, v := range u.rawAttrs { + if len(k) > maxUserDefAttrsKeyChars { + return fmt.Errorf("user defined attribute keys must not exceed %d characters", maxUserDefAttrsKeyChars) + } + + if !re.MatchString(k) { + return fmt.Errorf("user defined attribute keys must only contain lowercase alphabets, numbers, hyphens and underscores") + } + + switch value := v.(type) { + case string: + if len(value) > maxUserDefAttrsValsChars { + return fmt.Errorf("user defined attributes string values must not exceed %d characters", maxUserDefAttrsValsChars) + } + + u.keyTypes[k] = AttrString + continue + case bool: + u.keyTypes[k] = AttrBool + case float64: + if reflect.TypeOf(v).Kind() == reflect.Float64 { + if v == float64(int(value)) { + u.keyTypes[k] = AttrInt64 + } else { + u.keyTypes[k] = AttrFloat64 + } + } + continue + default: + return fmt.Errorf("user defined attribute values can be only string, number or boolean") + } + } + + return +} + +// HasItems returns true if user defined +// attribute is not empty. +func (u *UDAttribute) HasItems() bool { + return len(u.rawAttrs) > 0 +} + +// Parameterize provides user defined attributes in a +// compatible data structure that database query engines +// can directly consume. +func (u *UDAttribute) Parameterize() (attr map[string]any) { + attr = map[string]any{} + + val := "" + + for k, v := range u.rawAttrs { + switch v := v.(type) { + case bool: + val = strconv.FormatBool(v) + case float64: + val = strconv.FormatFloat(v, 'g', -1, 64) + case int64: + val = strconv.FormatInt(v, 10) + case string: + val = v + } + + attr[k] = fmt.Sprintf("('%s', '%s')", u.keyTypes[k].String(), val) + } + + return +} + +// Scan scans and stores user defined attribute +// data coming from a database query result. +func (u *UDAttribute) Scan(attrMap map[string][]any) { + for key, tuple := range attrMap { + intType := tuple[0] + if u.keyTypes == nil { + u.keyTypes = make(map[string]AttrType) + } + if u.rawAttrs == nil { + u.rawAttrs = make(map[string]any) + } + attrType := intType.(string) + switch attrType { + case AttrBool.String(): + u.keyTypes[key] = AttrBool + case AttrString.String(): + u.keyTypes[key] = AttrString + case AttrInt64.String(): + u.keyTypes[key] = AttrInt64 + case AttrFloat64.String(): + u.keyTypes[key] = AttrFloat64 + } + u.rawAttrs[key] = tuple[1] + } +} + +// GetUDAttrsOpMap provides a type wise list of operators +// for each type of user defined attribute keys. +func GetUDAttrsOpMap() (opmap map[string][]string) { + opmap = map[string][]string{ + AttrBool.String(): {OpEq.String(), OpNeq.String()}, + AttrString.String(): { + OpEq.String(), + OpNeq.String(), + OpContains.String(), + OpStartsWith.String(), + }, + AttrInt64.String(): { + OpEq.String(), + OpNeq.String(), + OpGt.String(), + OpLt.String(), + OpGte.String(), + OpLte.String(), + }, + AttrFloat64.String(): { + OpEq.String(), + OpNeq.String(), + OpGt.String(), + OpLt.String(), + OpGte.String(), + OpLte.String(), + }, + } + + return +} diff --git a/backend/api/event/userdefattr_test.go b/backend/api/event/userdefattr_test.go new file mode 100644 index 000000000..ef418bec5 --- /dev/null +++ b/backend/api/event/userdefattr_test.go @@ -0,0 +1,659 @@ +package event + +import ( + "reflect" + "testing" + + "github.com/leporo/sqlf" +) + +func TestEmpty(t *testing.T) { + exprEmpty := UDExpression{} + exprNotEmpty := UDExpression{ + Cmp: UDComparison{ + Key: "us_resident", + Type: AttrBool, + Op: OpEq, + Value: "true", + }, + } + cmpEmpty := UDComparison{} + + { + expected := true + got := exprEmpty.Empty() + + if expected != got { + t.Errorf("Expected empty expression to be %v, got %v", expected, got) + } + } + + { + expected := false + got := exprNotEmpty.Empty() + + if expected != got { + t.Errorf("Expected empty expression to be %v, got %v", expected, got) + } + } + + { + expected := true + got := cmpEmpty.Empty() + + if expected != got { + t.Errorf("Expected empty comparison to be %v, got %v", expected, got) + } + } +} + +func TestLeaf(t *testing.T) { + exprEmpty := UDExpression{} + exprNotEmpty := UDExpression{ + Cmp: UDComparison{ + Key: "us_resident", + Type: AttrBool, + Op: OpEq, + Value: "true", + }, + } + exprSingleAnd := UDExpression{ + And: []UDExpression{ + { + Cmp: UDComparison{ + Key: "us_resident", + Type: AttrBool, + Op: OpEq, + Value: "true", + }, + }, + { + Cmp: UDComparison{ + Key: "ca_resident", + Type: AttrBool, + Op: OpEq, + Value: "true", + }, + }, + }, + } + + { + expected := false + got := exprEmpty.Leaf() + + if expected != got { + t.Errorf("Expected leaf expression to be %v, got %v", expected, got) + } + } + + { + expected := true + got := exprNotEmpty.Leaf() + + if expected != got { + t.Errorf("Expected leaf expression to be %v, got %v", expected, got) + } + } + + { + expected := false + got := exprSingleAnd.Leaf() + + if expected != got { + t.Errorf("Expected leaf expression to be %v, got %v", expected, got) + } + } +} + +func TestDegree(t *testing.T) { + exprEmpty := UDExpression{} + exprSingleAnd := UDExpression{ + And: []UDExpression{ + { + Cmp: UDComparison{ + Key: "us_resident", + Type: AttrBool, + Op: OpEq, + Value: "true", + }, + }, + { + Cmp: UDComparison{ + Key: "ca_resident", + Type: AttrBool, + Op: OpEq, + Value: "true", + }, + }, + }, + } + exprSingleCmp := UDExpression{ + Cmp: UDComparison{ + Key: "credit_balance", + Type: AttrInt64, + Op: OpGte, + Value: "1000", + }, + } + + { + expected := 1 + got := exprEmpty.Degree() + + if expected != got { + t.Errorf("Expected %v degree, got %v", expected, got) + } + } + + { + expected := 2 + got := exprSingleAnd.Degree() + + if expected != got { + t.Errorf("Expected %v degree, got %v", expected, got) + } + } + + { + expected := 1 + got := exprSingleCmp.Degree() + + if expected != got { + t.Errorf("Expected %v degree, got %v", expected, got) + } + } +} + +func TestAugmentComparison(t *testing.T) { + cmpEmpty := UDComparison{} + cmpBoolEq := UDComparison{ + Key: "us_resident", + Type: AttrBool, + Op: OpEq, + Value: "true", + } + cmpBoolNeq := UDComparison{ + Key: "us_resident", + Type: AttrBool, + Op: OpNeq, + Value: "false", + } + cmpInt64Gt := UDComparison{ + Key: "credit_balance", + Type: AttrInt64, + Op: OpGt, + Value: "1000", + } + cmpInt64Lt := UDComparison{ + Key: "credit_balance", + Type: AttrInt64, + Op: OpLt, + Value: "1000", + } + cmpFloat64Lt := UDComparison{ + Key: "invested_amount", + Type: AttrFloat64, + Op: OpLt, + Value: "1000.00", + } + cmpFloat64Lte := UDComparison{ + Key: "invested_amount", + Type: AttrFloat64, + Op: OpLte, + Value: "1000.00", + } + cmpFloat64Gt := UDComparison{ + Key: "invested_amount", + Type: AttrFloat64, + Op: OpGt, + Value: "999.99", + } + cmpFloat64Gte := UDComparison{ + Key: "invested_amount", + Type: AttrFloat64, + Op: OpGte, + Value: "1000.00", + } + cmpStringContains := UDComparison{ + Key: "preference", + Type: AttrString, + Op: OpContains, + Value: "spicy", + } + cmpStringStartsWith := UDComparison{ + Key: "name", + Type: AttrString, + Op: OpStartsWith, + Value: "Dr", + } + + { + stmt := sqlf.From("users"). + Select("id"). + Select("name"). + Where("is_active = ?", true) + + defer stmt.Close() + + cmpEmpty.Augment(stmt) + expectedStmt := "SELECT id, name FROM users WHERE is_active = ?" + expectedArgs := []any{true} + gotStmt := stmt.String() + gotArgs := stmt.Args() + + if expectedStmt != gotStmt { + t.Errorf("Expected %q sql statement, got %q", expectedStmt, gotStmt) + } + + if !reflect.DeepEqual(expectedArgs, gotArgs) { + t.Errorf("Expected %v args, got %v", expectedArgs, gotArgs) + } + } + + { + stmt := sqlf.From("users"). + Select("id"). + Select("name"). + Where("is_active = ?", true) + + defer stmt.Close() + + cmpBoolEq.Augment(stmt) + expectedStmt := "SELECT id, name FROM users WHERE is_active = ? AND (key = ? AND type = ? AND value = ?)" + expectedArgs := []any{true, "us_resident", "bool", "true"} + gotStmt := stmt.String() + gotArgs := stmt.Args() + + if expectedStmt != gotStmt { + t.Errorf("Expected %q sql statement, got %q", expectedStmt, gotStmt) + } + + if !reflect.DeepEqual(expectedArgs, gotArgs) { + t.Errorf("Expected %+#v args, got %+#v", expectedArgs, gotArgs) + } + } + + { + stmt := sqlf.From("users"). + Select("id"). + Select("name"). + Where("is_active = ?", true) + + defer stmt.Close() + + cmpBoolNeq.Augment(stmt) + expectedStmt := "SELECT id, name FROM users WHERE is_active = ? AND (key = ? AND type = ? AND value != ?)" + expectedArgs := []any{true, "us_resident", "bool", "false"} + gotStmt := stmt.String() + gotArgs := stmt.Args() + + if expectedStmt != gotStmt { + t.Errorf("Expected %q sql statement, got %q", expectedStmt, gotStmt) + } + + if !reflect.DeepEqual(expectedArgs, gotArgs) { + t.Errorf("Expected %+#v args, got %+#v", expectedArgs, gotArgs) + } + } + + { + stmt := sqlf.From("users"). + Select("id"). + Select("name"). + Where("is_active = ?", true) + + defer stmt.Close() + + cmpInt64Gt.Augment(stmt) + expectedStmt := "SELECT id, name FROM users WHERE is_active = ? AND (key = ? AND type = ? AND value > ?)" + expectedArgs := []any{true, "credit_balance", "int64", "1000"} + gotStmt := stmt.String() + gotArgs := stmt.Args() + + if expectedStmt != gotStmt { + t.Errorf("Expected %q sql statement, got %q", expectedStmt, gotStmt) + } + + if !reflect.DeepEqual(expectedArgs, gotArgs) { + t.Errorf("Expected %+#v args, got %+#v", expectedArgs, gotArgs) + } + } + + { + stmt := sqlf.From("users"). + Select("id"). + Select("name"). + Where("is_active = ?", true) + + defer stmt.Close() + + cmpInt64Lt.Augment(stmt) + expectedStmt := "SELECT id, name FROM users WHERE is_active = ? AND (key = ? AND type = ? AND value < ?)" + expectedArgs := []any{true, "credit_balance", "int64", "1000"} + gotStmt := stmt.String() + gotArgs := stmt.Args() + + if expectedStmt != gotStmt { + t.Errorf("Expected %q sql statement, got %q", expectedStmt, gotStmt) + } + + if !reflect.DeepEqual(expectedArgs, gotArgs) { + t.Errorf("Expected %+#v args, got %+#v", expectedArgs, gotArgs) + } + } + + { + stmt := sqlf.From("users"). + Select("id"). + Select("name"). + Where("is_active = ?", true) + + defer stmt.Close() + + cmpFloat64Lt.Augment(stmt) + expectedStmt := "SELECT id, name FROM users WHERE is_active = ? AND (key = ? AND type = ? AND value < ?)" + expectedArgs := []any{true, "invested_amount", "float64", "1000.00"} + gotStmt := stmt.String() + gotArgs := stmt.Args() + + if expectedStmt != gotStmt { + t.Errorf("Expected %q sql statement, got %q", expectedStmt, gotStmt) + } + + if !reflect.DeepEqual(expectedArgs, gotArgs) { + t.Errorf("Expected %+#v args, got %+#v", expectedArgs, gotArgs) + } + } + + { + stmt := sqlf.From("users"). + Select("id"). + Select("name"). + Where("is_active = ?", true) + + defer stmt.Close() + + cmpFloat64Lte.Augment(stmt) + expectedStmt := "SELECT id, name FROM users WHERE is_active = ? AND (key = ? AND type = ? AND value <= ?)" + expectedArgs := []any{true, "invested_amount", "float64", "1000.00"} + gotStmt := stmt.String() + gotArgs := stmt.Args() + + if expectedStmt != gotStmt { + t.Errorf("Expected %q sql statement, got %q", expectedStmt, gotStmt) + } + + if !reflect.DeepEqual(expectedArgs, gotArgs) { + t.Errorf("Expected %+#v args, got %+#v", expectedArgs, gotArgs) + } + } + + { + stmt := sqlf.From("users"). + Select("id"). + Select("name"). + Where("is_active = ?", true) + + defer stmt.Close() + + cmpFloat64Gt.Augment(stmt) + expectedStmt := "SELECT id, name FROM users WHERE is_active = ? AND (key = ? AND type = ? AND value > ?)" + expectedArgs := []any{true, "invested_amount", "float64", "999.99"} + gotStmt := stmt.String() + gotArgs := stmt.Args() + + if expectedStmt != gotStmt { + t.Errorf("Expected %q sql statement, got %q", expectedStmt, gotStmt) + } + + if !reflect.DeepEqual(expectedArgs, gotArgs) { + t.Errorf("Expected %+#v args, got %+#v", expectedArgs, gotArgs) + } + } + + { + stmt := sqlf.From("users"). + Select("id"). + Select("name"). + Where("is_active = ?", true) + + defer stmt.Close() + + cmpFloat64Gte.Augment(stmt) + expectedStmt := "SELECT id, name FROM users WHERE is_active = ? AND (key = ? AND type = ? AND value >= ?)" + expectedArgs := []any{true, "invested_amount", "float64", "1000.00"} + gotStmt := stmt.String() + gotArgs := stmt.Args() + + if expectedStmt != gotStmt { + t.Errorf("Expected %q sql statement, got %q", expectedStmt, gotStmt) + } + + if !reflect.DeepEqual(expectedArgs, gotArgs) { + t.Errorf("Expected %+#v args, got %+#v", expectedArgs, gotArgs) + } + } + + { + stmt := sqlf.From("users"). + Select("id"). + Select("name"). + Where("is_active = ?", true) + + defer stmt.Close() + + cmpStringContains.Augment(stmt) + expectedStmt := "SELECT id, name FROM users WHERE is_active = ? AND (key = ? AND type = ? AND value ilike %?%)" + expectedArgs := []any{true, "preference", "string", "spicy"} + gotStmt := stmt.String() + gotArgs := stmt.Args() + + if expectedStmt != gotStmt { + t.Errorf("Expected %q sql statement, got %q", expectedStmt, gotStmt) + } + if !reflect.DeepEqual(expectedArgs, gotArgs) { + t.Errorf("Expected %+#v args, got %+#v", expectedArgs, gotArgs) + } + } + + { + stmt := sqlf.From("users"). + Select("id"). + Select("name"). + Where("is_active = ?", true) + + defer stmt.Close() + + cmpStringStartsWith.Augment(stmt) + expectedStmt := "SELECT id, name FROM users WHERE is_active = ? AND (key = ? AND type = ? AND value ilike ?%)" + expectedArgs := []any{true, "name", "string", "Dr"} + gotStmt := stmt.String() + gotArgs := stmt.Args() + + if expectedStmt != gotStmt { + t.Errorf("Expected %q sql statement, got %q", expectedStmt, gotStmt) + } + + if !reflect.DeepEqual(expectedArgs, gotArgs) { + t.Errorf("Expected %+#v args, got %+#v", expectedArgs, gotArgs) + } + } +} + +func TestAugmentExpression(t *testing.T) { + exprEmpty := UDExpression{} + exprNotEmpty := UDExpression{ + Cmp: UDComparison{ + Key: "us_resident", + Type: AttrBool, + Op: OpEq, + Value: "true", + }, + } + exprEscaped := UDExpression{ + Cmp: UDComparison{ + Key: "username", + Type: AttrString, + Op: OpContains, + Value: "ali%ce", + }, + } + exprAnd := UDExpression{ + And: []UDExpression{ + { + Cmp: UDComparison{ + Key: "username", + Type: AttrString, + Op: OpEq, + Value: "alice", + }, + }, + { + Cmp: UDComparison{ + Key: "premium_user", + Type: AttrBool, + Op: OpEq, + Value: "true", + }, + }, + }, + } + exprOr := UDExpression{ + Or: []UDExpression{ + { + Cmp: UDComparison{ + Key: "username", + Type: AttrString, + Op: OpEq, + Value: "alice", + }, + }, + { + Cmp: UDComparison{ + Key: "premium_user", + Type: AttrBool, + Op: OpEq, + Value: "true", + }, + }, + }, + } + + { + stmt := sqlf.From("users"). + Select("id"). + Select("name"). + Where("is_active = ?", true) + + defer stmt.Close() + exprEmpty.Augment(stmt) + + expectedStmt := "SELECT id, name FROM users WHERE is_active = ?" + gotStmt := stmt.String() + expectedArgs := []any{true} + gotArgs := stmt.Args() + + if expectedStmt != gotStmt { + t.Errorf("Expected %q sql statement, got %q", expectedStmt, gotStmt) + } + + if !reflect.DeepEqual(expectedArgs, gotArgs) { + t.Errorf("Expected %+#v args, got %+#v", expectedArgs, gotArgs) + } + } + + { + stmt := sqlf.From("users"). + Select("id"). + Select("name"). + Where("is_active = ?", true) + + defer stmt.Close() + exprNotEmpty.Augment(stmt) + + expectedStmt := "SELECT id, name FROM users WHERE is_active = ? AND (key = ? AND type = ? AND value = ?)" + gotStmt := stmt.String() + expectedArgs := []any{true, "us_resident", "bool", "true"} + gotArgs := stmt.Args() + + if expectedStmt != gotStmt { + t.Errorf("Expected %q sql statement, got %q", expectedStmt, gotStmt) + } + + if !reflect.DeepEqual(expectedArgs, gotArgs) { + t.Errorf("Expected %+#v args, got %+#v", expectedArgs, gotArgs) + } + } + + { + stmt := sqlf.From("users"). + Select("id"). + Select("name"). + Where("is_active = ?", true) + + defer stmt.Close() + exprEscaped.Augment(stmt) + + expectedStmt := "SELECT id, name FROM users WHERE is_active = ? AND (key = ? AND type = ? AND value ilike %?%)" + gotStmt := stmt.String() + expectedArgs := []any{true, "username", "string", "ali\\%ce"} + gotArgs := stmt.Args() + + if expectedStmt != gotStmt { + t.Errorf("Expected %q sql statement, got %q", expectedStmt, gotStmt) + } + + if !reflect.DeepEqual(expectedArgs, gotArgs) { + t.Errorf("Expected %+#v args, got %+#v", expectedArgs, gotArgs) + } + } + + { + stmt := sqlf.From("users"). + Select("id"). + Select("name"). + Where("is_active = ?", true) + + defer stmt.Close() + exprAnd.Augment(stmt) + + expectedStmt := "SELECT id, name FROM users WHERE is_active = ? AND (key = ? AND type = ? AND value = ?) OR (key = ? AND type = ? AND value = ?)" + + gotStmt := stmt.String() + expectedArgs := []any{true, "username", "string", "alice", "premium_user", "bool", "true"} + gotArgs := stmt.Args() + + if expectedStmt != gotStmt { + t.Errorf("Expected %q sql statement, got %q", expectedStmt, gotStmt) + } + + if !reflect.DeepEqual(expectedArgs, gotArgs) { + t.Errorf("Expected %+#v args, got %+#v", expectedArgs, gotArgs) + } + } + + { + stmt := sqlf.From("users"). + Select("id"). + Select("name"). + Where("is_active = ?", true) + + defer stmt.Close() + exprOr.Augment(stmt) + + expectedStmt := "SELECT id, name FROM users WHERE is_active = ? AND (key = ? AND type = ? AND value = ?) OR (key = ? AND type = ? AND value = ?)" + + gotStmt := stmt.String() + expectedArgs := []any{true, "username", "string", "alice", "premium_user", "bool", "true"} + gotArgs := stmt.Args() + + if expectedStmt != gotStmt { + t.Errorf("Expected %q sql statement, got %q", expectedStmt, gotStmt) + } + + if !reflect.DeepEqual(expectedArgs, gotArgs) { + t.Errorf("Expected %+#v args, got %+#v", expectedArgs, gotArgs) + } + } +} diff --git a/backend/api/filter/appfilter.go b/backend/api/filter/appfilter.go index 5721e2671..346385f39 100644 --- a/backend/api/filter/appfilter.go +++ b/backend/api/filter/appfilter.go @@ -1,8 +1,10 @@ package filter import ( + "backend/api/event" "backend/api/pairs" "backend/api/server" + "backend/api/text" "context" "crypto/md5" "encoding/hex" @@ -10,9 +12,11 @@ import ( "errors" "fmt" "slices" + "strings" "time" "github.com/google/uuid" + "github.com/jackc/pgx/v5" "github.com/leporo/sqlf" ) @@ -29,6 +33,15 @@ const MaxPaginationLimit = 1000 // as default for paginating items. const DefaultPaginationLimit = 10 +// Operator represents a comparison operator +// like `eq` for `equal` or `gte` for `greater +// than or equal` used for filtering various +// entities. +type Operator struct { + Code string + Type event.AttrType +} + // AppFilter represents various app filtering // operations and its parameters to query app's // issue journey map, metrics, exceptions and @@ -108,6 +121,19 @@ type AppFilter struct { // consider ANR events. ANR bool `form:"anr"` + // UDAttrKeys indicates a request to receive + // list of user defined attribute key & + // types. + UDAttrKeys bool `form:"ud_attr_keys"` + + // UDExpressionRaw contains the raw user defined + // attribute expression as string. + UDExpressionRaw string `form:"ud_expression"` + + // UDExpression contains the parsed user defined + // attribute expression. + UDExpression *event.UDExpression + // KeyID is the anchor point for keyset // pagination. KeyID string `form:"key_id"` @@ -138,17 +164,19 @@ type AppFilter struct { // used in filtering operations of app's issue journey map, // metrics, exceptions and ANRs. type FilterList struct { - Versions []string `json:"versions"` - VersionCodes []string `json:"version_codes"` - OsNames []string `json:"os_names"` - OsVersions []string `json:"os_versions"` - Countries []string `json:"countries"` - NetworkProviders []string `json:"network_providers"` - NetworkTypes []string `json:"network_types"` - NetworkGenerations []string `json:"network_generations"` - DeviceLocales []string `json:"locales"` - DeviceManufacturers []string `json:"device_manufacturers"` - DeviceNames []string `json:"device_names"` + Versions []string `json:"versions"` + VersionCodes []string `json:"version_codes"` + OsNames []string `json:"os_names"` + OsVersions []string `json:"os_versions"` + Countries []string `json:"countries"` + NetworkProviders []string `json:"network_providers"` + NetworkTypes []string `json:"network_types"` + NetworkGenerations []string `json:"network_generations"` + DeviceLocales []string `json:"locales"` + DeviceManufacturers []string `json:"device_manufacturers"` + DeviceNames []string `json:"device_names"` + UDKeyTypes []event.UDKeyType `json:"ud_keytypes"` + UDExpressionRaw string `json:"ud_expression"` } // Hash generates an MD5 hash of the FilterList struct. @@ -209,6 +237,16 @@ func (af *AppFilter) Validate() error { return fmt.Errorf("`limit` cannot be more than %d", MaxPaginationLimit) } + if af.UDExpressionRaw != "" { + if err := af.parseUDExpression(); err != nil { + return fmt.Errorf("`ud_expresssion` is invalid") + } + + if err := af.UDExpression.Validate(); err != nil { + return fmt.Errorf("`ud_expression` is invalid. %s", err.Error()) + } + } + return nil } @@ -252,47 +290,120 @@ func (af *AppFilter) OSVersionPairs() (osVersions *pairs.Pairs[string, string], return } -// Expand expands comma separated fields to slice -// of strings -func (af *AppFilter) Expand() { - filters, err := GetFiltersFromFilterShortCode(af.FilterShortCode, af.AppID) - if err != nil { +// Expand populates app filters by fetching stored filters +// via short code if short code exists, otherwise splits, trims +// formats existing available filters. +func (af *AppFilter) Expand(ctx context.Context) (err error) { + var filters *FilterList + if af.FilterShortCode != "" { + filters, err = GetFiltersFromCode(ctx, af.FilterShortCode, af.AppID) + if err != nil && !errors.Is(err, pgx.ErrNoRows) { + return + } + // it's critical to set err to nil + err = nil + } + + if filters != nil { + if len(filters.Versions) > 0 { + af.Versions = filters.Versions + } + if len(filters.VersionCodes) > 0 { + af.VersionCodes = filters.VersionCodes + } + if len(filters.OsNames) > 0 { + af.OsNames = filters.OsNames + } + if len(filters.OsVersions) > 0 { + af.OsVersions = filters.OsVersions + } + if len(filters.Countries) > 0 { + af.Countries = filters.Countries + } + if len(filters.DeviceNames) > 0 { + af.DeviceNames = filters.DeviceNames + } + if len(filters.DeviceManufacturers) > 0 { + af.DeviceManufacturers = filters.DeviceManufacturers + } + if len(filters.DeviceLocales) > 0 { + af.Locales = filters.DeviceLocales + } + if len(filters.NetworkProviders) > 0 { + af.NetworkProviders = filters.NetworkProviders + } + if len(filters.NetworkTypes) > 0 { + af.NetworkTypes = filters.NetworkTypes + } + if len(filters.NetworkGenerations) > 0 { + af.NetworkGenerations = filters.NetworkGenerations + } + if filters.UDExpressionRaw != "" { + af.UDExpressionRaw = filters.UDExpressionRaw + } return } - if len(filters.Versions) > 0 { - af.Versions = filters.Versions + // split and trim whitespace from each filter + if len(af.Versions) > 0 { + af.Versions = text.SplitTrimEmpty(af.Versions[0], ",") } - if len(filters.VersionCodes) > 0 { - af.VersionCodes = filters.VersionCodes + + if len(af.VersionCodes) > 0 { + af.VersionCodes = text.SplitTrimEmpty(af.VersionCodes[0], ",") + } + + if len(af.OsNames) > 0 { + af.OsNames = text.SplitTrimEmpty(af.OsNames[0], ",") } - if len(filters.OsNames) > 0 { - af.OsNames = filters.OsNames + + if len(af.OsVersions) > 0 { + af.OsVersions = text.SplitTrimEmpty(af.OsVersions[0], ",") } - if len(filters.OsVersions) > 0 { - af.OsVersions = filters.OsVersions + + if len(af.Countries) > 0 { + af.Countries = text.SplitTrimEmpty(af.Countries[0], ",") } - if len(filters.Countries) > 0 { - af.Countries = filters.Countries + + if len(af.DeviceNames) > 0 { + af.DeviceNames = text.SplitTrimEmpty(af.DeviceNames[0], ",") } - if len(filters.DeviceNames) > 0 { - af.DeviceNames = filters.DeviceNames + + if len(af.DeviceManufacturers) > 0 { + af.DeviceManufacturers = text.SplitTrimEmpty(af.DeviceManufacturers[0], ",") } - if len(filters.DeviceManufacturers) > 0 { - af.DeviceManufacturers = filters.DeviceManufacturers + + if len(af.Locales) > 0 { + af.Locales = text.SplitTrimEmpty(af.Locales[0], ",") } - if len(filters.DeviceLocales) > 0 { - af.Locales = filters.DeviceLocales + + if len(af.NetworkProviders) > 0 { + af.NetworkProviders = text.SplitTrimEmpty(af.NetworkProviders[0], ",") } - if len(filters.NetworkProviders) > 0 { - af.NetworkProviders = filters.NetworkProviders + + if len(af.NetworkTypes) > 0 { + af.NetworkTypes = text.SplitTrimEmpty(af.NetworkTypes[0], ",") } - if len(filters.NetworkTypes) > 0 { - af.NetworkTypes = filters.NetworkTypes + + if len(af.NetworkGenerations) > 0 { + af.NetworkGenerations = text.SplitTrimEmpty(af.NetworkGenerations[0], ",") } - if len(filters.NetworkGenerations) > 0 { - af.NetworkGenerations = filters.NetworkGenerations + + if len(af.UDExpressionRaw) > 0 { + af.UDExpressionRaw = strings.TrimSpace(af.UDExpressionRaw) } + + return +} + +// parseUDExpression parses the raw user defined +// attribute expression value. +func (af *AppFilter) parseUDExpression() (err error) { + af.UDExpression = &event.UDExpression{} + if err = json.Unmarshal([]byte(af.UDExpressionRaw), af.UDExpression); err != nil { + return + } + return } // HasTimeRange checks if the time values are @@ -379,6 +490,12 @@ func (af *AppFilter) HasTimezone() bool { return af.Timezone != "" } +// HasUDExpression returns true if a user +// defined expression was requested. +func (af *AppFilter) HasUDExpression() bool { + return af.UDExpressionRaw != "" && af.UDExpression != nil +} + // LimitAbs returns the absolute value of limit func (af *AppFilter) LimitAbs() int { if !af.HasPositiveLimit() { @@ -470,6 +587,21 @@ func (af *AppFilter) GetGenericFilters(ctx context.Context, fl *FilterList) erro return nil } +// GetUserDefinedAttrKeys provides list of unique user defined +// attribute keys and its data types by matching a subset of +// filters. +func (af *AppFilter) GetUserDefinedAttrKeys(ctx context.Context, fl *FilterList) (err error) { + if af.UDAttrKeys { + keytypes, err := af.getUDAttrKeys(ctx) + if err != nil { + return err + } + fl.UDKeyTypes = append(fl.UDKeyTypes, keytypes...) + } + + return +} + // hasKeyID checks if key id is a valid non-empty // value. func (af *AppFilter) hasKeyID() bool { @@ -835,6 +967,44 @@ func (af *AppFilter) getDeviceNames(ctx context.Context) (deviceNames []string, return } +// getUDAttrKeys finds distinct user defined attribute +// key and its types. +func (af *AppFilter) getUDAttrKeys(ctx context.Context) (keytypes []event.UDKeyType, err error) { + stmt := sqlf.From("user_def_attrs"). + Select("distinct key"). + Select("toString(type) type"). + Clause("prewhere app_id = toUUID(?) and end_of_month <= ?", af.AppID, af.To). + OrderBy("key") + + defer stmt.Close() + + if af.Crash { + stmt.Where("exception = true") + } + + if af.ANR { + stmt.Where("anr = true") + } + + rows, err := server.Server.ChPool.Query(ctx, stmt.String(), stmt.Args()...) + if err != nil { + return + } + + for rows.Next() { + var keytype event.UDKeyType + if err = rows.Scan(&keytype.Key, &keytype.Type); err != nil { + return + } + + keytypes = append(keytypes, keytype) + } + + err = rows.Err() + + return +} + // GetExcludedVersions computes list of app version // and version codes that are excluded from app filter. func (af *AppFilter) GetExcludedVersions(ctx context.Context) (versions Versions, err error) { diff --git a/backend/api/filter/appfilter_test.go b/backend/api/filter/appfilter_test.go new file mode 100644 index 000000000..12fe6547d --- /dev/null +++ b/backend/api/filter/appfilter_test.go @@ -0,0 +1,87 @@ +package filter + +import ( + "backend/api/event" + "testing" +) + +func TestParseRawUDExpression(t *testing.T) { + afOne := &AppFilter{ + UDExpressionRaw: `{"and":[{"cmp":{"key":"paid_user","type":"bool","op":"eq","value":"true"}},{"cmp":{"key":"credit_balance","type":"int64","op":"gte","value":"1000"}}]}`, + } + + afOne.parseUDExpression() + + // assert expression exists + if afOne.UDExpression == nil { + t.Error("Expected parsed user defined expression, got nil") + } + + // assert count of expressions + { + expectedAndLen := 2 + gotAndLen := len(afOne.UDExpression.And) + + if expectedAndLen != gotAndLen { + t.Errorf("Expected %d And expressions, got %d", expectedAndLen, gotAndLen) + } + } + + // assert expression structure and type + // for item 0 + { + expectedKeyName := "paid_user" + expectedKeyType := event.AttrBool + expectedOp := event.OpEq + expectedValue := "true" + gotKeyName := afOne.UDExpression.And[0].Cmp.Key + gotKeyType := afOne.UDExpression.And[0].Cmp.Type + gotOp := afOne.UDExpression.And[0].Cmp.Op + gotValue := afOne.UDExpression.And[0].Cmp.Value + + if expectedKeyName != gotKeyName { + t.Errorf("Expected %v key name, got %v", expectedKeyName, gotKeyName) + } + + if expectedKeyType != gotKeyType { + t.Errorf("Expected %v key type, got %v", expectedKeyType, gotKeyType) + } + + if expectedOp != gotOp { + t.Errorf("Expected %v operator, got %v", expectedOp, gotOp) + } + + if expectedValue != gotValue { + t.Errorf("Expected %v value, got %v", expectedValue, gotValue) + } + } + + // assert expression structure and type + // for item 1 + { + expectedKeyName := "credit_balance" + expectedKeyType := event.AttrInt64 + expectedOp := event.OpGte + expectedValue := "1000" + gotKeyName := afOne.UDExpression.And[1].Cmp.Key + gotKeyType := afOne.UDExpression.And[1].Cmp.Type + gotOp := afOne.UDExpression.And[1].Cmp.Op + gotValue := afOne.UDExpression.And[1].Cmp.Value + + if expectedKeyName != gotKeyName { + t.Errorf("Expected %v key name, got %v", expectedKeyName, gotKeyName) + } + + if expectedKeyType != gotKeyType { + t.Errorf("Expected %v key type, got %v", expectedKeyType, gotKeyType) + } + + if expectedOp != gotOp { + t.Errorf("Expected %v operator, got %v", expectedOp, gotOp) + } + + if expectedValue != gotValue { + t.Errorf("Expected %v value, got %v", expectedValue, gotValue) + } + } +} diff --git a/backend/api/filter/shortfilters.go b/backend/api/filter/shortfilters.go index 210b4954b..d37a946ab 100644 --- a/backend/api/filter/shortfilters.go +++ b/backend/api/filter/shortfilters.go @@ -3,6 +3,7 @@ package filter import ( "context" "encoding/json" + "errors" "fmt" "time" @@ -10,6 +11,8 @@ import ( "backend/api/server" "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" "github.com/leporo/sqlf" ) @@ -50,10 +53,17 @@ func NewShortFilters(appId uuid.UUID, filters FilterList) (*ShortFilters, error) }, nil } -func (shortFilters *ShortFilters) Create() error { +// Create persists the filter shortcode in database +// if it does not exist. +func (shortFilters *ShortFilters) Create(ctx context.Context) error { // If already exists, just return - _, err := GetFiltersFromFilterShortCode(shortFilters.Code, shortFilters.AppId) - if err == nil { + filters, err := GetFiltersFromCode(ctx, shortFilters.Code, shortFilters.AppId) + if err != nil && !errors.Is(err, pgx.ErrNoRows) { + fmt.Printf("Error fetching filters from filter short code %v: %v\n", shortFilters.Code, err) + return err + } + + if filters != nil { return nil } @@ -67,14 +77,20 @@ func (shortFilters *ShortFilters) Create() error { _, err = server.Server.PgPool.Exec(context.Background(), stmt.String(), stmt.Args()...) if err != nil { + // ignorel, if a short filter already exists + if pgErr, ok := err.(*pgconn.PgError); ok && pgErr.Code == "23505" { + return nil + } return err } return nil } -// Returns filters for a given short code and appId. If it doesn't exist, returns an error -func GetFiltersFromFilterShortCode(filterShortCode string, appId uuid.UUID) (*FilterList, error) { +// GetFiltersFromCode returns filters for a given short +// code and app id. Return an error, if a filter doesn't +// exist. +func GetFiltersFromCode(ctx context.Context, filterShortCode string, appId uuid.UUID) (*FilterList, error) { var filters FilterList stmt := sqlf.PostgreSQL. @@ -82,12 +98,10 @@ func GetFiltersFromFilterShortCode(filterShortCode string, appId uuid.UUID) (*Fi From("public.short_filters"). Where("code = ?", filterShortCode). Where("app_id = ?", appId) - defer stmt.Close() - err := server.Server.PgPool.QueryRow(context.Background(), stmt.String(), stmt.Args()...).Scan(&filters) + defer stmt.Close() - if err != nil { - fmt.Printf("Error fetching filters from filter short code %v: %v\n", filterShortCode, err) + if err := server.Server.PgPool.QueryRow(ctx, stmt.String(), stmt.Args()...).Scan(&filters); err != nil { return nil, err } diff --git a/backend/api/measure/app.go b/backend/api/measure/app.go index ea24ec636..ad96ca545 100644 --- a/backend/api/measure/app.go +++ b/backend/api/measure/app.go @@ -257,11 +257,21 @@ func (a App) GetExceptionGroupsWithFilter(ctx context.Context, af *filter.AppFil Select("distinct id"). Clause("prewhere app_id = toUUID(?) and exception.fingerprint = ?", af.AppID, exceptionGroup.Fingerprint). Where("type = ?", event.TypeException). - Where("exception.handled = ?", false). - GroupBy("id") + Where("exception.handled = ?", false) defer eventDataStmt.Close() + if af.HasUDExpression() && !af.UDExpression.Empty() { + subQuery := sqlf.From("user_def_attrs"). + Select("event_id id"). + Where("app_id = toUUID(?)", af.AppID). + Where("exception = true") + af.UDExpression.Augment(subQuery) + eventDataStmt.Clause("AND id in").SubQuery("(", ")", subQuery) + } + + eventDataStmt.GroupBy("id") + if len(af.Versions) > 0 { eventDataStmt.Where("attribute.app_version").In(af.Versions) } @@ -504,11 +514,20 @@ func (a App) GetANRGroupsWithFilter(ctx context.Context, af *filter.AppFilter) ( From("events"). Select("distinct id"). Clause("prewhere app_id = toUUID(?) and anr.fingerprint = ?", af.AppID, anrGroup.Fingerprint). - Where("type = ?", event.TypeANR). - GroupBy("id") + Where("type = ?", event.TypeANR) defer eventDataStmt.Close() + if af.HasUDExpression() && !af.UDExpression.Empty() { + subQuery := sqlf.From("user_def_attrs"). + Select("event_id id"). + Where("app_id = toUUID(?)", af.AppID) + af.UDExpression.Augment(subQuery) + eventDataStmt.Clause("AND id in").SubQuery("(", ")", subQuery) + } + + eventDataStmt.GroupBy("id") + if len(af.Versions) > 0 { eventDataStmt.Where("attribute.app_version").In(af.Versions) } @@ -1328,7 +1347,7 @@ func (a *App) GetSessionEvents(ctx context.Context, sessionId uuid.UUID) (*Sessi `warm_launch.process_start_requested_uptime`, `warm_launch.content_provider_attach_uptime`, `warm_launch.on_next_draw_uptime`, - `warm_launch.launched_activity`, + `toString(warm_launch.launched_activity)`, `warm_launch.has_saved_state`, `warm_launch.intent_data`, `warm_launch.duration`, @@ -1386,6 +1405,7 @@ func (a *App) GetSessionEvents(ctx context.Context, sessionId uuid.UUID) (*Sessi `toString(navigation.from)`, `toString(navigation.source)`, `toString(screen_view.name) `, + `user_defined_attribute`, } stmt := sqlf.From("default.events") @@ -1435,6 +1455,7 @@ func (a *App) GetSessionEvents(ctx context.Context, sessionId uuid.UUID) (*Sessi var cpuUsage event.CPUUsage var navigation event.Navigation var screenView event.ScreenView + var userDefAttr map[string][]any var coldLaunchDuration uint32 var warmLaunchDuration uint32 @@ -1641,12 +1662,20 @@ func (a *App) GetSessionEvents(ctx context.Context, sessionId uuid.UUID) (*Sessi // screen view &screenView.Name, + + // user defined attributes + &userDefAttr, } if err := rows.Scan(dest...); err != nil { return nil, err } + // populate user defined attribute + if len(userDefAttr) > 0 { + ev.UserDefinedAttribute.Scan(userDefAttr) + } + switch ev.Type { case event.TypeANR: if err := json.Unmarshal([]byte(anrExceptions), &anr.Exceptions); err != nil { @@ -1838,7 +1867,19 @@ func GetAppJourney(c *gin.Context) { return } - af.Expand() + if err := af.Expand(ctx); err != nil { + msg := `failed to expand filters` + fmt.Println(msg, err) + status := http.StatusInternalServerError + if errors.Is(err, pgx.ErrNoRows) { + status = http.StatusNotFound + } + c.JSON(status, gin.H{ + "error": msg, + "details": err.Error(), + }) + return + } msg := "app journey request validation failed" @@ -2086,7 +2127,19 @@ func GetAppMetrics(c *gin.Context) { return } - af.Expand() + if err := af.Expand(ctx); err != nil { + msg := `failed to expand filters` + fmt.Println(msg, err) + status := http.StatusInternalServerError + if errors.Is(err, pgx.ErrNoRows) { + status = http.StatusNotFound + } + c.JSON(status, gin.H{ + "error": msg, + "details": err.Error(), + }) + return + } msg := `app metrics request validation failed` @@ -2234,7 +2287,9 @@ func GetAppFilters(c *gin.Context) { if err != nil { msg := `id invalid or missing` fmt.Println(msg, err) - c.JSON(http.StatusBadRequest, gin.H{"error": msg}) + c.JSON(http.StatusBadRequest, gin.H{ + "error": msg, + }) return } @@ -2248,14 +2303,20 @@ func GetAppFilters(c *gin.Context) { if err := c.ShouldBindQuery(&af); err != nil { msg := `failed to parse query parameters` fmt.Println(msg, err) - c.JSON(http.StatusBadRequest, gin.H{"error": msg, "details": err.Error()}) + c.JSON(http.StatusBadRequest, gin.H{ + "error": msg, + "details": err.Error(), + }) return } if err := af.Validate(); err != nil { msg := "app filters request validation failed" fmt.Println(msg, err) - c.JSON(http.StatusBadRequest, gin.H{"error": msg, "details": err.Error()}) + c.JSON(http.StatusBadRequest, gin.H{ + "error": msg, + "details": err.Error(), + }) return } @@ -2267,12 +2328,16 @@ func GetAppFilters(c *gin.Context) { if err != nil { msg := "failed to get team from app id" fmt.Println(msg, err) - c.JSON(http.StatusInternalServerError, gin.H{"error": msg}) + c.JSON(http.StatusInternalServerError, gin.H{ + "error": msg, + }) return } if team == nil { msg := fmt.Sprintf("no team exists for app [%s]", app.ID) - c.JSON(http.StatusBadRequest, gin.H{"error": msg}) + c.JSON(http.StatusBadRequest, gin.H{ + "error": msg, + }) return } @@ -2281,7 +2346,9 @@ func GetAppFilters(c *gin.Context) { if err != nil { msg := `failed to perform authorization` fmt.Println(msg, err) - c.JSON(http.StatusInternalServerError, gin.H{"error": msg}) + c.JSON(http.StatusInternalServerError, gin.H{ + "error": msg, + }) return } @@ -2289,13 +2356,17 @@ func GetAppFilters(c *gin.Context) { if err != nil { msg := `failed to perform authorization` fmt.Println(msg, err) - c.JSON(http.StatusInternalServerError, gin.H{"error": msg}) + c.JSON(http.StatusInternalServerError, gin.H{ + "error": msg, + }) return } if !okTeam || !okApp { msg := `you are not authorized to access this app` - c.JSON(http.StatusForbidden, gin.H{"error": msg}) + c.JSON(http.StatusForbidden, gin.H{ + "error": msg, + }) return } @@ -2304,7 +2375,9 @@ func GetAppFilters(c *gin.Context) { if err := af.GetGenericFilters(ctx, &fl); err != nil { msg := `failed to query app filters` fmt.Println(msg, err) - c.JSON(http.StatusInternalServerError, gin.H{"error": msg}) + c.JSON(http.StatusInternalServerError, gin.H{ + "error": msg, + }) return } @@ -2322,6 +2395,24 @@ func GetAppFilters(c *gin.Context) { osVersions = append(osVersions, osVersion) } + udAttrs := gin.H{ + "operator_types": nil, + "key_types": nil, + } + + if af.UDAttrKeys { + if err := af.GetUserDefinedAttrKeys(ctx, &fl); err != nil { + msg := `failed to query user defined attribute keys` + fmt.Println(msg, err) + c.JSON(http.StatusInternalServerError, gin.H{ + "error": msg, + }) + return + } + udAttrs["operator_types"] = event.GetUDAttrsOpMap() + udAttrs["key_types"] = fl.UDKeyTypes + } + c.JSON(http.StatusOK, gin.H{ "versions": versions, "os_versions": osVersions, @@ -2332,6 +2423,7 @@ func GetAppFilters(c *gin.Context) { "locales": fl.DeviceLocales, "device_manufacturers": fl.DeviceManufacturers, "device_names": fl.DeviceNames, + "ud_attrs": udAttrs, }) } @@ -2341,7 +2433,9 @@ func GetCrashOverview(c *gin.Context) { if err != nil { msg := `id invalid or missing` fmt.Println(msg, err) - c.JSON(http.StatusBadRequest, gin.H{"error": msg}) + c.JSON(http.StatusBadRequest, gin.H{ + "error": msg, + }) return } @@ -2353,16 +2447,36 @@ func GetCrashOverview(c *gin.Context) { if err := c.ShouldBindQuery(&af); err != nil { msg := `failed to parse query parameters` fmt.Println(msg, err) - c.JSON(http.StatusBadRequest, gin.H{"error": msg, "details": err.Error()}) + c.JSON(http.StatusBadRequest, gin.H{ + "error": msg, + "details": err.Error(), + }) return } - af.Expand() + fmt.Println("ud expression raw:", af.UDExpressionRaw) + + if err := af.Expand(ctx); err != nil { + msg := `failed to expand filters` + fmt.Println(msg, err) + status := http.StatusInternalServerError + if errors.Is(err, pgx.ErrNoRows) { + status = http.StatusNotFound + } + c.JSON(status, gin.H{ + "error": msg, + "details": err.Error(), + }) + return + } msg := "crash overview request validation failed" if err := af.Validate(); err != nil { fmt.Println(msg, err) - c.JSON(http.StatusBadRequest, gin.H{"error": msg, "details": err.Error()}) + c.JSON(http.StatusBadRequest, gin.H{ + "error": msg, + "details": err.Error(), + }) return } @@ -2388,12 +2502,16 @@ func GetCrashOverview(c *gin.Context) { if err != nil { msg := "failed to get team from app id" fmt.Println(msg, err) - c.JSON(http.StatusInternalServerError, gin.H{"error": msg}) + c.JSON(http.StatusInternalServerError, gin.H{ + "error": msg, + }) return } if team == nil { msg := fmt.Sprintf("no team exists for app [%s]", app.ID) - c.JSON(http.StatusBadRequest, gin.H{"error": msg}) + c.JSON(http.StatusBadRequest, gin.H{ + "error": msg, + }) return } @@ -2402,7 +2520,9 @@ func GetCrashOverview(c *gin.Context) { if err != nil { msg := `failed to perform authorization` fmt.Println(msg, err) - c.JSON(http.StatusInternalServerError, gin.H{"error": msg}) + c.JSON(http.StatusInternalServerError, gin.H{ + "error": msg, + }) return } @@ -2410,13 +2530,17 @@ func GetCrashOverview(c *gin.Context) { if err != nil { msg := `failed to perform authorization` fmt.Println(msg, err) - c.JSON(http.StatusInternalServerError, gin.H{"error": msg}) + c.JSON(http.StatusInternalServerError, gin.H{ + "error": msg, + }) return } if !okTeam || !okApp { msg := `you are not authorized to access this app` - c.JSON(http.StatusForbidden, gin.H{"error": msg}) + c.JSON(http.StatusForbidden, gin.H{ + "error": msg, + }) return } @@ -2424,7 +2548,9 @@ func GetCrashOverview(c *gin.Context) { if err != nil { msg := "failed to get app's exception groups with filter" fmt.Println(msg, err) - c.JSON(http.StatusInternalServerError, gin.H{"error": msg}) + c.JSON(http.StatusInternalServerError, gin.H{ + "error": msg, + }) return } @@ -2479,7 +2605,20 @@ func GetCrashOverviewPlotInstances(c *gin.Context) { return } - af.Expand() + if err := af.Expand(ctx); err != nil { + msg := `failed to expand filters` + fmt.Println(msg, err) + status := http.StatusInternalServerError + if errors.Is(err, pgx.ErrNoRows) { + status = http.StatusNotFound + } + c.JSON(status, gin.H{ + "error": msg, + "details": err.Error(), + }) + return + } + msg := `crash overview request validation failed` if err := af.Validate(); err != nil { @@ -2616,7 +2755,19 @@ func GetCrashDetailCrashes(c *gin.Context) { return } - af.Expand() + if err := af.Expand(ctx); err != nil { + msg := `failed to expand filters` + fmt.Println(msg, err) + status := http.StatusInternalServerError + if errors.Is(err, pgx.ErrNoRows) { + status = http.StatusNotFound + } + c.JSON(status, gin.H{ + "error": msg, + "details": err.Error(), + }) + return + } msg := "app filters request validation failed" if err := af.Validate(); err != nil { @@ -2758,7 +2909,19 @@ func GetCrashDetailPlotInstances(c *gin.Context) { return } - af.Expand() + if err := af.Expand(ctx); err != nil { + msg := `failed to expand filters` + fmt.Println(msg, err) + status := http.StatusInternalServerError + if errors.Is(err, pgx.ErrNoRows) { + status = http.StatusNotFound + } + c.JSON(status, gin.H{ + "error": msg, + "details": err.Error(), + }) + return + } msg := "app filters request validation failed" if err := af.Validate(); err != nil { @@ -2901,7 +3064,19 @@ func GetCrashDetailAttributeDistribution(c *gin.Context) { return } - af.Expand() + if err := af.Expand(ctx); err != nil { + msg := `failed to expand filters` + fmt.Println(msg, err) + status := http.StatusInternalServerError + if errors.Is(err, pgx.ErrNoRows) { + status = http.StatusNotFound + } + c.JSON(status, gin.H{ + "error": msg, + "details": err.Error(), + }) + return + } msg := "app filters request validation failed" if err := af.Validate(); err != nil { @@ -3014,7 +3189,19 @@ func GetCrashDetailPlotJourney(c *gin.Context) { return } - af.Expand() + if err := af.Expand(ctx); err != nil { + msg := `failed to expand filters` + fmt.Println(msg, err) + status := http.StatusInternalServerError + if errors.Is(err, pgx.ErrNoRows) { + status = http.StatusNotFound + } + c.JSON(status, gin.H{ + "error": msg, + "details": err.Error(), + }) + return + } msg := `crash detail journey plot request validation failed` if err := af.Validate(); err != nil { @@ -3206,7 +3393,19 @@ func GetANROverview(c *gin.Context) { return } - af.Expand() + if err := af.Expand(ctx); err != nil { + msg := `failed to expand filters` + fmt.Println(msg, err) + status := http.StatusInternalServerError + if errors.Is(err, pgx.ErrNoRows) { + status = http.StatusNotFound + } + c.JSON(status, gin.H{ + "error": msg, + "details": err.Error(), + }) + return + } msg := "anr overview request validation failed" if err := af.Validate(); err != nil { @@ -3328,7 +3527,19 @@ func GetANROverviewPlotInstances(c *gin.Context) { return } - af.Expand() + if err := af.Expand(ctx); err != nil { + msg := `failed to expand filters` + fmt.Println(msg, err) + status := http.StatusInternalServerError + if errors.Is(err, pgx.ErrNoRows) { + status = http.StatusNotFound + } + c.JSON(status, gin.H{ + "error": msg, + "details": err.Error(), + }) + return + } msg := "ANR overview request validation failed" if err := af.Validate(); err != nil { @@ -3467,7 +3678,19 @@ func GetANRDetailANRs(c *gin.Context) { return } - af.Expand() + if err := af.Expand(ctx); err != nil { + msg := `failed to expand filters` + fmt.Println(msg, err) + status := http.StatusInternalServerError + if errors.Is(err, pgx.ErrNoRows) { + status = http.StatusNotFound + } + c.JSON(status, gin.H{ + "error": msg, + "details": err.Error(), + }) + return + } msg := "app filters request validation failed" if err := af.Validate(); err != nil { @@ -3609,7 +3832,19 @@ func GetANRDetailPlotInstances(c *gin.Context) { return } - af.Expand() + if err := af.Expand(ctx); err != nil { + msg := `failed to expand filters` + fmt.Println(msg, err) + status := http.StatusInternalServerError + if errors.Is(err, pgx.ErrNoRows) { + status = http.StatusNotFound + } + c.JSON(status, gin.H{ + "error": msg, + "details": err.Error(), + }) + return + } msg := "app filters request validation failed" if err := af.Validate(); err != nil { @@ -3746,7 +3981,19 @@ func GetANRDetailAttributeDistribution(c *gin.Context) { return } - af.Expand() + if err := af.Expand(ctx); err != nil { + msg := `failed to expand filters` + fmt.Println(msg, err) + status := http.StatusInternalServerError + if errors.Is(err, pgx.ErrNoRows) { + status = http.StatusNotFound + } + c.JSON(status, gin.H{ + "error": msg, + "details": err.Error(), + }) + return + } msg := "app filters request validation failed" if err := af.Validate(); err != nil { @@ -3859,7 +4106,19 @@ func GetANRDetailPlotJourney(c *gin.Context) { return } - af.Expand() + if err := af.Expand(ctx); err != nil { + msg := `failed to expand filters` + fmt.Println(msg, err) + status := http.StatusInternalServerError + if errors.Is(err, pgx.ErrNoRows) { + status = http.StatusNotFound + } + c.JSON(status, gin.H{ + "error": msg, + "details": err.Error(), + }) + return + } msg := `ANR detail journey plot request validation failed` if err := af.Validate(); err != nil { @@ -4098,7 +4357,19 @@ func GetSessionsOverview(c *gin.Context) { return } - af.Expand() + if err := af.Expand(ctx); err != nil { + msg := `failed to expand filters` + fmt.Println(msg, err) + status := http.StatusInternalServerError + if errors.Is(err, pgx.ErrNoRows) { + status = http.StatusNotFound + } + c.JSON(status, gin.H{ + "error": msg, + "details": err.Error(), + }) + return + } msg := "sessions overview request validation failed" if err := af.Validate(); err != nil { @@ -4208,7 +4479,20 @@ func GetSessionsOverviewPlotInstances(c *gin.Context) { return } - af.Expand() + if err := af.Expand(ctx); err != nil { + msg := `failed to expand filters` + fmt.Println(msg, err) + status := http.StatusInternalServerError + if errors.Is(err, pgx.ErrNoRows) { + status = http.StatusNotFound + } + c.JSON(status, gin.H{ + "error": msg, + "details": err.Error(), + }) + return + } + msg := `sessions overview request validation failed` if err := af.Validate(); err != nil { @@ -4814,12 +5098,15 @@ func RenameApp(c *gin.Context) { } func CreateShortFilters(c *gin.Context) { + ctx := c.Request.Context() userId := c.GetString("userId") appId, err := uuid.Parse(c.Param("id")) if err != nil { msg := `app id invalid or missing` fmt.Println(msg, err) - c.JSON(http.StatusBadRequest, gin.H{"error": msg}) + c.JSON(http.StatusBadRequest, gin.H{ + "error": msg, + }) return } @@ -4831,12 +5118,16 @@ func CreateShortFilters(c *gin.Context) { if err != nil { msg := "failed to get team from app id" fmt.Println(msg, err) - c.JSON(http.StatusInternalServerError, gin.H{"error": msg}) + c.JSON(http.StatusInternalServerError, gin.H{ + "error": msg, + }) return } if team == nil { msg := fmt.Sprintf("no team exists for app [%s]", app.ID) - c.JSON(http.StatusBadRequest, gin.H{"error": msg}) + c.JSON(http.StatusBadRequest, gin.H{ + "error": msg, + }) return } @@ -4844,12 +5135,16 @@ func CreateShortFilters(c *gin.Context) { if err != nil { msg := `couldn't perform authorization checks` fmt.Println(msg, err) - c.JSON(http.StatusInternalServerError, gin.H{"error": msg}) + c.JSON(http.StatusInternalServerError, gin.H{ + "error": msg, + }) return } if !ok { msg := fmt.Sprintf(`you don't have permissions to create short filters in team [%s]`, team.ID.String()) - c.JSON(http.StatusForbidden, gin.H{"error": msg}) + c.JSON(http.StatusForbidden, gin.H{ + "error": msg, + }) return } @@ -4857,7 +5152,9 @@ func CreateShortFilters(c *gin.Context) { if err := c.ShouldBindJSON(&payload); err != nil { msg := `failed to parse filters json payload` fmt.Println(msg, err) - c.JSON(http.StatusBadRequest, gin.H{"error": msg}) + c.JSON(http.StatusBadRequest, gin.H{ + "error": msg, + }) return } @@ -4865,11 +5162,22 @@ func CreateShortFilters(c *gin.Context) { if err != nil { msg := `failed to create generate filter hash` fmt.Println(msg, err) - c.JSON(http.StatusBadRequest, gin.H{"error": msg}) + c.JSON(http.StatusBadRequest, gin.H{ + "error": msg, + }) return } - shortFilters.Create() + if err = shortFilters.Create(ctx); err != nil { + msg := `failed to create short code from filters` + fmt.Println(msg, err) + c.JSON(http.StatusInternalServerError, gin.H{ + "error": msg, + }) + return + } - c.JSON(http.StatusOK, gin.H{"filter_short_code": shortFilters.Code}) + c.JSON(http.StatusOK, gin.H{ + "filter_short_code": shortFilters.Code, + }) } diff --git a/backend/api/measure/event.go b/backend/api/measure/event.go index e46582c1e..e13b2d61b 100644 --- a/backend/api/measure/event.go +++ b/backend/api/measure/event.go @@ -494,6 +494,18 @@ func (e eventreq) validate() error { return err } + // only process user defined attributes + // if the payload contains any. + // + // this check is super important to have + // because older SDKs won't ever send these + // attributes. + if !e.events[i].UserDefinedAttribute.Empty() { + if err := e.events[i].UserDefinedAttribute.Validate(); err != nil { + return err + } + } + if e.hasAttachments() { for j := range e.events[i].Attachments { if err := e.events[i].Attachments[j].Validate(); err != nil { @@ -600,6 +612,9 @@ func (e eventreq) ingest(ctx context.Context) error { Set(`attribute.network_generation`, e.events[i].Attribute.NetworkGeneration). Set(`attribute.network_provider`, e.events[i].Attribute.NetworkProvider). + // user defined attribute + Set(`user_defined_attribute`, e.events[i].UserDefinedAttribute.Parameterize()). + // attachments Set(`attachments`, attachments) @@ -1076,8 +1091,18 @@ func GetExceptionsWithFilter(ctx context.Context, group *group.ExceptionGroup, a Where("attribute.network_type in ?", af.NetworkTypes). Where("attribute.network_provider in ?", af.NetworkProviders). Where("attribute.network_generation in ?", af.NetworkGenerations). - Where("timestamp >= ? and timestamp <= ?", af.From, af.To). - GroupBy("id, type, timestamp, session_id, attribute.app_version, attribute.app_build, attribute.device_manufacturer, attribute.device_model, attribute.network_type, exceptions, threads, attachments") + Where("timestamp >= ? and timestamp <= ?", af.From, af.To) + + if af.HasUDExpression() && !af.UDExpression.Empty() { + subQuery := sqlf.From("user_def_attrs"). + Select("event_id id"). + Where("app_id = toUUID(?)", af.AppID). + Where("exception = true") + af.UDExpression.Augment(subQuery) + substmt.Clause("AND id in").SubQuery("(", ")", subQuery) + } + + substmt.GroupBy("id, type, timestamp, session_id, attribute.app_version, attribute.app_build, attribute.device_manufacturer, attribute.device_model, attribute.network_type, exceptions, threads, attachments") stmt := sqlf.New("with ? as page_size, ? as last_timestamp, ? as last_id select", pageSize, keyTimestamp, af.KeyID) @@ -1185,9 +1210,9 @@ func GetExceptionPlotInstances(ctx context.Context, af *filter.AppFilter) (issue Select("round((1 - (exception_sessions / total_sessions)) * 100, 2) as crash_free_sessions"). Select("uniq(session_id) as total_sessions"). Select("uniqIf(session_id, type = ? and exception.handled = false) as exception_sessions", event.TypeException). - Clause("prewhere app_id = toUUID(?)", af.AppID). - GroupBy("app_version, datetime"). - OrderBy("app_version, datetime") + Clause("prewhere app_id = toUUID(?)", af.AppID) + + defer stmt.Close() if len(af.Versions) > 0 { stmt.Where("attribute.app_version in ?", af.Versions) @@ -1237,7 +1262,17 @@ func GetExceptionPlotInstances(ctx context.Context, af *filter.AppFilter) (issue stmt.Where("timestamp >= ? and timestamp <= ?", af.From, af.To) } - defer stmt.Close() + if af.HasUDExpression() && !af.UDExpression.Empty() { + subQuery := sqlf.From("user_def_attrs"). + Select("event_id id"). + Where("app_id = toUUID(?)", af.AppID). + Where("exception = true") + af.UDExpression.Augment(subQuery) + stmt.Clause("AND id in").SubQuery("(", ")", subQuery) + } + + stmt.GroupBy("app_version, datetime"). + OrderBy("app_version, datetime") rows, err := server.Server.ChPool.Query(ctx, stmt.String(), stmt.Args()...) if err != nil { @@ -1325,8 +1360,18 @@ func GetANRsWithFilter(ctx context.Context, group *group.ANRGroup, af *filter.Ap Where("attribute.network_type in ?", af.NetworkTypes). Where("attribute.network_provider in ?", af.NetworkProviders). Where("attribute.network_generation in ?", af.NetworkGenerations). - Where("timestamp >= ? and timestamp <= ?", af.From, af.To). - GroupBy("id, type, timestamp, session_id, attribute.app_version, attribute.app_build, attribute.device_manufacturer, attribute.device_model, attribute.network_type, exceptions, threads, attachments") + Where("timestamp >= ? and timestamp <= ?", af.From, af.To) + + if af.HasUDExpression() && !af.UDExpression.Empty() { + subQuery := sqlf.From("user_def_attrs"). + Select("event_id id"). + Where("app_id = toUUID(?)", af.AppID). + Where("anr = true") + af.UDExpression.Augment(subQuery) + substmt.Clause("AND id in").SubQuery("(", ")", subQuery) + } + + substmt.GroupBy("id, type, timestamp, session_id, attribute.app_version, attribute.app_build, attribute.device_manufacturer, attribute.device_model, attribute.network_type, exceptions, threads, attachments") stmt := sqlf.New("with ? as page_size, ? as last_timestamp, ? as last_id select", pageSize, keyTimestamp, af.KeyID) @@ -1434,9 +1479,9 @@ func GetANRPlotInstances(ctx context.Context, af *filter.AppFilter) (issueInstan Select("round((1 - (anr_sessions / total_sessions)) * 100, 2) as anr_free_sessions"). Select("uniq(session_id) as total_sessions"). Select("uniqIf(session_id, type = ?) as anr_sessions", event.TypeANR). - Clause("prewhere app_id = toUUID(?)", af.AppID). - GroupBy("app_version, datetime"). - OrderBy("app_version, datetime") + Clause("prewhere app_id = toUUID(?)", af.AppID) + + defer stmt.Close() if len(af.Versions) > 0 { stmt.Where("attribute.app_version in ?", af.Versions) @@ -1486,7 +1531,17 @@ func GetANRPlotInstances(ctx context.Context, af *filter.AppFilter) (issueInstan stmt.Where("timestamp >= ? and timestamp <= ?", af.From, af.To) } - defer stmt.Close() + if af.HasUDExpression() && !af.UDExpression.Empty() { + subQuery := sqlf.From("user_def_attrs"). + Select("event_id id"). + Where("app_id = toUUID(?)", af.AppID). + Where("anr = true") + af.UDExpression.Augment(subQuery) + stmt.Clause("AND id in").SubQuery("(", ")", subQuery) + } + + stmt.GroupBy("app_version, datetime"). + OrderBy("app_version, datetime") rows, err := server.Server.ChPool.Query(ctx, stmt.String(), stmt.Args()...) if err != nil { @@ -1545,6 +1600,8 @@ func GetIssuesAttributeDistribution(ctx context.Context, g group.IssueGroup, af GroupBy("locale"). GroupBy("device") + defer stmt.Close() + // Add filters as necessary stmt.Where("timestamp >= ? and timestamp <= ?", af.From, af.To) if len(af.Versions) > 0 { @@ -1578,6 +1635,15 @@ func GetIssuesAttributeDistribution(ctx context.Context, g group.IssueGroup, af stmt.Where("attribute.device_name in ?", af.DeviceNames) } + if af.HasUDExpression() && !af.UDExpression.Empty() { + subQuery := sqlf.From("user_def_attrs"). + Select("event_id id"). + Where("app_id = toUUID(?)", af.AppID). + Where(fmt.Sprintf("%s = true", groupType)) + af.UDExpression.Augment(subQuery) + stmt.Clause("AND id in").SubQuery("(", ")", subQuery) + } + // Execute the query and parse results rows, err := server.Server.ChPool.Query(ctx, stmt.String(), stmt.Args()...) if err != nil { @@ -1652,12 +1718,24 @@ func GetIssuesPlot(ctx context.Context, g group.IssueGroup, af *filter.AppFilter Select("formatDateTime(timestamp, '%Y-%m-%d', ?) as datetime", af.Timezone). Select("concat(toString(attribute.app_version), ' ', '(', toString(attribute.app_build),')') as version"). Select("uniq(id) as instances"). - Clause(fmt.Sprintf("prewhere app_id = toUUID(?) and %s.fingerprint = ?", groupType), af.AppID, fingerprint). - GroupBy("version, datetime"). - OrderBy("version, datetime") + Clause(fmt.Sprintf("prewhere app_id = toUUID(?) and %s.fingerprint = ?", groupType), af.AppID, fingerprint) + + defer stmt.Close() stmt.Where("timestamp >= ? and timestamp <= ?", af.From, af.To) + if af.HasUDExpression() && !af.UDExpression.Empty() { + subQuery := sqlf.From("user_def_attrs"). + Select("event_id id"). + Where("app_id = toUUID(?)", af.AppID). + Where(fmt.Sprintf("%s = true", groupType)) + af.UDExpression.Augment(subQuery) + stmt.Clause("AND id in").SubQuery("(", ")", subQuery) + } + + stmt.GroupBy("version, datetime"). + OrderBy("version, datetime") + if len(af.Versions) > 0 { stmt.Where("attribute.app_version in ?", af.Versions) } @@ -1698,8 +1776,6 @@ func GetIssuesPlot(ctx context.Context, g group.IssueGroup, af *filter.AppFilter stmt.Where(("attribute.device_name in ?"), af.DeviceNames) } - defer stmt.Close() - rows, err := server.Server.ChPool.Query(ctx, stmt.String(), stmt.Args()...) if err != nil { return diff --git a/backend/api/measure/session.go b/backend/api/measure/session.go index 99d4f4ebf..bfd923888 100644 --- a/backend/api/measure/session.go +++ b/backend/api/measure/session.go @@ -356,6 +356,14 @@ func GetSessionsWithFilter(ctx context.Context, af *filter.AppFilter) (sessions stmt.Where("device_name in ?", af.DeviceNames) } + if af.HasUDExpression() && !af.UDExpression.Empty() { + subQuery := sqlf.From("user_def_attrs"). + Select("distinct session_id"). + Where("app_id = toUUID(?)", af.AppID) + af.UDExpression.Augment(subQuery) + stmt.Clause("AND session_id in").SubQuery("(", ")", subQuery) + } + applyGroupBy := af.Crash || af.ANR || af.HasCountries() || diff --git a/backend/api/replay/critical.go b/backend/api/replay/critical.go index 58fd1b768..9188a50eb 100644 --- a/backend/api/replay/critical.go +++ b/backend/api/replay/critical.go @@ -14,6 +14,7 @@ import ( // for session replay. type Exception struct { EventType string `json:"event_type"` + UDAttribute *event.UDAttribute `json:"user_defined_attribute"` UserTriggered bool `json:"user_triggered"` GroupId *uuid.UUID `json:"group_id"` Type string `json:"type"` @@ -45,6 +46,7 @@ func (e Exception) GetTimestamp() time.Time { // for session replay. type ANR struct { EventType string `json:"event_type"` + UDAttribute *event.UDAttribute `json:"user_defined_attribute"` GroupId *uuid.UUID `json:"group_id"` Type string `json:"type"` Message string `json:"message"` @@ -97,6 +99,7 @@ func ComputeExceptions(ctx context.Context, appId *uuid.UUID, events []event.Eve exceptions := Exception{ event.Type, + &event.UserDefinedAttribute, event.UserTriggered, groupId, event.Exception.GetType(), @@ -142,6 +145,7 @@ func ComputeANRs(ctx context.Context, appId *uuid.UUID, events []event.EventFiel anrs := ANR{ event.Type, + &event.UserDefinedAttribute, groupId, event.ANR.GetType(), event.ANR.GetMessage(), diff --git a/backend/api/replay/exit.go b/backend/api/replay/exit.go index fd7546e0a..b7547353a 100644 --- a/backend/api/replay/exit.go +++ b/backend/api/replay/exit.go @@ -8,8 +8,9 @@ import ( // AppExit represents app exit events // suitable for session replay. type AppExit struct { - EventType string `json:"event_type"` - ThreadName string `json:"thread_name"` + EventType string `json:"event_type"` + UDAttribute *event.UDAttribute `json:"user_defined_attribute"` + ThreadName string `json:"thread_name"` *event.AppExit Timestamp time.Time `json:"timestamp"` } @@ -32,6 +33,7 @@ func ComputeAppExits(events []event.EventField) (result []ThreadGrouper) { for _, event := range events { appExits := AppExit{ event.Type, + &event.UserDefinedAttribute, event.Attribute.ThreadName, event.AppExit, event.Timestamp, diff --git a/backend/api/replay/gesture.go b/backend/api/replay/gesture.go index 201cbe06b..c15e4cd1d 100644 --- a/backend/api/replay/gesture.go +++ b/backend/api/replay/gesture.go @@ -8,15 +8,16 @@ import ( // GestureClick represents click events suitable // for session replay. type GestureClick struct { - EventType string `json:"event_type"` - ThreadName string `json:"thread_name"` - Target string `json:"target"` - TargetID string `json:"target_id"` - Width uint16 `json:"width"` - Height uint16 `json:"height"` - X float32 `json:"x"` - Y float32 `json:"y"` - Timestamp time.Time `json:"timestamp"` + EventType string `json:"event_type"` + UDAttribute *event.UDAttribute `json:"user_defined_attribute"` + ThreadName string `json:"thread_name"` + Target string `json:"target"` + TargetID string `json:"target_id"` + Width uint16 `json:"width"` + Height uint16 `json:"height"` + X float32 `json:"x"` + Y float32 `json:"y"` + Timestamp time.Time `json:"timestamp"` } // GetThreadName provides the name of the thread @@ -34,15 +35,16 @@ func (gc GestureClick) GetTimestamp() time.Time { // GestureLongClick represents long press events // suitable for session replay. type GestureLongClick struct { - EventType string `json:"event_type"` - ThreadName string `json:"thread_name"` - Target string `json:"target"` - TargetID string `json:"target_id"` - Width uint16 `json:"width"` - Height uint16 `json:"height"` - X float32 `json:"x"` - Y float32 `json:"y"` - Timestamp time.Time `json:"timestamp"` + EventType string `json:"event_type"` + UDAttribute *event.UDAttribute `json:"user_defined_attribute"` + ThreadName string `json:"thread_name"` + Target string `json:"target"` + TargetID string `json:"target_id"` + Width uint16 `json:"width"` + Height uint16 `json:"height"` + X float32 `json:"x"` + Y float32 `json:"y"` + Timestamp time.Time `json:"timestamp"` } // GetThreadName provides the name of the thread @@ -60,16 +62,17 @@ func (glc GestureLongClick) GetTimestamp() time.Time { // GestureScroll represents scroll gesture events // suitable for session replay. type GestureScroll struct { - EventType string `json:"event_type"` - ThreadName string `json:"thread_name"` - Target string `json:"target"` - TargetID string `json:"target_id"` - X float32 `json:"x"` - Y float32 `json:"y"` - EndX float32 `json:"end_x"` - EndY float32 `json:"end_y"` - Direction string `json:"direction"` - Timestamp time.Time `json:"timestamp"` + EventType string `json:"event_type"` + UDAttribute *event.UDAttribute `json:"user_defined_attribute"` + ThreadName string `json:"thread_name"` + Target string `json:"target"` + TargetID string `json:"target_id"` + X float32 `json:"x"` + Y float32 `json:"y"` + EndX float32 `json:"end_x"` + EndY float32 `json:"end_y"` + Direction string `json:"direction"` + Timestamp time.Time `json:"timestamp"` } // GetThreadName provides the name of the thread @@ -90,6 +93,7 @@ func ComputeGestureClicks(events []event.EventField) (result []ThreadGrouper) { for _, event := range events { gestureClicks := GestureClick{ event.Type, + &event.UserDefinedAttribute, event.Attribute.ThreadName, event.GestureClick.Target, event.GestureClick.TargetID, @@ -111,6 +115,7 @@ func ComputeGestureLongClicks(events []event.EventField) (result []ThreadGrouper for _, event := range events { gestureLongClicks := GestureLongClick{ event.Type, + &event.UserDefinedAttribute, event.Attribute.ThreadName, event.GestureLongClick.Target, event.GestureLongClick.TargetID, @@ -132,6 +137,7 @@ func ComputeGestureScrolls(events []event.EventField) (result []ThreadGrouper) { for _, event := range events { gestureScrolls := GestureScroll{ event.Type, + &event.UserDefinedAttribute, event.Attribute.ThreadName, event.GestureScroll.Target, event.GestureScroll.TargetID, diff --git a/backend/api/replay/launch.go b/backend/api/replay/launch.go index da7ff75d3..8402f70ab 100644 --- a/backend/api/replay/launch.go +++ b/backend/api/replay/launch.go @@ -12,10 +12,11 @@ var NominalColdLaunchThreshold = 30 * time.Second // ColdLaunch represents cold launch events // suitable for session replay. type ColdLaunch struct { - EventType string `json:"event_type"` - ThreadName string `json:"thread_name"` - Duration time.Duration `json:"duration"` - Timestamp time.Time `json:"timestamp"` + EventType string `json:"event_type"` + UDAttribute *event.UDAttribute `json:"user_defined_attribute"` + ThreadName string `json:"thread_name"` + Duration time.Duration `json:"duration"` + Timestamp time.Time `json:"timestamp"` } // GetThreadName provides the name of the thread @@ -33,13 +34,14 @@ func (cl ColdLaunch) GetTimestamp() time.Time { // WarmLaunch represents warm launch events // suitable for session replay. type WarmLaunch struct { - EventType string `json:"event_type"` - ThreadName string `json:"thread_name"` - Duration time.Duration `json:"duration"` - LaunchedActivity string `json:"launched_activity"` - HasSavedState bool `json:"has_saved_state"` - IntentData string `json:"intent_data"` - Timestamp time.Time `json:"timestamp"` + EventType string `json:"event_type"` + UDAttribute *event.UDAttribute `json:"user_defined_attribute"` + ThreadName string `json:"thread_name"` + Duration time.Duration `json:"duration"` + LaunchedActivity string `json:"launched_activity"` + HasSavedState bool `json:"has_saved_state"` + IntentData string `json:"intent_data"` + Timestamp time.Time `json:"timestamp"` } // GetThreadName provides the name of the thread @@ -57,13 +59,14 @@ func (wl WarmLaunch) GetTimestamp() time.Time { // HotLaunch represents hot launch events // suitable for session replay. type HotLaunch struct { - EventType string `json:"event_type"` - ThreadName string `json:"thread_name"` - Duration time.Duration `json:"duration"` - LaunchedActivity string `json:"launched_activity"` - HasSavedState bool `json:"has_saved_state"` - IntentData string `json:"intent_data"` - Timestamp time.Time `json:"timestamp"` + EventType string `json:"event_type"` + UDAttribute *event.UDAttribute `json:"user_defined_attribute"` + ThreadName string `json:"thread_name"` + Duration time.Duration `json:"duration"` + LaunchedActivity string `json:"launched_activity"` + HasSavedState bool `json:"has_saved_state"` + IntentData string `json:"intent_data"` + Timestamp time.Time `json:"timestamp"` } // GetThreadName provides the name of the thread @@ -84,6 +87,7 @@ func ComputeColdLaunches(events []event.EventField) (result []ThreadGrouper) { for _, event := range events { coldLaunches := ColdLaunch{ event.Type, + &event.UserDefinedAttribute, event.Attribute.ThreadName, event.ColdLaunch.Duration, event.Timestamp, @@ -100,6 +104,7 @@ func ComputeWarmLaunches(events []event.EventField) (result []ThreadGrouper) { for _, event := range events { warmLaunches := WarmLaunch{ event.Type, + &event.UserDefinedAttribute, event.Attribute.ThreadName, event.WarmLaunch.Duration, event.WarmLaunch.LaunchedActivity, @@ -119,6 +124,7 @@ func ComputeHotLaunches(events []event.EventField) (result []ThreadGrouper) { for _, event := range events { hotLaunches := HotLaunch{ event.Type, + &event.UserDefinedAttribute, event.Attribute.ThreadName, event.HotLaunch.Duration, event.HotLaunch.LaunchedActivity, diff --git a/backend/api/replay/lifecycle.go b/backend/api/replay/lifecycle.go index 61ce5e6d9..cfba3871b 100644 --- a/backend/api/replay/lifecycle.go +++ b/backend/api/replay/lifecycle.go @@ -8,8 +8,9 @@ import ( // LifecycleActivity represents lifecycle // activity events suitable for session replay. type LifecycleActivity struct { - EventType string `json:"event_type"` - ThreadName string `json:"thread_name"` + EventType string `json:"event_type"` + UDAttribute *event.UDAttribute `json:"user_defined_attribute"` + ThreadName string `json:"thread_name"` *event.LifecycleActivity Timestamp time.Time `json:"timestamp"` } @@ -29,8 +30,9 @@ func (la LifecycleActivity) GetTimestamp() time.Time { // LifecycleFragment represents lifecycle // fragment events suitable for session replay. type LifecycleFragment struct { - EventType string `json:"event_type"` - ThreadName string `json:"thread_name"` + EventType string `json:"event_type"` + UDAttribute *event.UDAttribute `json:"user_defined_attribute"` + ThreadName string `json:"thread_name"` *event.LifecycleFragment Timestamp time.Time `json:"timestamp"` } @@ -50,8 +52,9 @@ func (lf LifecycleFragment) GetTimestamp() time.Time { // LifecycleApp represents lifecycle // app events suitable for session replay. type LifecycleApp struct { - EventType string `json:"event_type"` - ThreadName string `json:"thread_name"` + EventType string `json:"event_type"` + UDAttribute *event.UDAttribute `json:"user_defined_attribute"` + ThreadName string `json:"thread_name"` *event.LifecycleApp Timestamp time.Time `json:"timestamp"` } @@ -74,6 +77,7 @@ func ComputeLifecycleActivities(events []event.EventField) (result []ThreadGroup for _, event := range events { activities := LifecycleActivity{ event.Type, + &event.UserDefinedAttribute, event.Attribute.ThreadName, event.LifecycleActivity, event.Timestamp, @@ -90,6 +94,7 @@ func ComputeLifecycleFragments(events []event.EventField) (result []ThreadGroupe for _, event := range events { fragments := LifecycleFragment{ event.Type, + &event.UserDefinedAttribute, event.Attribute.ThreadName, event.LifecycleFragment, event.Timestamp, @@ -106,6 +111,7 @@ func ComputeLifecycleApps(events []event.EventField) (result []ThreadGrouper) { for _, event := range events { apps := LifecycleApp{ event.Type, + &event.UserDefinedAttribute, event.Attribute.ThreadName, event.LifecycleApp, event.Timestamp, diff --git a/backend/api/replay/log.go b/backend/api/replay/log.go index 77de7cb1e..5e60ac27c 100644 --- a/backend/api/replay/log.go +++ b/backend/api/replay/log.go @@ -8,8 +8,9 @@ import ( // LogString represents log events suitable // for session replay. type LogString struct { - EventType string `json:"event_type"` - ThreadName string `json:"thread_name"` + EventType string `json:"event_type"` + UDAttribute *event.UDAttribute `json:"user_defined_attribute"` + ThreadName string `json:"thread_name"` *event.LogString Timestamp time.Time `json:"timestamp"` } @@ -32,6 +33,7 @@ func ComputeLogString(events []event.EventField) (result []ThreadGrouper) { for _, event := range events { logs := LogString{ event.Type, + &event.UserDefinedAttribute, event.Attribute.ThreadName, event.LogString, event.Timestamp, diff --git a/backend/api/replay/memory.go b/backend/api/replay/memory.go index e1d07036e..0b742d29a 100644 --- a/backend/api/replay/memory.go +++ b/backend/api/replay/memory.go @@ -15,8 +15,9 @@ type MemoryUsage struct { // TrimMemory represents trim memory events // suitable for session replay. type TrimMemory struct { - EventType string `json:"event_type"` - ThreadName string `json:"thread_name"` + EventType string `json:"event_type"` + UDAttribute *event.UDAttribute `json:"user_defined_attribute"` + ThreadName string `json:"thread_name"` *event.TrimMemory Timestamp time.Time `json:"timestamp"` } @@ -36,8 +37,9 @@ func (tm TrimMemory) GetTimestamp() time.Time { // LowMemory represents low memory events // suitable for session replay. type LowMemory struct { - EventType string `json:"event_type"` - ThreadName string `json:"thread_name"` + EventType string `json:"event_type"` + UDAttribute *event.UDAttribute `json:"user_defined_attribute"` + ThreadName string `json:"thread_name"` *event.LowMemory Timestamp time.Time `json:"timestamp"` } @@ -74,6 +76,7 @@ func ComputeTrimMemories(events []event.EventField) (result []ThreadGrouper) { for _, event := range events { memories := TrimMemory{ event.Type, + &event.UserDefinedAttribute, event.Attribute.ThreadName, event.TrimMemory, event.Timestamp, @@ -90,6 +93,7 @@ func ComputeLowMemories(events []event.EventField) (result []ThreadGrouper) { for _, event := range events { lowMemories := LowMemory{ event.Type, + &event.UserDefinedAttribute, event.Attribute.ThreadName, event.LowMemory, event.Timestamp, diff --git a/backend/api/replay/nav.go b/backend/api/replay/nav.go index d1b3d9964..dacd2bdc5 100644 --- a/backend/api/replay/nav.go +++ b/backend/api/replay/nav.go @@ -8,9 +8,10 @@ import ( // Navigation represents navigation events suitable // for session replay. type Navigation struct { - EventType string `json:"event_type"` - ThreadName string `json:"thread_name"` - UserTriggered bool `json:"user_triggered"` + EventType string `json:"event_type"` + UDAttribute *event.UDAttribute `json:"user_defined_attribute"` + ThreadName string `json:"thread_name"` + UserTriggered bool `json:"user_triggered"` *event.Navigation Timestamp time.Time `json:"timestamp"` } @@ -18,9 +19,10 @@ type Navigation struct { // ScreenView represents screen view events suitable // for session replay. type ScreenView struct { - EventType string `json:"event_type"` - ThreadName string `json:"thread_name"` - UserTriggered bool `json:"user_triggered"` + EventType string `json:"event_type"` + UDAttribute *event.UDAttribute `json:"user_defined_attribute"` + ThreadName string `json:"thread_name"` + UserTriggered bool `json:"user_triggered"` *event.ScreenView Timestamp time.Time `json:"timestamp"` } @@ -43,6 +45,7 @@ func ComputeNavigation(events []event.EventField) (result []ThreadGrouper) { for _, event := range events { navs := Navigation{ event.Type, + &event.UserDefinedAttribute, event.Attribute.ThreadName, event.UserTriggered, event.Navigation, @@ -60,6 +63,7 @@ func ComputeScreenViews(events []event.EventField) (result []ThreadGrouper) { for _, event := range events { sv := ScreenView{ event.Type, + &event.UserDefinedAttribute, event.Attribute.ThreadName, event.UserTriggered, event.ScreenView, diff --git a/backend/api/replay/network.go b/backend/api/replay/network.go index ea2453c9c..9037805cc 100644 --- a/backend/api/replay/network.go +++ b/backend/api/replay/network.go @@ -8,8 +8,9 @@ import ( // NetworkChange represents network change events // suitable for session replay. type NetworkChange struct { - EventType string `json:"event_type"` - ThreadName string `json:"thread_name"` + EventType string `json:"event_type"` + UDAttribute *event.UDAttribute `json:"user_defined_attribute"` + ThreadName string `json:"thread_name"` *event.NetworkChange Timestamp time.Time `json:"timestamp"` } @@ -29,9 +30,10 @@ func (nc NetworkChange) GetTimestamp() time.Time { // Http represents http events // suitable for session replay. type Http struct { - EventType string `json:"event_type"` - ThreadName string `json:"thread_name"` - UserTriggered bool `json:"user_triggered"` + EventType string `json:"event_type"` + UDAttribute *event.UDAttribute `json:"user_defined_attribute"` + ThreadName string `json:"thread_name"` + UserTriggered bool `json:"user_triggered"` *event.Http Duration time.Duration `json:"duration"` Timestamp time.Time `json:"timestamp"` @@ -55,6 +57,7 @@ func ComputeNetworkChange(events []event.EventField) (result []ThreadGrouper) { for _, event := range events { netChanges := NetworkChange{ event.Type, + &event.UserDefinedAttribute, event.Attribute.ThreadName, event.NetworkChange, event.Timestamp, @@ -73,6 +76,7 @@ func ComputeHttp(events []event.EventField) (result []ThreadGrouper) { startTime := event.Http.StartTime http := Http{ event.Type, + &event.UserDefinedAttribute, event.Attribute.ThreadName, event.UserTriggered, event.Http, diff --git a/docs/api/dashboard/README.md b/docs/api/dashboard/README.md index 6554fb465..54884abae 100644 --- a/docs/api/dashboard/README.md +++ b/docs/api/dashboard/README.md @@ -466,8 +466,9 @@ Fetch an app's filters. #### Usage Notes - App's UUID must be passed in the URI -- Pass `crash=1` as query string parameter to only return filters for unhandled exceptions +- Pass `crash=1` as query string parameter to only return filters for crashes - Pass `anr=1` as query string parameter to only return filters for ANRs +- Pass `ud_attr_keys=1` as query string parameter to return user defined attribute keys - If no query string parameters are passed, the API computes filters from all events #### Authorization & Content Type @@ -496,36 +497,104 @@ These headers must be present in each request. ```json { - "versions": [ - { - "code": "9400", - "name": "7.61" - }, - { - "code": "9300", - "name": "7.60" - }, - { - "code": "9200", - "name": "7.59" - } - ], "countries": [ "bogon" ], - "network_providers": null, - "network_types": [ - "wifi" + "device_manufacturers": [ + "Google" + ], + "device_names": [ + "emu64a", + "emu64a16k" ], - "network_generations": null, "locales": [ "en-US" ], - "device_manufacturers": [ - "Google" + "network_generations": [ + "3g", + "unknown" ], - "device_names": [ - "sunfish" + "network_providers": [ + "T-Mobile", + "unknown" + ], + "network_types": [ + "cellular", + "no_network", + "unknown", + "wifi" + ], + "os_versions": [ + { + "name": "android", + "version": "35" + }, + { + "name": "android", + "version": "33" + } + ], + "ud_attrs": { + "key_types": [ + { + "key": "username", + "type": "string" + }, + { + "key": "paid_user", + "type": "bool" + }, + { + "key": "credit_balance", + "type": "int64" + }, + { + "key": "latitude", + "type": "float64" + } + ], + "operator_types": { + "bool": [ + "eq", + "neq" + ], + "float64": [ + "eq", + "neq", + "gt", + "lt", + "gte", + "lte" + ], + "int64": [ + "eq", + "neq", + "gt", + "lt", + "gte", + "lte" + ], + "string": [ + "eq", + "neq", + "contains", + "startsWith" + ] + } + }, + "versions": [ + { + "code": "800", + "name": "0.8.0-SNAPSHOT.debug" + }, + { + "code": "700", + "name": "0.7.0-SNAPSHOT.debug" + }, + { + "code": "1", + "name": "1.0" + } ] } ``` @@ -573,6 +642,8 @@ Fetch an app's crash overview. - `version_codes` (_optional_) - List of comma separated version codes to return crash groups that have events matching the version code. - `key_id` (_optional_) - UUID of the last item. Used for keyset based pagination. Should be used along with `limit`. - `limit` (_optional_) - Number of items to return. Used for keyset based pagination. Should be used along with `key_id`. Negative values traverses backward along with `limit`. + - `filter_short_code` (_optional_) - Code representing combination of filters. + - `ud_expression` (_optional_) - Expression in JSON to filter using user defined attributes. #### Authorization & Content Type @@ -690,6 +761,8 @@ Fetch an app's crash overview instances plot aggregated by date range & version. - `to` (_optional_) - End time boundary for temporal filtering. ISO8601 Datetime string. If not passed, a default value is assumed. - `versions` (_optional_) - List of comma separated version identifier strings to return crash groups that have events matching the version. - `version_codes` (_optional_) - List of comma separated version codes to return crash groups that have events matching the version code. + - `filter_short_code` (_optional_) - Code representing combination of filters. + - `ud_expression` (_optional_) - Expression in JSON to filter using user defined attributes. - Both `from` and `to` **MUST** be present when specifyng date range. #### Authorization & Content Type @@ -792,6 +865,8 @@ Fetch an app's crash detail. - `key_id` (_optional_) - UUID of the last item. Used for keyset based pagination. Should be used along with `key_timestamp` & `limit`. - `key_timestamp` (_optional_) - ISO8601 timestamp of the last item. Used for keyset based pagination. Should be used along with `key_id` & `limit`. - `limit` (_optional_) - Number of items to return. Used for keyset based pagination. Should be used along with `key_id` & `key_timestamp`. + - `filter_short_code` (_optional_) - Code representing combination of filters. + - `ud_expression` (_optional_) - Expression in JSON to filter using user defined attributes. - For multiple comma separated fields, make sure no whitespace characters exist before or after comma. #### Authorization & Content Type @@ -1032,6 +1107,8 @@ Fetch an app's crash detail instances aggregrated by date range & version. - `network_providers` (_optional_) - List of comma separated network provider identifier strings to return only matching crashes. - `network_types` (_optional_) - List of comma separated network type identifier strings to return only matching crashes. - `network_generations` (_optional_) - List of comma separated network generation identifier strings to return only matching crashes. + - `filter_short_code` (_optional_) - Code representing combination of filters. + - `ud_expression` (_optional_) - Expression in JSON to filter using user defined attributes. - For multiple comma separated fields, make sure no whitespace characters exist before or after comma. #### Authorization & Content Type @@ -1266,6 +1343,8 @@ Fetch an app's ANR overview. - `version_codes` (_optional_) - List of comma separated version codes to return anr groups that have events matching the version code. - `key_id` (_optional_) - UUID of the last item. Used for keyset based pagination. Should be used along with `limit`. - `limit` (_optional_) - Number of items to return. Used for keyset based pagination. Should be used along with `key_id`. Negative values traverses backward along with `limit`. + - `filter_short_code` (_optional_) - Code representing combination of filters. + - `ud_expression` (_optional_) - Expression in JSON to filter using user defined attributes. #### Authorization & Content Type @@ -1363,6 +1442,8 @@ Fetch an app's ANR overview instances plot aggregated by date range & version. - `to` (_optional_) - End time boundary for temporal filtering. ISO8601 Datetime string. If not passed, a default value is assumed. - `versions` (_optional_) - List of comma separated version identifier strings to return crash groups that have events matching the version. - `version_codes` (_optional_) - List of comma separated version codes to return crash groups that have events matching the version code. + - `filter_short_code` (_optional_) - Code representing combination of filters. + - `ud_expression` (_optional_) - Expression in JSON to filter using user defined attributes. - Both `from` and `to` **MUST** be present when specifyng date range. #### Authorization & Content Type @@ -1465,6 +1546,8 @@ Fetch an app's ANR detail. - `key_id` (_optional_) - UUID of the last item. Used for keyset based pagination. Should be used along with `key_timestamp` & `limit`. - `key_timestamp` (_optional_) - ISO8601 timestamp of the last item. Used for keyset based pagination. Should be used along with `key_id` & `limit`. - `limit` (_optional_) - Number of items to return. Used for keyset based pagination. Should be used along with `key_id` & `key_timestamp`. + - `filter_short_code` (_optional_) - Code representing combination of filters. + - `ud_expression` (_optional_) - Expression in JSON to filter using user defined attributes. - For multiple comma separated fields, make sure no whitespace characters exist before or after comma. #### Authorization & Content Type @@ -1738,6 +1821,8 @@ Fetch an app's ANR detail instances aggregated by date range & version. - `network_providers` (_optional_) - List of comma separated network provider identifier strings to return only matching crashes. - `network_types` (_optional_) - List of comma separated network type identifier strings to return only matching crashes. - `network_generations` (_optional_) - List of comma separated network generation identifier strings to return only matching crashes. + - `filter_short_code` (_optional_) - Code representing combination of filters. + - `ud_expression` (_optional_) - Expression in JSON to filter using user defined attributes. - For multiple comma separated fields, make sure no whitespace characters exist before or after comma. #### Authorization & Content Type @@ -1981,6 +2066,8 @@ Fetch an app's sessions by applying various optional filters. - `free_text` (_optional_) - A sequence of characters used to filter sessions matching various criteria like user_id, even type, exception message and so on. - `offset` (_optional_) - Number of items to skip when paginating. Use with `limit` parameter to control amount of items fetched. - `limit` (_optional_) - Number of items to return. Used for pagination. Should be used along with `offset`. + - `filter_short_code` (_optional_) - Code representing combination of filters. + - `ud_expression` (_optional_) - Expression in JSON to filter using user defined attributes. - For multiple comma separated fields, make sure no whitespace characters exist before or after comma. - Pass `limit` and `offset` values to paginate results @@ -2381,6 +2468,12 @@ These headers must be present in each request. "OkHttp https://httpbin.org/...": [ { "event_type": "http", + "user_defined_attribute": { + "username": "alice", + "paid_user": true, + "credit_balance": 12345, + "latitude": 30.2661403415387 + }, "thread_name": "OkHttp https://httpbin.org/...", "user_triggered": false, "url": "https://httpbin.org/", @@ -2414,6 +2507,12 @@ These headers must be present in each request. "Thread-2": [ { "event_type": "anr", + "user_defined_attribute": { + "username": "alice", + "paid_user": true, + "credit_balance": 12345, + "latitude": 30.2661403415387 + }, "title": "sh.measure.android.anr.AnrError@ExceptionDemoActivity.kt:66", "thread_name": "Thread-2", "stacktrace": "sh.measure.android.anr.AnrError: Application Not Responding for at least 5000 ms.\n\tat sh.measure.sample.ExceptionDemoActivity.deadLock$lambda$10(ExceptionDemoActivity.kt:66)\n\tat sh.measure.sample.ExceptionDemoActivity.$r8$lambda$G4MY09CRhRk9ettfD7HPDD_b1n4\n\tat sh.measure.sample.ExceptionDemoActivity$$ExternalSyntheticLambda0.run(R8$$SyntheticClass)\n\tat android.os.Handler.handleCallback(Handler.java:942)\n\tat android.os.Handler.dispatchMessage(Handler.java:99)\n\tat android.os.Looper.loopOnce(Looper.java:201)\n\tat android.os.Looper.loop(Looper.java:288)\n\tat android.app.ActivityThread.main(ActivityThread.java:7872)\n\tat java.lang.reflect.Method.invoke(Method.java:-2)\n\tat com.android.internal.os.RuntimeInit$MethodAndArgsCaller.run(RuntimeInit.java:548)\n\tat com.android.internal.os.ZygoteInit.main(ZygoteInit.java:936)", @@ -2433,6 +2532,12 @@ These headers must be present in each request. "main": [ { "event_type": "lifecycle_activity", + "user_defined_attribute": { + "username": "alice", + "paid_user": true, + "credit_balance": 12345, + "latitude": 30.2661403415387 + }, "thread_name": "main", "type": "created", "class_name": "sh.measure.sample.ExceptionDemoActivity", @@ -2442,12 +2547,24 @@ These headers must be present in each request. }, { "event_type": "lifecycle_app", + "user_defined_attribute": { + "username": "alice", + "paid_user": true, + "credit_balance": 12345, + "latitude": 30.2661403415387 + }, "thread_name": "main", "type": "foreground", "timestamp": "2024-05-03T23:34:17.74Z" }, { "event_type": "lifecycle_activity", + "user_defined_attribute": { + "username": "alice", + "paid_user": true, + "credit_balance": 12345, + "latitude": 30.2661403415387 + }, "thread_name": "main", "type": "resumed", "class_name": "sh.measure.sample.ExceptionDemoActivity", @@ -2457,12 +2574,24 @@ These headers must be present in each request. }, { "event_type": "cold_launch", + "user_defined_attribute": { + "username": "alice", + "paid_user": true, + "credit_balance": 12345, + "latitude": 30.2661403415387 + }, "thread_name": "main", "duration": 698, "timestamp": "2024-05-03T23:34:17.825Z" }, { "event_type": "gesture_click", + "user_defined_attribute": { + "username": "alice", + "paid_user": true, + "credit_balance": 12345, + "latitude": 30.2661403415387 + }, "thread_name": "main", "target": "com.google.android.material.button.MaterialButton", "target_id": "btn_okhttp", @@ -2474,6 +2603,12 @@ These headers must be present in each request. }, { "event_type": "lifecycle_activity", + "user_defined_attribute": { + "username": "alice", + "paid_user": true, + "credit_balance": 12345, + "latitude": 30.2661403415387 + }, "thread_name": "main", "type": "paused", "class_name": "sh.measure.sample.ExceptionDemoActivity", @@ -2483,6 +2618,12 @@ These headers must be present in each request. }, { "event_type": "lifecycle_activity", + "user_defined_attribute": { + "username": "alice", + "paid_user": true, + "credit_balance": 12345, + "latitude": 30.2661403415387 + }, "thread_name": "main", "type": "created", "class_name": "sh.measure.sample.OkHttpActivity", @@ -2492,6 +2633,12 @@ These headers must be present in each request. }, { "event_type": "lifecycle_activity", + "user_defined_attribute": { + "username": "alice", + "paid_user": true, + "credit_balance": 12345, + "latitude": 30.2661403415387 + }, "thread_name": "main", "type": "resumed", "class_name": "sh.measure.sample.OkHttpActivity", @@ -2501,6 +2648,12 @@ These headers must be present in each request. }, { "event_type": "lifecycle_activity", + "user_defined_attribute": { + "username": "alice", + "paid_user": true, + "credit_balance": 12345, + "latitude": 30.2661403415387 + }, "thread_name": "main", "type": "paused", "class_name": "sh.measure.sample.OkHttpActivity", @@ -2510,6 +2663,12 @@ These headers must be present in each request. }, { "event_type": "lifecycle_activity", + "user_defined_attribute": { + "username": "alice", + "paid_user": true, + "credit_balance": 12345, + "latitude": 30.2661403415387 + }, "thread_name": "main", "type": "resumed", "class_name": "sh.measure.sample.ExceptionDemoActivity", @@ -2519,6 +2678,12 @@ These headers must be present in each request. }, { "event_type": "lifecycle_activity", + "user_defined_attribute": { + "username": "alice", + "paid_user": true, + "credit_balance": 12345, + "latitude": 30.2661403415387 + }, "thread_name": "main", "type": "destroyed", "class_name": "sh.measure.sample.OkHttpActivity", @@ -2528,6 +2693,12 @@ These headers must be present in each request. }, { "event_type": "gesture_click", + "user_defined_attribute": { + "username": "alice", + "paid_user": true, + "credit_balance": 12345, + "latitude": 30.2661403415387 + }, "thread_name": "main", "target": "com.google.android.material.button.MaterialButton", "target_id": "btn_deadlock", @@ -2541,6 +2712,12 @@ These headers must be present in each request. "msr-bg": [ { "event_type": "app_exit", + "user_defined_attribute": { + "username": "alice", + "paid_user": true, + "credit_balance": 12345, + "latitude": 30.2661403415387 + }, "thread_name": "msr-bg", "reason": "ANR", "importance": "FOREGROUND", @@ -2553,6 +2730,12 @@ These headers must be present in each request. "msr-ee": [ { "event_type": "http", + "user_defined_attribute": { + "username": "alice", + "paid_user": true, + "credit_balance": 12345, + "latitude": 30.2661403415387 + }, "thread_name": "msr-ee", "user_triggered": false, "url": "http://10.0.2.2:8080/events", @@ -2572,6 +2755,12 @@ These headers must be present in each request. }, { "event_type": "http", + "user_defined_attribute": { + "username": "alice", + "paid_user": true, + "credit_balance": 12345, + "latitude": 30.2661403415387 + }, "thread_name": "msr-ee", "user_triggered": false, "url": "http://10.0.2.2:8080/events", @@ -2591,6 +2780,12 @@ These headers must be present in each request. }, { "event_type": "http", + "user_defined_attribute": { + "username": "alice", + "paid_user": true, + "credit_balance": 12345, + "latitude": 30.2661403415387 + }, "thread_name": "msr-ee", "user_triggered": false, "url": "http://10.0.2.2:8080/events", @@ -2610,6 +2805,12 @@ These headers must be present in each request. }, { "event_type": "http", + "user_defined_attribute": { + "username": "alice", + "paid_user": true, + "credit_balance": 12345, + "latitude": 30.2661403415387 + }, "thread_name": "msr-ee", "user_triggered": false, "url": "http://10.0.2.2:8080/events", @@ -2629,6 +2830,12 @@ These headers must be present in each request. }, { "event_type": "http", + "user_defined_attribute": { + "username": "alice", + "paid_user": true, + "credit_balance": 12345, + "latitude": 30.2661403415387 + }, "thread_name": "msr-ee", "user_triggered": false, "url": "http://10.0.2.2:8080/events", @@ -2648,6 +2855,12 @@ These headers must be present in each request. }, { "event_type": "http", + "user_defined_attribute": { + "username": "alice", + "paid_user": true, + "credit_balance": 12345, + "latitude": 30.2661403415387 + }, "thread_name": "msr-ee", "user_triggered": false, "url": "http://10.0.2.2:8080/events", @@ -2667,6 +2880,12 @@ These headers must be present in each request. }, { "event_type": "http", + "user_defined_attribute": { + "username": "alice", + "paid_user": true, + "credit_balance": 12345, + "latitude": 30.2661403415387 + }, "thread_name": "msr-ee", "user_triggered": false, "url": "http://10.0.2.2:8080/events", @@ -2686,6 +2905,12 @@ These headers must be present in each request. }, { "event_type": "http", + "user_defined_attribute": { + "username": "alice", + "paid_user": true, + "credit_balance": 12345, + "latitude": 30.2661403415387 + }, "thread_name": "msr-ee", "user_triggered": false, "url": "http://10.0.2.2:8080/events", @@ -2705,6 +2930,12 @@ These headers must be present in each request. }, { "event_type": "http", + "user_defined_attribute": { + "username": "alice", + "paid_user": true, + "credit_balance": 12345, + "latitude": 30.2661403415387 + }, "thread_name": "msr-ee", "user_triggered": false, "url": "http://10.0.2.2:8080/events", @@ -2724,6 +2955,12 @@ These headers must be present in each request. }, { "event_type": "http", + "user_defined_attribute": { + "username": "alice", + "paid_user": true, + "credit_balance": 12345, + "latitude": 30.2661403415387 + }, "thread_name": "msr-ee", "user_triggered": false, "url": "http://10.0.2.2:8080/events", @@ -2743,6 +2980,12 @@ These headers must be present in each request. }, { "event_type": "http", + "user_defined_attribute": { + "username": "alice", + "paid_user": true, + "credit_balance": 12345, + "latitude": 30.2661403415387 + }, "thread_name": "msr-ee", "user_triggered": false, "url": "http://10.0.2.2:8080/events", diff --git a/docs/api/sdk/README.md b/docs/api/sdk/README.md index ec72ba3cf..cc2fa260f 100644 --- a/docs/api/sdk/README.md +++ b/docs/api/sdk/README.md @@ -19,6 +19,7 @@ Find all the endpoints, resources and detailed documentation for Measure SDK RES - [Status Codes \& Troubleshooting](#status-codes--troubleshooting-1) - [References](#references) - [Attributes](#attributes) + - [User Defined Attributes](#user-defined-attributes) - [Attachments](#attachments) - [Events](#events) - [Event Types](#event-types) @@ -156,7 +157,7 @@ To understand the shape of the multipart/form-data payload, take a look at this --PieBoundary123456789012345678901234567 Content-Disposition: form-data; name="event" -{"type":"string","id":"233a2fbc-a0d1-4912-a92f-9e43e72afbc6","session_id":"633a2fbc-a0d1-4912-a92f-9e43e72afbc6","string":{"severity_text":"INFO","string":"This is a log from the Android logcat"},"timestamp":"2023-08-24T14:51:38.000000534Z","attribute":{"user_id":null,"installation_id":"322a2fbc-a0d1-1212-a92f-9e43e72afbc7","device_name":"sunfish","device_model":"SM-G950F","device_manufacturer":"samsung","device_type":"phone","device_is_foldable":true,"device_is_physical":false,"device_density_dpi":100,"device_width_px":480,"device_height_px":800,"device_density":2,"os_name":"android","os_version":"31","platform":"android","app_version":"1.0.1","app_build":"576358","app_unique_id":"com.example.app","network_type":"cellular","network_provider":"airtel","network_generation":"4g","measure_sdk_version":"0.0.1"},"attachments":[]} +{"type":"string","id":"233a2fbc-a0d1-4912-a92f-9e43e72afbc6","session_id":"633a2fbc-a0d1-4912-a92f-9e43e72afbc6","string":{"severity_text":"INFO","string":"This is a log from the Android logcat"},"timestamp":"2023-08-24T14:51:38.000000534Z","attribute":{"user_id":null,"installation_id":"322a2fbc-a0d1-1212-a92f-9e43e72afbc7","device_name":"sunfish","device_model":"SM-G950F","device_manufacturer":"samsung","device_type":"phone","device_is_foldable":true,"device_is_physical":false,"device_density_dpi":100,"device_width_px":480,"device_height_px":800,"device_density":2,"os_name":"android","os_version":"31","platform":"android","app_version":"1.0.1","app_build":"576358","app_unique_id":"com.example.app","network_type":"cellular","network_provider":"airtel","network_generation":"4g","measure_sdk_version":"0.0.1"},"user_defined_attribute":{"username":"alice","paid_user":true,"credit_balance":12345,"latitude":30.2661403415387},"attachments":[]} --PieBoundary123456789012345678901234567 Content-Disposition: form-data; name="event" @@ -315,6 +316,42 @@ Events can contain the following attributes, some of which are mandatory. | `network_provider` | string | No | Example: airtel, T-mobile or "unknown" if unavailable. | | `network_generation` | string | No | One of:
- 2g
- 3g
- 4g
- 5g
- unknown | +### User Defined Attributes + +Events can optionally contain attributes defined by the SDK user. A `user_defined_attribute` is a JSON key/value pair object. There are some constraints you should be aware of. + +- An event may contain a maximum of 100 user defined attributes. +- Key names should not exceed 256 characters. +- Key names must be unique in a user defined attribute key/value object. +- Key names must only contain lowercase alphabets, numbers, underscores and hyphens. +- Value can be regular JSON types. String, Boolean, Number. +- String values should not exceed 256 characters. + +```jsonc +{ + "id": "1c8a5e51-4d7d-4b2c-9be8-1abb31d38f90", + "type": "gesture_click", + "session_id": "633a2fbc-a0d1-4912-a92f-9e43e72afbc6", + "timestamp": "2023-08-24T14:51:41.000000534Z", + "user_triggered": false, + "gesture_click": { + // snip gesture_click fields + }, + "attribute": { + // snip attributes fields + }, + "user_defined_attribute": { + "username": "alice", + "paid_user": true, + "credit_balance": 12345, + "latitude": 30.2661403415387 + }, + "attachments": { + // snip attachment fields + } +} +``` + ### Attachments Attachments are arbitrary files associated with the session each having the following properties. @@ -342,6 +379,9 @@ Event objects have the following shape. Additionally, each object must contain o "attribute": { // snip attributes fields }, + "user_defined_attribute": { + // snip user defined attributes fields + }, "attachments": { // snip attachment fields } @@ -577,19 +617,19 @@ Use the `lifecycle_fragment` type for Android's fragment lifecycle events. Use the `lifecycle_view_controller` type for iOS ViewController lifecycle events. -| Field | Type | Optional | Comment | -| ----------------- | ------ | -------- | ------------------------------------------------------------------------------------------ | -| `type` | string | No | One of the following:
- `loadView`
- `viewDidLoad`
- `viewWillAppear`
- `viewDidAppear`
- `viewWillDisappear`
- `viewDidDisappear`
- `didReceiveMemoryWarning`
- `initWithNibName`
- `initWithCoder`
- `vcDeinit` | -| `class_name` | string | No | View Controller class name | +| Field | Type | Optional | Comment | +| ------------ | ------ | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `type` | string | No | One of the following:
- `loadView`
- `viewDidLoad`
- `viewWillAppear`
- `viewDidAppear`
- `viewWillDisappear`
- `viewDidDisappear`
- `didReceiveMemoryWarning`
- `initWithNibName`
- `initWithCoder`
- `vcDeinit` | +| `class_name` | string | No | View Controller class name | #### **`lifecycle_swift_ui`** Use the `lifecycle_swift_ui` type for iOS SwiftUI view lifecycle events. -| Field | Type | Optional | Comment | -| ----------------- | ------ | -------- | ------------------------------------------------------------------------------------------ | -| `type` | string | No | One of the following:
- `on_appear`
- `on_disappear` | -| `view_name` | string | No | SwiftUI View class name | +| Field | Type | Optional | Comment | +| ----------- | ------ | -------- | -------------------------------------------------------------- | +| `type` | string | No | One of the following:
- `on_appear`
- `on_disappear` | +| `view_name` | string | No | SwiftUI View class name | #### **`lifecycle_app`** diff --git a/self-host/clickhouse/20241112203748_alter_events_table.sql b/self-host/clickhouse/20241112203748_alter_events_table.sql new file mode 100644 index 000000000..87834a69e --- /dev/null +++ b/self-host/clickhouse/20241112203748_alter_events_table.sql @@ -0,0 +1,15 @@ +-- migrate:up +alter table events + add column if not exists user_defined_attribute Map(LowCardinality(String), Tuple(Enum('string' = 1, 'int64', 'float64', 'bool'), String)) codec(ZSTD(3)) after `attribute.network_provider`, + comment column if exists user_defined_attribute 'user defined attributes', + add index if not exists user_defined_attribute_key_bloom_idx mapKeys(user_defined_attribute) type bloom_filter(0.01) granularity 16, + add index if not exists user_defined_attribute_key_minmax_idx mapKeys(user_defined_attribute) type minmax granularity 16, + materialize index if exists user_defined_attribute_key_bloom_idx, + materialize index if exists user_defined_attribute_key_minmax_idx; + + +-- migrate:down +alter table events + drop column if exists user_defined_attribute, + drop index if exists user_defined_attribute_key_bloom_idx, + drop index if exists user_defined_attribute_key_minmax_idx; diff --git a/self-host/clickhouse/20241113073411_create_user_def_attrs_table.sql b/self-host/clickhouse/20241113073411_create_user_def_attrs_table.sql new file mode 100644 index 000000000..bf9b6b000 --- /dev/null +++ b/self-host/clickhouse/20241113073411_create_user_def_attrs_table.sql @@ -0,0 +1,32 @@ +-- migrate:up +create table if not exists user_def_attrs +( + `app_id` UUID not null comment 'associated app id' codec(LZ4), + `event_id` UUID not null comment 'id of the event' codec(LZ4), + `session_id` UUID not null comment 'id of the session' codec(LZ4), + `end_of_month` DateTime not null comment 'last day of the month' codec(DoubleDelta, ZSTD(3)), + `app_version` Tuple(LowCardinality(String), LowCardinality(String)) not null comment 'composite app version' codec(ZSTD(3)), + `os_version` Tuple(LowCardinality(String), LowCardinality(String)) comment 'composite os version' codec (ZSTD(3)), + `exception` Bool comment 'true if source is exception event' codec (ZSTD(3)), + `anr` Bool comment 'true if source is anr event' codec (ZSTD(3)), + `key` LowCardinality(String) comment 'key of the user defined attribute' codec (ZSTD(3)), + `type` Enum('string' = 1, 'int64', 'float64', 'bool') comment 'type of the user defined attribute' codec (ZSTD(3)), + `value` String comment 'value of the user defined attribute' codec (ZSTD(3)), + index end_of_month_minmax_idx end_of_month type minmax granularity 2, + index exception_bloom_idx exception type bloom_filter granularity 2, + index anr_bloom_idx anr type bloom_filter granularity 2, + index key_bloom_idx key type bloom_filter(0.05) granularity 1, + index key_set_idx key type set(1000) granularity 2, + index session_bloom_idx session_id type bloom_filter granularity 2 +) +engine = ReplacingMergeTree +partition by toYYYYMM(end_of_month) +order by (app_id, end_of_month, app_version, os_version, exception, anr, + key, type, value, event_id, session_id) +settings index_granularity = 8192 +comment 'derived user defined attributes'; + + +-- migrate:down +drop table if exists user_def_attrs; + diff --git a/self-host/clickhouse/20241113110703_create_user_def_attrs_mv.sql b/self-host/clickhouse/20241113110703_create_user_def_attrs_mv.sql new file mode 100644 index 000000000..498db6b4a --- /dev/null +++ b/self-host/clickhouse/20241113110703_create_user_def_attrs_mv.sql @@ -0,0 +1,28 @@ +-- migrate:up +create materialized view user_def_attrs_mv to user_def_attrs as +select distinct app_id, + id as event_id, + session_id, + toLastDayOfMonth(timestamp) as end_of_month, + (toString(attribute.app_version), + toString(attribute.app_build)) as app_version, + (toString(attribute.os_name), + toString(attribute.os_version)) as os_version, + if(events.type = 'exception' and exception.handled = false, + true, false) as exception, + if(events.type = 'anr', true, false) as anr, + arr_key as key, + tupleElement(arr_val, 1) as type, + tupleElement(arr_val, 2) as value +from events + array join + mapKeys(user_defined_attribute) as arr_key, + mapValues(user_defined_attribute) as arr_val +where length(user_defined_attribute) > 0 +group by app_id, end_of_month, app_version, os_version, events.type, + exception.handled, key, type, value, event_id, session_id +order by app_id; + + +-- migrate:down +drop view if exists user_def_attrs_mv;