From 0dd4cbe56cf0b58b4d7340b5e6b48a3b52983cc1 Mon Sep 17 00:00:00 2001 From: Debjeet Biswas Date: Wed, 4 Sep 2024 08:32:58 +0530 Subject: [PATCH] chore(backend): wip user defined attributes --- backend/api/event/event.go | 4 + backend/api/event/usedefattr.go | 137 ++++++++++++++++++ backend/api/measure/event.go | 14 ++ go.work.sum | 1 + ...903022512_add_usedef_attrs_keys_column.sql | 5 + ...03103401_add_usedef_attrs_types_column.sql | 5 + ...03103423_add_usedef_string_vals_column.sql | 5 + ...0903103435_add_usedef_bool_vals_column.sql | 5 + ...903103443_add_usedef_int64_vals_column.sql | 5 + ...3103450_add_usedef_float64_vals_column.sql | 5 + 10 files changed, 186 insertions(+) create mode 100644 backend/api/event/usedefattr.go create mode 100644 self-host/clickhouse/20240903022512_add_usedef_attrs_keys_column.sql create mode 100644 self-host/clickhouse/20240903103401_add_usedef_attrs_types_column.sql create mode 100644 self-host/clickhouse/20240903103423_add_usedef_string_vals_column.sql create mode 100644 self-host/clickhouse/20240903103435_add_usedef_bool_vals_column.sql create mode 100644 self-host/clickhouse/20240903103443_add_usedef_int64_vals_column.sql create mode 100644 self-host/clickhouse/20240903103450_add_usedef_float64_vals_column.sql diff --git a/backend/api/event/event.go b/backend/api/event/event.go index 90271ab85..2b35c9034 100644 --- a/backend/api/event/event.go +++ b/backend/api/event/event.go @@ -50,6 +50,9 @@ const ( maxNavigationToChars = 128 maxNavigationFromChars = 128 maxNavigationSourceChars = 128 + maxUseDefAttrsCount = 100 + maxUseDefAttrsKeyChars = 256 + maxUseDefAttrsValChars = 256 ) const TypeANR = "anr" @@ -360,6 +363,7 @@ type EventField struct { Type string `json:"type" binding:"required"` UserTriggered bool `json:"user_triggered" binding:"required"` Attribute Attribute `json:"attribute" binding:"required"` + UseDefAttrs UDAttribute `json:"user_defined_attributes" binding:"required"` Attachments []Attachment `json:"attachments" binding:"required"` ANR *ANR `json:"anr,omitempty"` Exception *Exception `json:"exception,omitempty"` diff --git a/backend/api/event/usedefattr.go b/backend/api/event/usedefattr.go new file mode 100644 index 000000000..40e9e2159 --- /dev/null +++ b/backend/api/event/usedefattr.go @@ -0,0 +1,137 @@ +package event + +import ( + "encoding/json" + "errors" + "fmt" + "reflect" +) + +type attr_type int + +const ( + attr_unknown attr_type = iota + attr_string + attr_number + attr_bool +) + +// type AttrNumber interface { +// int | float64 +// } + +// type AttrVal interface { +// string | bool | AttrNumber +// } + +// type UDKey struct { +// Type int +// Val string +// } + +// type UDVal[T AttrVal] struct { +// Val T +// } + +// type UDAttribute map[string]any + +// type rawAttrs map[string]any + +// UDAttribute defines User Defined Attributes designed +// to be attached to each event. An event can have multiple +// user defined attributes. +type UDAttribute struct { + rawAttrs map[string]any + keyTypes map[string]string + keys []string + types []string + stringVals []string + boolVals []bool + int64Vals []int + float64Vals []float64 +} + +// UnmarshalJSON unmarshals bytes resembling user defined +// attributes to an internal representation. +func (u *UDAttribute) UnmarshalJSON(data []byte) (err error) { + return json.Unmarshal(data, &u.rawAttrs) +} + +// MarshalJSON marshals internal representation of user defined +// attributes to JSON. +func (u UDAttribute) MarshalJSON() (data []byte, err error) { + return json.Marshal(u.rawAttrs) +} + +// Validate validate the user defined attributes bag. +func (u *UDAttribute) Validate() (err error) { + if u.rawAttrs == nil { + return errors.New("user defined attribute must not be empty") + } + + count := len(u.rawAttrs) + + if count > maxUseDefAttrsCount { + return fmt.Errorf("user defined attributes must not exceed %d items", maxUseDefAttrsCount) + } + + if u.keyTypes == nil { + u.keyTypes = make(map[string]string) + } + + index := -1 + + for k, v := range u.rawAttrs { + if len(k) > maxUseDefAttrsKeyChars { + return fmt.Errorf("user defined attribute keys must not exceed %d characters", maxUseDefAttrsKeyChars) + } + + index += 1 + + switch value := v.(type) { + case string: + if len(v.(string)) > maxUseDefAttrsValChars { + return fmt.Errorf("user defined attributes string values must not exceed %d characters", maxUseDefAttrsValChars) + } + u.keyTypes[k] = "string" + u.types = append(u.types, "string") + + continue + case bool: + u.keyTypes[k] = "bool" + u.types = append(u.types, "bool") + continue + case float64: + if reflect.TypeOf(v).Kind() == reflect.Float64 { + if v == float64(int(value)) { + u.keyTypes[k] = "int64" + u.types = append(u.types, "int64") + } else { + u.keyTypes[k] = "float64" + u.types = append(u.types, "float64") + } + } + continue + default: + return fmt.Errorf("user defined attribute values can be only string, number or boolean") + } + } + + return +} + +func (u *UDAttribute) HasItems() bool { + return len(u.rawAttrs) > 0 +} + +func (u UDAttribute) GetKeys() (keys []string) { + return u.keys +} + +func (u UDAttribute) GetTypes() (types []string) { + return u.types +} + +func (u UDAttribute) BuildArgs() (err error) { + return +} diff --git a/backend/api/measure/event.go b/backend/api/measure/event.go index 467bf1701..62a6f4b6d 100644 --- a/backend/api/measure/event.go +++ b/backend/api/measure/event.go @@ -416,10 +416,15 @@ func (e eventreq) validate() error { if err := e.events[i].Validate(); err != nil { return err } + if err := e.events[i].Attribute.Validate(); err != nil { return err } + if err := e.events[i].UseDefAttrs.Validate(); err != nil { + return err + } + if e.hasAttachments() { for j := range e.events[i].Attachments { if err := e.events[i].Attachments[j].Validate(); err != nil { @@ -528,6 +533,11 @@ func (e eventreq) ingest(ctx context.Context) error { // attachments Set(`attachments`, attachments) + // user defined attributes + if e.events[i].UseDefAttrs.HasItems() { + e.events[i].UseDefAttrs.BuildArgs() + } + // anr if e.events[i].IsANR() { row. @@ -1777,6 +1787,10 @@ func PutEvents(c *gin.Context) { return } + // temporary + // c.JSON(http.StatusOK, gin.H{"events": eventReq.events}) + // return + if seen, err := eventReq.seen(ctx); err != nil { msg := `failed to check existing event request` fmt.Println(msg, err.Error()) diff --git a/go.work.sum b/go.work.sum index 5465ef886..452f1169d 100644 --- a/go.work.sum +++ b/go.work.sum @@ -258,6 +258,7 @@ golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72 golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/exp v0.0.0-20230510235704-dd950f8aeaea h1:vLCWI/yYrdEHyN2JzIzPO3aaQJHQdp89IZBA/+azVC4= golang.org/x/exp v0.0.0-20230510235704-dd950f8aeaea/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= diff --git a/self-host/clickhouse/20240903022512_add_usedef_attrs_keys_column.sql b/self-host/clickhouse/20240903022512_add_usedef_attrs_keys_column.sql new file mode 100644 index 000000000..52dd27ce1 --- /dev/null +++ b/self-host/clickhouse/20240903022512_add_usedef_attrs_keys_column.sql @@ -0,0 +1,5 @@ +-- migrate:up +alter table events add column if not exists `usedef_attrs.keys` Array(String) after `navigation.source`; + +-- migrate:down +drop column if exists `usedef_attrs.keys`; diff --git a/self-host/clickhouse/20240903103401_add_usedef_attrs_types_column.sql b/self-host/clickhouse/20240903103401_add_usedef_attrs_types_column.sql new file mode 100644 index 000000000..45fb82655 --- /dev/null +++ b/self-host/clickhouse/20240903103401_add_usedef_attrs_types_column.sql @@ -0,0 +1,5 @@ +-- migrate:up +alter table events add column if not exists `usedef_attrs.types` Array(String) after `usedef_attrs.keys`; + +-- migrate:down +drop column if exists `usedef_attrs.types`; diff --git a/self-host/clickhouse/20240903103423_add_usedef_string_vals_column.sql b/self-host/clickhouse/20240903103423_add_usedef_string_vals_column.sql new file mode 100644 index 000000000..9f488c6d0 --- /dev/null +++ b/self-host/clickhouse/20240903103423_add_usedef_string_vals_column.sql @@ -0,0 +1,5 @@ +-- migrate:up +alter table events add column if not exists `usedef_attrs.string_vals` Array(String) after `usedef_attrs.types`; + +-- migrate:down +drop column if exists `usedef_attrs.string_vals`; diff --git a/self-host/clickhouse/20240903103435_add_usedef_bool_vals_column.sql b/self-host/clickhouse/20240903103435_add_usedef_bool_vals_column.sql new file mode 100644 index 000000000..ff626fc9e --- /dev/null +++ b/self-host/clickhouse/20240903103435_add_usedef_bool_vals_column.sql @@ -0,0 +1,5 @@ +-- migrate:up +alter table events add column if not exists `usedef_attrs.bool_vals` Array(Bool) after `usedef_attrs.string_vals`; + +-- migrate:down +drop column if exists `usedef_attrs.bool_vals`; diff --git a/self-host/clickhouse/20240903103443_add_usedef_int64_vals_column.sql b/self-host/clickhouse/20240903103443_add_usedef_int64_vals_column.sql new file mode 100644 index 000000000..733eaf391 --- /dev/null +++ b/self-host/clickhouse/20240903103443_add_usedef_int64_vals_column.sql @@ -0,0 +1,5 @@ +-- migrate:up +alter table events add column if not exists `usedef_attrs.int64_vals` Array(Int64) after `usedef_attrs.bool_vals`; + +-- migrate:down +drop column if exists `usedef_attrs.int64_vals`; diff --git a/self-host/clickhouse/20240903103450_add_usedef_float64_vals_column.sql b/self-host/clickhouse/20240903103450_add_usedef_float64_vals_column.sql new file mode 100644 index 000000000..3401bd4e0 --- /dev/null +++ b/self-host/clickhouse/20240903103450_add_usedef_float64_vals_column.sql @@ -0,0 +1,5 @@ +-- migrate:up +alter table events add column if not exists `usedef_attrs.float64_vals` Array(Float64) after `usedef_attrs.int64_vals`; + +-- migrate:down +drop column if exists `usedef_attrs.float64_vals`;