feat(backend): use short codes for list filters

closes #1408

backend/api/filter/appfilter.go

@@ -3,8 +3,10 @@ package filter
 import (
 	"backend/api/pairs"
 	"backend/api/server"
-	"backend/api/text"
 	"context"
+	"crypto/md5"
+	"encoding/hex"
+	"encoding/json"
 	"fmt"
 	"slices"
 	"time"
@@ -50,6 +52,9 @@ type AppFilter struct {
 	// client
 	Timezone string `form:"timezone"`
 
+	// Represents a short code for list filters
+	FilterShortCode string `form:"filter_short_code"`
+
 	// Versions is the list of version string
 	// to be matched & filtered on.
 	Versions []string `form:"versions"`
@@ -141,6 +146,20 @@ type FilterList struct {
 	DeviceNames         []string `json:"device_names"`
 }
 
+// Hash generates an MD5 hash of the FilterList struct.
+func (f *FilterList) Hash() (string, error) {
+	data, err := json.Marshal(f)
+	if err != nil {
+		return "", err
+	}
+
+	// Compute MD5 hash
+	md5Hash := md5.Sum(data)
+
+	// Convert hash to hex string
+	return hex.EncodeToString(md5Hash[:]), nil
+}
+
 // Versions represents a list of
 // (version, code) pairs.
 type Versions struct {
@@ -227,48 +246,43 @@ func (af *AppFilter) OSVersionPairs() (osVersions *pairs.Pairs[string, string],
 // Expand expands comma separated fields to slice
 // of strings
 func (af *AppFilter) Expand() {
-	if len(af.Versions) > 0 {
-		af.Versions = text.SplitTrimEmpty(af.Versions[0], ",")
+	filters, err := GetFiltersFromFilterShortCode(af.FilterShortCode, af.AppID)
+	if err != nil {
+		return
 	}
 
-	if len(af.VersionCodes) > 0 {
-		af.VersionCodes = text.SplitTrimEmpty(af.VersionCodes[0], ",")
+	if len(filters.Versions) > 0 {
+		af.Versions = filters.Versions
 	}
-
-	if len(af.OsNames) > 0 {
-		af.OsNames = text.SplitTrimEmpty(af.OsNames[0], ",")
+	if len(filters.VersionCodes) > 0 {
+		af.VersionCodes = filters.VersionCodes
 	}
-
-	if len(af.OsVersions) > 0 {
-		af.OsVersions = text.SplitTrimEmpty(af.OsVersions[0], ",")
+	if len(filters.OsNames) > 0 {
+		af.OsNames = filters.OsNames
 	}
-
-	if len(af.Countries) > 0 {
-		af.Countries = text.SplitTrimEmpty(af.Countries[0], ",")
+	if len(filters.OsVersions) > 0 {
+		af.OsVersions = filters.OsVersions
 	}
-
-	if len(af.DeviceNames) > 0 {
-		af.DeviceNames = text.SplitTrimEmpty(af.DeviceNames[0], ",")
+	if len(filters.Countries) > 0 {
+		af.Countries = filters.Countries
 	}
-
-	if len(af.DeviceManufacturers) > 0 {
-		af.DeviceManufacturers = text.SplitTrimEmpty(af.DeviceManufacturers[0], ",")
+	if len(filters.DeviceNames) > 0 {
+		af.DeviceNames = filters.DeviceNames
 	}
-
-	if len(af.Locales) > 0 {
-		af.Locales = text.SplitTrimEmpty(af.Locales[0], ",")
+	if len(filters.DeviceManufacturers) > 0 {
+		af.DeviceManufacturers = filters.DeviceManufacturers
 	}
-
-	if len(af.NetworkProviders) > 0 {
-		af.NetworkProviders = text.SplitTrimEmpty(af.NetworkProviders[0], ",")
+	if len(filters.DeviceLocales) > 0 {
+		af.Locales = filters.DeviceLocales
 	}
-
-	if len(af.NetworkTypes) > 0 {
-		af.NetworkTypes = text.SplitTrimEmpty(af.NetworkTypes[0], ",")
+	if len(filters.NetworkProviders) > 0 {
+		af.NetworkProviders = filters.NetworkProviders
 	}
-
-	if len(af.NetworkGenerations) > 0 {
-		af.NetworkGenerations = text.SplitTrimEmpty(af.NetworkGenerations[0], ",")
+	if len(filters.NetworkTypes) > 0 {
+		af.NetworkTypes = filters.NetworkTypes
+	}
+	if len(filters.NetworkGenerations) > 0 {
+		af.NetworkGenerations = filters.NetworkGenerations
 	}
 }
 

backend/api/filter/shortfilters.go

@@ -0,0 +1,99 @@
+package filter
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"time"
+
+	"backend/api/chrono"
+	"backend/api/server"
+
+	"github.com/google/uuid"
+	"github.com/leporo/sqlf"
+)
+
+type ShortFilters struct {
+	Code      string
+	AppId     uuid.UUID
+	Filters   FilterList
+	UpdatedAt time.Time
+	CreatedAt time.Time
+}
+
+type ShortFiltersPayload struct {
+	Filters FilterList `json:"filters"`
+}
+
+func (shortFilters *ShortFilters) MarshalJSON() ([]byte, error) {
+	apiMap := make(map[string]any)
+	apiMap["code"] = shortFilters.Code
+	apiMap["app_id"] = shortFilters.AppId
+	apiMap["filters"] = shortFilters.Filters
+	apiMap["created_at"] = shortFilters.CreatedAt.Format(chrono.ISOFormatJS)
+	apiMap["updated_at"] = shortFilters.UpdatedAt.Format(chrono.ISOFormatJS)
+	return json.Marshal(apiMap)
+}
+
+func NewShortFilters(appId uuid.UUID, filters FilterList) (*ShortFilters, error) {
+	hash, err := filters.Hash()
+	if err != nil {
+		return nil, err
+	}
+
+	return &ShortFilters{
+		Code:      hash,
+		AppId:     appId,
+		Filters:   filters,
+		CreatedAt: time.Now(),
+		UpdatedAt: time.Now(),
+	}, nil
+}
+
+func (shortFilters *ShortFilters) Create() error {
+	// If already exists, just return
+	_, err := GetFiltersFromFilterShortCode(shortFilters.Code, shortFilters.AppId)
+	if err == nil {
+		return nil
+	}
+
+	stmt := sqlf.PostgreSQL.InsertInto("public.short_filters").
+		Set("code", shortFilters.Code).
+		Set("app_id", shortFilters.AppId).
+		Set("filters", shortFilters.Filters).
+		Set("created_at", shortFilters.CreatedAt).
+		Set("updated_at", shortFilters.UpdatedAt)
+	defer stmt.Close()
+
+	_, err = server.Server.PgPool.Exec(context.Background(), stmt.String(), stmt.Args()...)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// Returns filters for a given short code and appId. If it doesn't exist, returns an error
+func GetFiltersFromFilterShortCode(filterShortCode string, appId uuid.UUID) (*FilterList, error) {
+	var filters FilterList
+
+	stmt := sqlf.PostgreSQL.
+		Select("filters").
+		From("public.short_filters").
+		Where("code = ?", filterShortCode).
+		Where("app_id = ?", appId)
+	defer stmt.Close()
+
+	err := server.Server.PgPool.QueryRow(context.Background(), stmt.String(), stmt.Args()...).Scan(&filters)
+
+	if err != nil {
+		fmt.Printf("Error fetching filters from filter short code %v: %v\n", filterShortCode, err)
+		return nil, err
+	}
+
+	return &filters, nil
+}
+
+func (shortFilters *ShortFilters) String() string {
+	return fmt.Sprintf("ShortFilters - code: %s, app_id: %s, filters: %v, created_at: %v, updated_at: %v ", shortFilters.Code, shortFilters.AppId, shortFilters.Filters, shortFilters.CreatedAt, shortFilters.UpdatedAt)
+}

backend/api/main.go

@@ -109,6 +109,7 @@ func main() {
 		apps.GET(":id/settings", measure.GetAppSettings)
 		apps.PATCH(":id/settings", measure.UpdateAppSettings)
 		apps.PATCH(":id/rename", measure.RenameApp)
+		apps.POST(":id/shortFilters", measure.CreateShortFilters)
 	}
 
 	teams := r.Group("/teams", measure.ValidateAccessToken())

backend/api/measure/app.go

@@ -5066,3 +5066,64 @@ func RenameApp(c *gin.Context) {
 
 	c.JSON(http.StatusOK, gin.H{"ok": "done"})
 }
+
+func CreateShortFilters(c *gin.Context) {
+	userId := c.GetString("userId")
+	appId, err := uuid.Parse(c.Param("id"))
+	if err != nil {
+		msg := `app id invalid or missing`
+		fmt.Println(msg, err)
+		c.JSON(http.StatusBadRequest, gin.H{"error": msg})
+		return
+	}
+
+	app := App{
+		ID: &appId,
+	}
+
+	team, err := app.getTeam(c)
+	if err != nil {
+		msg := "failed to get team from app id"
+		fmt.Println(msg, err)
+		c.JSON(http.StatusInternalServerError, gin.H{"error": msg})
+		return
+	}
+	if team == nil {
+		msg := fmt.Sprintf("no team exists for app [%s]", app.ID)
+		c.JSON(http.StatusBadRequest, gin.H{"error": msg})
+		return
+	}
+
+	ok, err := PerformAuthz(userId, team.ID.String(), *ScopeAppRead)
+	if err != nil {
+		msg := `couldn't perform authorization checks`
+		fmt.Println(msg, err)
+		c.JSON(http.StatusInternalServerError, gin.H{"error": msg})
+		return
+	}
+	if !ok {
+		msg := fmt.Sprintf(`you don't have permissions to create short filters in team [%s]`, team.ID.String())
+		c.JSON(http.StatusForbidden, gin.H{"error": msg})
+		return
+	}
+
+	var payload filter.ShortFiltersPayload
+	if err := c.ShouldBindJSON(&payload); err != nil {
+		msg := `failed to parse filters json payload`
+		fmt.Println(msg, err)
+		c.JSON(http.StatusBadRequest, gin.H{"error": msg})
+		return
+	}
+
+	shortFilters, err := filter.NewShortFilters(appId, payload.Filters)
+	if err != nil {
+		msg := `failed to create generate filter hash`
+		fmt.Println(msg, err)
+		c.JSON(http.StatusBadRequest, gin.H{"error": msg})
+		return
+	}
+
+	shortFilters.Create()
+
+	c.JSON(http.StatusOK, gin.H{"filter_short_code": shortFilters.Code})
+}

backend/cleanup/cleanup/cleanup.go

@@ -32,6 +32,10 @@ type StaleData struct {
 }
 
 func DeleteStaleData(ctx context.Context) {
+	// Delete shortened filters
+	deleteStaleShortenedFilters(ctx)
+
+	// Delete events and attachments
 	staleData, err := fetchStaleData(ctx)
 
 	if err != nil {
@@ -40,7 +44,6 @@ func DeleteStaleData(ctx context.Context) {
 	}
 
 	for _, st := range staleData {
-
 		// Delete attachments from object storage
 		if len(st.Attachments) > 0 {
 			fmt.Printf("Deleting %v attachments for app_id: %v\n", len(st.Attachments), st.AppID)
@@ -73,7 +76,21 @@ func DeleteStaleData(ctx context.Context) {
 	}
 
 	staleDataJson, _ := json.MarshalIndent(staleData, "", "    ")
-	fmt.Printf("Succesfully deleted stale data %v\n", string(staleDataJson))
+	fmt.Printf("Succesfully deleted stale stale data %v\n", string(staleDataJson))
+}
+
+func deleteStaleShortenedFilters(ctx context.Context) {
+	threshold := time.Now().Add(-60 * time.Minute) // 1 hour expiry
+	stmt := sqlf.PostgreSQL.DeleteFrom("public.short_filters").
+		Where("created_at < ?", threshold)
+
+	_, err := server.Server.PgPool.Exec(ctx, stmt.String(), stmt.Args()...)
+	if err != nil {
+		fmt.Printf("Failed to delete stale short filter codes: %v\n", err)
+		return
+	}
+
+	fmt.Printf("Succesfully deleted stale short filters\n")
 }
 
 func fetchStaleData(ctx context.Context) ([]StaleData, error) {