Skip to content

Commit

Permalink
feat: config query grammar (#1224)
Browse files Browse the repository at this point in the history
* feat: config query grammar

* chore: add peg to resource selector search

* chore: fix grammar errors and add comprehensive tests

* chore: remove test focus

* chore: remove duplicate ParseFilteringQueryV2 and fix remaining tests

* chore: remove extra logging

* chore: fix lint

* chore: fix relative time test

---------

Co-authored-by: Yash Mehrotra <[email protected]>
  • Loading branch information
moshloop and yashmehrotra authored Jan 1, 2025
1 parent f02849f commit 4be7147
Show file tree
Hide file tree
Showing 18 changed files with 3,416 additions and 52 deletions.
1 change: 1 addition & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and
$(CONTROLLER_GEN) object paths="./models/..."
$(CONTROLLER_GEN) object paths="./shell/..."
$(CONTROLLER_GEN) object paths="./"
PATH=$(LOCALBIN):${PATH} go generate ./...

$(LOCALBIN):
mkdir -p $(LOCALBIN)
Expand Down
13 changes: 9 additions & 4 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,6 @@ module github.com/flanksource/duty

go 1.23

toolchain go1.23.4

require (
ariga.io/atlas v0.14.2
cloud.google.com/go/cloudsqlconn v1.5.1
Expand Down Expand Up @@ -34,6 +32,7 @@ require (
github.com/invopop/jsonschema v0.12.0
github.com/itchyny/gojq v0.12.17
github.com/jackc/pgerrcode v0.0.0-20240316143900-6e2875d9b438
github.com/jackc/pgx/v4 v4.18.1
github.com/jackc/pgx/v5 v5.6.0
github.com/json-iterator/go v1.1.12
github.com/labstack/echo/v4 v4.12.0
Expand All @@ -44,6 +43,7 @@ require (
github.com/onsi/gomega v1.34.1
github.com/orcaman/concurrent-map/v2 v2.0.1
github.com/patrickmn/go-cache v2.1.0+incompatible
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.20.5
github.com/robfig/cron/v3 v3.0.1
github.com/rodaine/table v1.3.0
Expand All @@ -70,7 +70,7 @@ require (
gopkg.in/yaml.v2 v2.4.0
gopkg.in/yaml.v3 v3.0.1
gorm.io/driver/postgres v1.5.9
gorm.io/gorm v1.25.11
gorm.io/gorm v1.25.12
gorm.io/plugin/prometheus v0.1.0
k8s.io/api v0.31.1
k8s.io/apimachinery v0.31.1
Expand Down Expand Up @@ -166,8 +166,14 @@ require (
github.com/imdario/mergo v0.3.16 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/itchyny/timefmt-go v0.1.6 // indirect
github.com/jackc/chunkreader/v2 v2.0.1 // indirect
github.com/jackc/pgconn v1.14.0 // indirect
github.com/jackc/pgio v1.0.0 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgproto3/v2 v2.3.2 // indirect
github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 // indirect
github.com/jackc/pgtype v1.14.0 // indirect
github.com/jackc/puddle v1.3.0 // indirect
github.com/jackc/puddle/v2 v2.2.1 // indirect
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
github.com/jeremywohl/flatten v0.0.0-20180923035001-588fe0d4c603 // indirect
Expand Down Expand Up @@ -197,7 +203,6 @@ require (
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
github.com/pjbgf/sha1cd v0.3.0 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pkg/sftp v1.13.6 // indirect
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
github.com/prometheus/client_model v0.6.1 // indirect
Expand Down
100 changes: 98 additions & 2 deletions go.sum

Large diffs are not rendered by default.

98 changes: 79 additions & 19 deletions query/commons.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,40 +5,100 @@ import (
"net/url"
"strings"

"github.com/flanksource/duty/types"
"github.com/samber/lo"
"gorm.io/gorm"
"gorm.io/gorm/clause"
)

var LocalFilter = "deleted_at is NULL AND agent_id = '00000000-0000-0000-0000-000000000000' OR agent_id IS NULL"

type expressions struct {
In []interface{}
Prefix []string
Suffix []string
}

type Expressions []clause.Expression

// postgrestValues returns ["a", "b", "c"] as `"a","b","c"`
func postgrestValues(val []any) string {
return strings.Join(lo.Map(val, func(s any, i int) string {
return fmt.Sprintf(`"%s"`, s)
}), ",")
}

func (query FilteringQuery) AppendPostgrest(key string,
queryParam url.Values) {

if len(query.In) > 0 {
queryParam.Add(key, fmt.Sprintf("in.(%s)", postgrestValues(query.In)))
}

if len(query.Not.In) > 0 {
queryParam.Add(key, fmt.Sprintf("not.in.(%s)", postgrestValues(query.Not.In)))
}

for _, p := range query.Prefix {
queryParam.Add(key, fmt.Sprintf("like.%s*", p))
}

for _, p := range query.Suffix {
queryParam.Add(key, fmt.Sprintf("like.*%s", p))
}

}

func (e expressions) ToExpression(field string) []clause.Expression {

var clauses []clause.Expression
if len(e.In) > 0 {
clauses = append(clauses, clause.IN{Column: clause.Column{Name: field}, Values: e.In})
}

for _, p := range e.Prefix {
clauses = append(clauses, clause.Like{
Column: clause.Column{Name: field},
Value: p + "%",
})
}

for _, s := range e.Suffix {
clauses = append(clauses, clause.Like{
Column: clause.Column{Name: field},
Value: "%" + s,
})
}

return clauses
}

// ParseFilteringQuery parses a filtering query string.
// It returns four slices: 'in', 'notIN', 'prefix', and 'suffix'.
type FilteringQuery struct {
expressions
Not expressions
}

func (fq *FilteringQuery) ToExpression(field string) []clause.Expression {
exprs := fq.expressions.ToExpression(field)
not := clause.Not(fq.Not.ToExpression(field)...)
return append(exprs, not)
}

// ParseFilteringQuery parses a filtering query string.
// It returns four slices: 'in', 'notIN', 'prefix', and 'suffix'.
func ParseFilteringQuery(query string, decodeURL bool) (in []interface{}, notIN []interface{}, prefix, suffix []string, err error) {
if query == "" {
return
}

items := strings.Split(query, ",")
for _, item := range items {
if decodeURL {
item, err = url.QueryUnescape(item)
if err != nil {
return nil, nil, nil, nil, fmt.Errorf("failed to unescape query (%s): %v", item, err)
}
}
q, err := types.ParseFilteringQueryV2(query, decodeURL)

if strings.HasPrefix(item, "!") {
notIN = append(notIN, strings.TrimPrefix(item, "!"))
} else if strings.HasPrefix(item, "*") {
suffix = append(suffix, strings.TrimPrefix(item, "*"))
} else if strings.HasSuffix(item, "*") {
prefix = append(prefix, strings.TrimSuffix(item, "*"))
} else {
in = append(in, item)
}
if err != nil {
return nil, nil, nil, nil, err
}

return
return q.In, q.Not.In, q.Prefix, q.Suffix, nil
}

func parseAndBuildFilteringQuery(query, field string, decodeURL bool) ([]clause.Expression, error) {
Expand Down
10 changes: 5 additions & 5 deletions query/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -229,24 +229,24 @@ func (t *ConfigSummaryRequest) filterClause(q *gorm.DB) *gorm.DB {
var excludeClause *gorm.DB

for k, v := range t.Filter {
in, notIN, _, _, _ := ParseFilteringQuery(v, true)
query, _ := types.ParseFilteringQueryV2(v, true)

if len(notIN) > 0 {
if len(query.Not.In) > 0 {
if excludeClause == nil {
excludeClause = q
}

for _, excludeValue := range notIN {
for _, excludeValue := range query.Not.In {
excludeClause = excludeClause.Where("NOT (config_items.labels @> ?)", types.JSONStringMap{k: excludeValue.(string)})
}
}

if len(in) > 0 {
if len(query.In) > 0 {
if includeClause == nil {
includeClause = q
}

for _, includeValue := range in {
for _, includeValue := range query.In {
includeClause = includeClause.Or("config_items.labels @> ?", types.JSONStringMap{k: includeValue.(string)})
}
}
Expand Down
172 changes: 172 additions & 0 deletions query/grammar/grammar_parser.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,172 @@
//go:generate go run github.com/mna/[email protected] -o grammer.go grammer.peg
package grammar

import (
"fmt"
"strings"

"github.com/flanksource/commons/logger"
"github.com/flanksource/duty/types"
)

type Source struct {
Name string `json:"name,omitempty"`
Path []string `json:"path,omitempty"`
}

type NumberUnit struct {
Number interface{} `json:"number,omitempty"` //int64/float64
Units string `json:"units,omitempty"`
}

func makeSource(name interface{}, path interface{}) (string, error) {
ps := path.([]interface{})

paths := make([]string, 0)
for _, p := range ps {
pa := p.([]interface{})
px := pa[1:]
for _, pi := range px {
paths = append(paths, pi.(string))
}
}
return strings.Join(append([]string{name.(string)}, paths...), "."), nil
}

func makeFQFromQuery(a interface{}) (interface{}, error) {
return a.(*types.QueryField), nil
}

//nolint:unused
func makeCatchAll(f interface{}) (*types.QueryField, error) {
logger.Warnf("ctach all %v (%T)", f, f)

switch v := f.(type) {
case string:
return &types.QueryField{Op: "rest", Value: v}, nil
case []byte:
return &types.QueryField{Op: "rest", Value: string(v)}, nil
case []interface{}:

rest := ""
for _, i := range v {
rest += fmt.Sprintf("%s", i)
}
return &types.QueryField{Op: "rest", Value: rest}, nil
}
return &types.QueryField{Op: "rest", Value: f}, nil
}

func makeFQFromField(f interface{}) (*types.QueryField, error) {
return f.(*types.QueryField), nil
}

//nolint:unused
func makeQuery(a, b interface{}) (*types.QueryField, error) {
q := &types.QueryField{
Op: "or",
}

switch v := a.(type) {
case *types.QueryField:
q.Fields = append(q.Fields, v)
default:
logger.Warnf("Unknown type for query.a: %v = %T", a, a)
}

switch v := b.(type) {
case *types.QueryField:
q.Fields = append(q.Fields, v)
case []interface{}:
for _, i := range v {
switch v2 := i.(type) {
case *types.QueryField:
q.Fields = append(q.Fields, v2)

default:
logger.Warnf("Unknown array item: %v (%T)", i, i)
}
}
default:
logger.Warnf("Unknown type for query.b: %v = %T", b, b)
}

return q, nil
}

func makeAndQuery(a any, b any) (*types.QueryField, error) {

q := &types.QueryField{Op: "and"}

switch v := a.(type) {
case *types.QueryField:
q.Fields = append(q.Fields, v)

default:
logger.Warnf("Unknown type for a: %v = %T", a, a)
}

switch v := b.(type) {
case *types.QueryField:
q.Fields = append(q.Fields, v)
case []interface{}:
for _, i := range v {
switch v2 := i.(type) {
case *types.QueryField:
q.Fields = append(q.Fields, v2)
default:
logger.Warnf("Unknown array item: %v (%T)", i, i)
}
}
default:
logger.Warnf("Unknown type for b: %v = %T", b, b)
}

return q, nil
}

func makeValue(val interface{}) (interface{}, error) {
return val, nil
}

func makeMeasure(num interface{}, units interface{}) (*NumberUnit, error) {
retVal := &NumberUnit{Number: num, Units: units.(string)}

return retVal, nil
}

func stringFromChars(chars interface{}) string {
str := ""
r := chars.([]interface{})
for _, i := range r {
j := i.([]uint8)
str += string(j[0])
}
return str
}

func FlatFields(qf *types.QueryField) []string {
var fields []string
if qf.Field != "" {
fields = append(fields, qf.Field)
}
for _, f := range qf.Fields {
fields = append(fields, FlatFields(f)...)
}
return fields
}

func ParsePEG(peg string) (*types.QueryField, error) {
stats := Stats{}

v, err := Parse("", []byte(peg), Statistics(&stats, "no match"))
if err != nil {
return nil, fmt.Errorf("error parsing peg: %w", err)
}

rv, ok := v.(*types.QueryField)
if !ok {
return nil, fmt.Errorf("return type not types.QueryField")
}
return rv, nil
}
Loading

0 comments on commit 4be7147

Please sign in to comment.