diff --git a/generators/esgen/bridgeutil.go b/generators/esgen/bridgeutil.go index a4fab73..a509ddf 100644 --- a/generators/esgen/bridgeutil.go +++ b/generators/esgen/bridgeutil.go @@ -3,6 +3,8 @@ package esgen import ( "fmt" "strconv" + "strings" + "time" "github.com/araddon/dateparse" "github.com/lytics/qlbridge/expr" @@ -133,6 +135,142 @@ func makeRange(lhs *gentypes.FieldType, op lex.TokenType, rhs expr.Node) (any, e return r, nil } +// makeRecurringQuery builds a Painless script matching documents whose date +// field lands on a recurrence of itself relative to `now`. `period` is either a +// string ("yearly"/"monthly"/"weekly"/"daily") or an integer node (every N +// days). offsetDays shifts the recurrence (e.g. yearly+182 = half-birthday). +func makeRecurringQuery(lhs *gentypes.FieldType, period expr.Node, offsetDays int, now time.Time) (any, error) { + if lhs.Nested() { + return nil, fmt.Errorf("'recurring' unsupported for nested/map field %q", lhs.Field) + } + if lhs.Type != value.TimeType { + return nil, fmt.Errorf("'recurring' requires a date field, got %s for %q", lhs.Type, lhs.Field) + } + + q := painlessQuote(lhs.Field) + exists := fmt.Sprintf("doc[%s].size() != 0", q) + + // Numeric period => every N days. + if num, ok := period.(*expr.NumberNode); ok { + if !num.IsInt || num.Int64 <= 0 { + return nil, fmt.Errorf("'recurring' day period must be a positive integer, got %v", period) + } + n := num.Int64 + todayDay := now.UTC().Unix() / 86400 + src := fmt.Sprintf("if (%s) { long d = params.todayDay - (doc[%s].value.toInstant().getEpochSecond() / 86400) - params.offset; return d >= 0 && d %% params.n == 0; } return false;", + exists, q) + return Script(src, map[string]any{"todayDay": todayDay, "offset": offsetDays, "n": n}), nil + } + + pnode, ok := period.(*expr.StringNode) + if !ok { + return nil, fmt.Errorf("'recurring' period must be a string or integer, got %s", period.NodeType()) + } + + target := now.UTC().AddDate(0, 0, -offsetDays) + switch strings.ToLower(pnode.Text) { + case "yearly": + src := fmt.Sprintf("%s && doc[%s].value.getMonthValue() == params.month && doc[%s].value.getDayOfMonth() == params.day", exists, q, q) + return Script(src, map[string]any{"month": int(target.Month()), "day": target.Day()}), nil + case "monthly": + src := fmt.Sprintf("%s && doc[%s].value.getDayOfMonth() == params.day", exists, q) + return Script(src, map[string]any{"day": target.Day()}), nil + case "weekly": + src := fmt.Sprintf("%s && doc[%s].value.getDayOfWeek().getValue() == params.dow", exists, q) + return Script(src, map[string]any{"dow": isoDayOfWeek(target)}), nil + case "daily": + return Script(exists, nil), nil + default: + return nil, fmt.Errorf("'recurring' unsupported period %q (want yearly/monthly/weekly/daily or an integer)", pnode.Text) + } +} + +// isoDayOfWeek maps Go's Weekday (Sunday=0) to ISO-8601 (Monday=1..Sunday=7), +// matching Painless's ZonedDateTime.getDayOfWeek().getValue(). +func isoDayOfWeek(t time.Time) int { + if wd := int(t.Weekday()); wd != 0 { + return wd + } + return 7 +} + +// makeFieldRange compares two document fields (e.g. `a < b`). Elasticsearch +// range queries can only compare a field to a constant, so a field-to-field +// comparison is expressed as a Painless script. +func makeFieldRange(lhs *gentypes.FieldType, op lex.TokenType, rhs *gentypes.FieldType) (any, error) { + if lhs.Nested() || rhs.Nested() { + return nil, fmt.Errorf("field-to-field comparison unsupported for nested/map fields: %q, %q", lhs.Field, rhs.Field) + } + if painlessTypeClass(lhs.Type) != painlessTypeClass(rhs.Type) { + return nil, fmt.Errorf("field-to-field comparison requires comparable types, got %s and %s", lhs.Type, rhs.Type) + } + + cmp, err := painlessCompareOp(op) + if err != nil { + return nil, err + } + laccess, err := painlessFieldAccess(lhs) + if err != nil { + return nil, err + } + raccess, err := painlessFieldAccess(rhs) + if err != nil { + return nil, err + } + + src := fmt.Sprintf("doc[%s].size() != 0 && doc[%s].size() != 0 && %s %s %s", + painlessQuote(lhs.Field), painlessQuote(rhs.Field), laccess, cmp, raccess) + return Script(src, nil), nil +} + +func painlessCompareOp(op lex.TokenType) (string, error) { + switch op { + case lex.TokenGE: + return ">=", nil + case lex.TokenLE: + return "<=", nil + case lex.TokenGT: + return ">", nil + case lex.TokenLT: + return "<", nil + default: + return "", fmt.Errorf("qlindex: unsupported range operator %s", op) + } +} + +// painlessTypeClass groups field types that are mutually comparable. +func painlessTypeClass(t value.ValueType) string { + switch t { + case value.TimeType: + return "time" + case value.IntType, value.NumberType: + return "number" + default: + return "unsupported" + } +} + +func painlessFieldAccess(f *gentypes.FieldType) (string, error) { + q := painlessQuote(f.Field) + switch f.Type { + case value.TimeType: + // Day-granular: compare calendar dates, not instants, so `a < b` matches + // for the whole day. Matches the in-process evaluator's epoch-day floor. + return fmt.Sprintf("doc[%s].value.toLocalDate().toEpochDay()", q), nil + case value.IntType, value.NumberType: + return fmt.Sprintf("doc[%s].value", q), nil + default: + return "", fmt.Errorf("field-to-field comparison unsupported for type %s field %q", f.Type, f.Field) + } +} + +// painlessQuote returns a single-quoted Painless string literal. +func painlessQuote(s string) string { + s = strings.ReplaceAll(s, `\`, `\\`) + s = strings.ReplaceAll(s, `'`, `\'`) + return "'" + s + "'" +} + // makeBetween returns a range filter for Elasticsearch given the 3 nodes that // make up a comparison. func makeBetween(lhs *gentypes.FieldType, lower, upper any) (any, error) { diff --git a/generators/esgen/esgenerator.go b/generators/esgen/esgenerator.go index 3b6b641..66f829c 100644 --- a/generators/esgen/esgenerator.go +++ b/generators/esgen/esgenerator.go @@ -195,6 +195,17 @@ func (fg *FilterGenerator) binaryExpr(node *expr.BinaryNode, _ int) (any, error) switch op := node.Operator.T; op { case lex.TokenGE, lex.TokenLE, lex.TokenGT, lex.TokenLT: + // Field-to-field comparison (e.g. `a < b`): the RHS is a field + // identity rather than a constant, so it cannot be a range query. + if rident, ok := node.Args[1].(*expr.IdentityNode); ok { + if _, isScalar := scalar(rident); !isScalar { + rhs, err := fg.fieldType(node.Args[1]) + if err != nil { + return nil, err + } + return makeFieldRange(lhs, op, rhs) + } + } return makeRange(lhs, op, node.Args[1]) case lex.TokenEqual, lex.TokenEqualEqual: // the VM supports both = and == @@ -361,6 +372,26 @@ func (fg *FilterGenerator) funcExpr(node *expr.FuncNode, _ int) (any, error) { return nil, fmt.Errorf("qlindex: unsupported type for 'geodistance' distance argument. must be number, got %s", node.Args[2].NodeType()) } return makeGeoDistanceQuery(lhs, lat, lon, distance), nil + case "recurring": + // recurring(date_field, period [, offsetDays]) matches profiles whose + // date_field lands on a recurrence of itself relative to the eval-time + // "now" (fg.ts): yearly/monthly/weekly/daily anniversaries, or every N days. + if len(node.Args) < 2 || len(node.Args) > 3 { + return nil, fmt.Errorf("'recurring' function requires 2 or 3 arguments, got %d", len(node.Args)) + } + lhs, err := fg.fieldType(node.Args[0]) + if err != nil { + return nil, err + } + offset := 0 + if len(node.Args) == 3 { + off, ok := node.Args[2].(*expr.NumberNode) + if !ok || !off.IsInt { + return nil, fmt.Errorf("qlindex: 'recurring' offset must be an integer number of days, got %v", node.Args[2]) + } + offset = int(off.Int64) + } + return makeRecurringQuery(lhs, node.Args[1], offset, fg.ts) } return nil, fmt.Errorf("qlindex: unsupported function: %s", node.Name) } diff --git a/generators/esgen/estypes.go b/generators/esgen/estypes.go index 01c95c9..c22ab8b 100644 --- a/generators/esgen/estypes.go +++ b/generators/esgen/estypes.go @@ -213,3 +213,28 @@ func Wildcard(field, value string, addStars bool) *wildcard { type GeoDistanceFilter struct { GeoDistance map[string]any `json:"geo_distance"` } + +type ScriptFilter struct { + Script ScriptQry `json:"script"` +} + +type ScriptQry struct { + Script ScriptBody `json:"script"` +} + +type ScriptBody struct { + Source string `json:"source"` + Lang string `json:"lang"` + Params map[string]any `json:"params,omitempty"` +} + +// Script builds a Painless script filter. Used for comparisons whose right-hand +// side is not a constant (field-to-field, or eval-time computed values) and so +// cannot be expressed as a native range/term query. +func Script(source string, params map[string]any) *ScriptFilter { + return &ScriptFilter{Script: ScriptQry{Script: ScriptBody{ + Source: source, + Lang: "painless", + Params: params, + }}} +} diff --git a/generators/esgen/fieldcompare_test.go b/generators/esgen/fieldcompare_test.go new file mode 100644 index 0000000..fce7b87 --- /dev/null +++ b/generators/esgen/fieldcompare_test.go @@ -0,0 +1,96 @@ +//go:build !slow + +package esgen + +import ( + "testing" + "time" + + "github.com/lytics/qlbridge/generators/gentypes" + "github.com/lytics/qlbridge/rel" + "github.com/lytics/qlbridge/value" + "github.com/stretchr/testify/require" +) + +func TestFieldToFieldCompare(t *testing.T) { + t.Parallel() + + ts := time.Date(2026, 6, 29, 0, 0, 0, 0, time.UTC) + s := schema{cols: map[string]value.ValueType{ + "consent": value.TimeType, + "signup": value.TimeType, + "clicks": value.IntType, + "score": value.NumberType, + "name": value.StringType, + }} + g := NewGenerator(ts, nil, s) + + tests := []struct { + name string + filter string + wantSrc string + wantErr bool + }{ + { + name: "time lt time", + filter: `FILTER consent < signup`, + wantSrc: "doc['consent'].size() != 0 && doc['signup'].size() != 0 && doc['consent'].value.toLocalDate().toEpochDay() < doc['signup'].value.toLocalDate().toEpochDay()", + }, + { + name: "time gt time", + filter: `FILTER consent > signup`, + wantSrc: "doc['consent'].size() != 0 && doc['signup'].size() != 0 && doc['consent'].value.toLocalDate().toEpochDay() > doc['signup'].value.toLocalDate().toEpochDay()", + }, + { + name: "int ge number", + filter: `FILTER clicks >= score`, + wantSrc: "doc['clicks'].size() != 0 && doc['score'].size() != 0 && doc['clicks'].value >= doc['score'].value", + }, + { + name: "type mismatch time vs number", + filter: `FILTER consent < score`, + wantErr: true, + }, + { + name: "string unsupported", + filter: `FILTER name < name`, + wantErr: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + fs, err := rel.ParseFilterQL(tc.filter) + require.NoError(t, err) + p, err := g.WalkExpr(fs.Filter) + if tc.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + sf, ok := p.Filter.(*ScriptFilter) + require.True(t, ok, "expected *ScriptFilter, got %T", p.Filter) + require.Equal(t, "painless", sf.Script.Script.Lang) + require.Equal(t, tc.wantSrc, sf.Script.Script.Source) + }) + } +} + +func TestFieldToFieldNestedUnsupported(t *testing.T) { + t.Parallel() + + ts := time.Date(2026, 6, 29, 0, 0, 0, 0, time.UTC) + g := NewGenerator(ts, nil, nestedSchema{}) + + fs, err := rel.ParseFilterQL(`FILTER a < b`) + require.NoError(t, err) + _, err = g.WalkExpr(fs.Filter) + require.Error(t, err) +} + +type nestedSchema struct{} + +func (nestedSchema) Column(string) (value.ValueType, bool) { return value.TimeType, true } +func (nestedSchema) ColumnInfo(f string) (*gentypes.FieldType, bool) { + return &gentypes.FieldType{Field: f, Type: value.TimeType, Path: "map_events", Prefix: "t"}, true +} diff --git a/generators/esgen/recurring_test.go b/generators/esgen/recurring_test.go new file mode 100644 index 0000000..eb6d11c --- /dev/null +++ b/generators/esgen/recurring_test.go @@ -0,0 +1,92 @@ +//go:build !slow + +package esgen + +import ( + "testing" + "time" + + "github.com/lytics/qlbridge/rel" + "github.com/lytics/qlbridge/value" + "github.com/stretchr/testify/require" +) + +func TestRecurring(t *testing.T) { + t.Parallel() + + // 2026-06-29 is a Monday (ISO day-of-week 1). + ts := time.Date(2026, 6, 29, 12, 0, 0, 0, time.UTC) + s := schema{cols: map[string]value.ValueType{ + "birthday": value.TimeType, + "signup": value.TimeType, + "name": value.StringType, + }} + g := NewGenerator(ts, nil, s) + + tests := []struct { + name string + filter string + wantSrc string + wantParams map[string]any + wantErr bool + }{ + { + name: "yearly", + filter: `FILTER recurring(birthday, "yearly")`, + wantSrc: "doc['birthday'].size() != 0 && doc['birthday'].value.getMonthValue() == params.month && doc['birthday'].value.getDayOfMonth() == params.day", + wantParams: map[string]any{"month": 6, "day": 29}, + }, + { + name: "monthly", + filter: `FILTER recurring(birthday, "monthly")`, + wantSrc: "doc['birthday'].size() != 0 && doc['birthday'].value.getDayOfMonth() == params.day", + wantParams: map[string]any{"day": 29}, + }, + { + name: "weekly", + filter: `FILTER recurring(birthday, "weekly")`, + wantSrc: "doc['birthday'].size() != 0 && doc['birthday'].value.getDayOfWeek().getValue() == params.dow", + wantParams: map[string]any{"dow": 1}, + }, + { + name: "half-birthday yearly+182", + filter: `FILTER recurring(birthday, "yearly", 182)`, + wantSrc: "doc['birthday'].size() != 0 && doc['birthday'].value.getMonthValue() == params.month && doc['birthday'].value.getDayOfMonth() == params.day", + wantParams: map[string]any{"month": 12, "day": 29}, // 2026-06-29 minus 182 days = 2025-12-29 + }, + { + name: "every 90 days", + filter: `FILTER recurring(signup, 90)`, + wantSrc: "if (doc['signup'].size() != 0) { long d = params.todayDay - (doc['signup'].value.toInstant().getEpochSecond() / 86400) - params.offset; return d >= 0 && d % params.n == 0; } return false;", + wantParams: map[string]any{"todayDay": ts.UTC().Unix() / 86400, "offset": 0, "n": int64(90)}, + }, + { + name: "bad period", + filter: `FILTER recurring(birthday, "fortnightly")`, + wantErr: true, + }, + { + name: "non-date field", + filter: `FILTER recurring(name, "yearly")`, + wantErr: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + fs, err := rel.ParseFilterQL(tc.filter) + require.NoError(t, err) + p, err := g.WalkExpr(fs.Filter) + if tc.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + sf, ok := p.Filter.(*ScriptFilter) + require.True(t, ok, "expected *ScriptFilter, got %T", p.Filter) + require.Equal(t, "painless", sf.Script.Script.Lang) + require.Equal(t, tc.wantSrc, sf.Script.Script.Source) + require.Equal(t, tc.wantParams, sf.Script.Script.Params) + }) + } +}