feat: ordered evaluation of response_rules (#466)

* breaking change - response_rules are now evaluated in the order they defined in the config and they distinguish between invalid/setup based on the rule's type attribute, rather than setting invalid/setup block within response_rules configuration block

* update hcl configs in e2e tests

* add rules type validation

Author: oo-sleeper

assets/docs/configuration/targets/http-full-example.hcl

@@ -78,24 +78,23 @@ target {
     rejection_threshold_in_millis = 100
 
     # Optional HTTP response rules which are used to match HTTP response code/body and categorize it as either invalid data or target setup error.
-    # For example, we can have 2 invalid + 1 setup error rules:
+    # Rules are evaluated in order as declared. First matching rule determines the error type.
     response_rules {
-      # This one is a match when... 
-      invalid {
-          # ...HTTP statuses match...
+      # Invalid rule for purchase field validation error
+      rule {
+          type = "invalid"
           http_codes = [400]
-          # AND this string exists in a response body
           body =  "Invalid value for 'purchase' field"
         }
-      # If no match yet, we can check the next one... 
-      invalid {
-          # again 400 status...
+      # Invalid rule for attributes field validation error
+      rule {
+          type = "invalid"
           http_codes = [400]
-          # BUT we expect different error message in the response body
           body =  "Invalid value for 'attributes' field"
         }
-      # Same for 'setup' rules..  
-      setup {
+      # Setup rule for authentication errors
+      rule {
+          type = "setup"
           http_codes =  [401, 403]
         }
     }

config/component_test.go

@@ -92,8 +92,7 @@ func TestCreateTargetComponentHCL(t *testing.T) {
 				IncludeTimingHeaders:       false,
 				RejectionThresholdInMillis: 150,
 				ResponseRules: &target.ResponseRules{
-					Invalid:    []target.Rule{},
-					SetupError: []target.Rule{},
+					Rules: []target.Rule{},
 				},
 			},
 		},
@@ -125,18 +124,19 @@ func TestCreateTargetComponentHCL(t *testing.T) {
 				IncludeTimingHeaders:       true,
 				RejectionThresholdInMillis: 100,
 				ResponseRules: &target.ResponseRules{
-					Invalid: []target.Rule{
+					Rules: []target.Rule{
 						{
+							Type:              target.ResponseRuleTypeInvalid,
 							MatchingHTTPCodes: []int{400},
 							MatchingBodyPart:  "Invalid value for 'purchase' field",
 						},
 						{
+							Type:              target.ResponseRuleTypeInvalid,
 							MatchingHTTPCodes: []int{400},
 							MatchingBodyPart:  "Invalid value for 'attributes' field",
 						},
-					},
-					SetupError: []target.Rule{
 						{
+							Type:              target.ResponseRuleTypeSetup,
 							MatchingHTTPCodes: []int{401, 403},
 						},
 					},

pkg/target/http.go

@@ -19,6 +19,7 @@ import (
 	"io"
 	"net/http"
 	"os"
+	"slices"
 	"strconv"
 	"strings"
 	"text/template"
@@ -71,14 +72,32 @@ type HTTPTargetConfig struct {
 
 // ResponseRules is part of HTTP target configuration. It provides rules how HTTP respones should be handled. Response can be categerized as 'invalid' (bad data), as setup error or (if none of the rules matches) as a transient error.
 type ResponseRules struct {
-	Invalid    []Rule `hcl:"invalid,block"`
-	SetupError []Rule `hcl:"setup,block"`
+	Rules []Rule `hcl:"rule,block"`
+	// Invalid    []Rule `hcl:"invalid,block"`
+	// SetupError []Rule `hcl:"setup,block"`
+}
+
+type ResponseRuleType string
+
+const (
+	ResponseRuleTypeInvalid ResponseRuleType = "invalid"
+	ResponseRuleTypeSetup   ResponseRuleType = "setup"
+)
+
+func isValidResponseRuleType(ruleType ResponseRuleType) bool {
+	switch ruleType {
+	case ResponseRuleTypeInvalid, ResponseRuleTypeSetup:
+		return true
+	default:
+		return false
+	}
 }
 
 // Rule configuration defines what kind of values are expected to exist in HTTP response, like status code or message in the body.
 type Rule struct {
-	MatchingHTTPCodes []int  `hcl:"http_codes,optional"`
-	MatchingBodyPart  string `hcl:"body,optional"`
+	Type              ResponseRuleType `hcl:"type,optional"`
+	MatchingHTTPCodes []int            `hcl:"http_codes,optional"`
+	MatchingBodyPart  string           `hcl:"body,optional"`
 }
 
 // Helper struct storing response HTTP status code and parsed response body
@@ -244,6 +263,15 @@ func HTTPTargetConfigFunction(c *HTTPTargetConfig) (*HTTPTarget, error) {
 		return nil, errors.New("target error: Byte limit must be larger than template size")
 	}
 
+	// validating response rules from config
+	if c.ResponseRules != nil {
+		for _, rule := range c.ResponseRules.Rules {
+			if !isValidResponseRuleType(rule.Type) {
+				return nil, fmt.Errorf("target error: Invalid response rule type '%s'. Valid types are: 'invalid', 'setup'", rule.Type)
+			}
+		}
+	}
+
 	return &HTTPTarget{
 		client:               client,
 		httpURL:              c.URL,
@@ -289,8 +317,7 @@ func defaultConfiguration() *HTTPTargetConfig {
 
 		ContentType: "application/json",
 		ResponseRules: &ResponseRules{
-			Invalid:    []Rule{},
-			SetupError: []Rule{},
+			Rules: []Rule{},
 		},
 		IncludeTimingHeaders:       false,
 		RejectionThresholdInMillis: 150,
@@ -420,42 +447,53 @@ func (ht *HTTPTarget) Write(messages []*models.Message) (*models.TargetWriteResu
 				})
 			}
 
-			if matchedRule := findMatchingRule(response, ht.responseRules.Invalid); matchedRule != nil {
-				for _, msg := range goodMsgs {
-					msg.SetError(&models.ApiError{
-						StatusCode:   resp.Status,
-						ResponseBody: response.Body,
-						SafeMessage:  "Invalid error",
-					})
+			// Find first matching rule in order
+			var matchedRule *Rule
+			for _, rule := range ht.responseRules.Rules {
+				if ruleMatches(response, rule) {
+					matchedRule = &rule
+					break
 				}
-
-				invalid = append(invalid, goodMsgs...)
-				continue
 			}
 
-			var errorDetails error
-			if rule := findMatchingRule(response, ht.responseRules.SetupError); rule != nil {
-				hitSetupError = true
-
-				if rule.MatchingBodyPart != "" {
-					errorDetails = fmt.Errorf("got setup error, response status: '%s' with error details: '%s'", resp.Status, rule.MatchingBodyPart)
-				} else {
-					errorDetails = fmt.Errorf("got setup error, response status: '%s'", resp.Status)
-				}
+			if matchedRule != nil {
+				switch matchedRule.Type {
+				case ResponseRuleTypeInvalid:
+					for _, msg := range goodMsgs {
+						msg.SetError(&models.ApiError{
+							StatusCode:   resp.Status,
+							ResponseBody: response.Body,
+							SafeMessage:  "Invalid error",
+						})
+					}
+					invalid = append(invalid, goodMsgs...)
+					continue
+
+				case ResponseRuleTypeSetup:
+					hitSetupError = true
+					var errorDetails error
+					if matchedRule.MatchingBodyPart != "" {
+						errorDetails = fmt.Errorf("got setup error, response status: '%s' with error details: '%s'", resp.Status, matchedRule.MatchingBodyPart)
+					} else {
+						errorDetails = fmt.Errorf("got setup error, response status: '%s'", resp.Status)
+					}
 
-				for _, msg := range goodMsgs {
-					msg.SetError(&models.ApiError{
-						StatusCode:   resp.Status,
-						ResponseBody: response.Body,
-						SafeMessage:  "Setup error",
-					})
+					for _, msg := range goodMsgs {
+						msg.SetError(&models.ApiError{
+							StatusCode:   resp.Status,
+							ResponseBody: response.Body,
+							SafeMessage:  "Setup error",
+						})
+					}
+					errResult = multierror.Append(errResult, errorDetails)
+					failed = append(failed, goodMsgs...)
 				}
-
 			} else {
-				errorDetails = fmt.Errorf("got transient error, response status: '%s'", resp.Status)
+				// No rule matched - transient error
+				errorDetails := fmt.Errorf("got transient error, response status: '%s'", resp.Status)
+				errResult = multierror.Append(errResult, errorDetails)
+				failed = append(failed, goodMsgs...)
 			}
-			errResult = multierror.Append(errResult, errorDetails)
-			failed = append(failed, goodMsgs...)
 		}
 	}
 
@@ -467,15 +505,6 @@ func (ht *HTTPTarget) Write(messages []*models.Message) (*models.TargetWriteResu
 	return models.NewTargetWriteResult(sent, failed, oversized, invalid), errResult
 }
 
-func findMatchingRule(res response, rules []Rule) *Rule {
-	for _, rule := range rules {
-		if ruleMatches(res, rule) {
-			return &rule
-		}
-	}
-	return nil
-}
-
 func ruleMatches(res response, rule Rule) bool {
 	codeMatch := httpStatusMatches(res.Status, rule.MatchingHTTPCodes)
 	if rule.MatchingBodyPart != "" {
@@ -485,12 +514,7 @@ func ruleMatches(res response, rule Rule) bool {
 }
 
 func httpStatusMatches(actual int, expectedCodes []int) bool {
-	for _, expected := range expectedCodes {
-		if expected == actual {
-			return true
-		}
-	}
-	return false
+	return slices.Contains(expectedCodes, actual)
 }
 
 func responseBodyMatches(actual string, bodyPattern string) bool {