json_mode option is added, tests are passing. ready for review

revert changes made to JQ transformation

update code as per ticket discussion; update tests

fix config tests

address comments

clecar up SpModeV2 tests; disable interState for SpModeV2

remove config update

cleanup the test

update test case

improve tests

fix test

revert back to json_mode only

Author: oo-sleeper

assets/docs/configuration/transformations/custom-scripts/examples/js-snowplow-config-example.hcl

@@ -7,6 +7,8 @@ transform {
     # For this example, we use an environment variable, to facilitate unit tests. A hardcoded value may also be provided (eg. "/tmp/myscript.js")
     script_path = env.JS_SNOWPLOW_SCRIPT_PATH
 
-    snowplow_mode       = true # Snowplow mode enabled - this transforms the tsv to an object on input
+    # If the data is in JSON format, then it would be converted into an object to use in JS script
+    # If the data is not in JSON format, transformation would fail
+    json_mode = true # If true, then overrides snowplow_mode
   }
 }

assets/docs/configuration/transformations/custom-scripts/js-configuration-full-example.hcl

@@ -21,6 +21,12 @@ transform {
     # This will transform the data so that the `Data` field contains an object representation of the event - with keys as returned by the Snowplow Analytics SDK.
     snowplow_mode       = true
 
+    # optional, if used overrides snowplow_mode value
+    # If the data is in JSON format, then it would be converted into an object to use in JS script
+    # If the data is not in JSON format, transformation would fail
+    # Otherwise, the transformation will fail
+    json_mode = true
+
     # optional, removes null or empty keys from JSON object output of transformation. Only applicable when the script returns an object. Does not remove null or empty elements of arrays.
     remove_nulls = true
 

docs/configuration_transformations_docs_test.go

@@ -106,7 +106,6 @@ func TestScriptTransformationCustomScripts(t *testing.T) {
 			testJSScriptCompiles(t, file)
 		case ".hcl":
 			isFull := strings.Contains(file, "full-example")
-
 			testTransformationConfig(t, file, isFull)
 		case "":
 			// If there's no extension, fail the test.
@@ -116,7 +115,6 @@ func TestScriptTransformationCustomScripts(t *testing.T) {
 			// Otherwise it's likely a typo or error.
 			assert.Fail("unexpected file extension found: %v", file)
 		}
-
 	}
 }
 

pkg/transform/engine/engine_javascript.go

@@ -33,6 +33,7 @@ type JSEngineConfig struct {
 	Script         string `hcl:"script,optional"`
 	RunTimeout     int    `hcl:"timeout_sec,optional"`
 	SpMode         bool   `hcl:"snowplow_mode,optional"`
+	JsonMode       bool   `hcl:"json_mode,optional"`
 	RemoveNulls    bool   `hcl:"remove_nulls,optional"`
 	HashSaltSecret string `hcl:"hash_salt_secret,optional"`
 }
@@ -42,6 +43,7 @@ type JSEngine struct {
 	Code           *goja.Program
 	RunTimeout     time.Duration
 	SpMode         bool
+	JsonMode       bool
 	RemoveNulls    bool
 	HashSaltSecret string
 }
@@ -134,6 +136,7 @@ func NewJSEngine(c *JSEngineConfig, script string) (*JSEngine, error) {
 		Code:           compiledCode,
 		RunTimeout:     time.Duration(c.RunTimeout) * time.Second,
 		SpMode:         c.SpMode,
+		JsonMode:       c.JsonMode,
 		RemoveNulls:    c.RemoveNulls,
 		HashSaltSecret: c.HashSaltSecret,
 	}
@@ -331,10 +334,31 @@ func mkJSEngineInput(e *JSEngine, message *models.Message, interState any) (*eng
 		HTTPHeaders: message.HTTPHeaders,
 	}
 
+	if e.JsonMode {
+		var cInput map[string]any
+		jErr := json.Unmarshal(message.Data, &cInput)
+		if jErr != nil {
+			return nil, fmt.Errorf("failed to transform input into JSON: %w", jErr)
+		}
+
+		candidate.Data = cInput
+		return candidate, nil
+	}
+
 	if !e.SpMode {
 		return candidate, nil
 	}
 
+	spMap, err := parseAsTSV(interState, message)
+	if err != nil {
+		return nil, err
+	}
+
+	candidate.Data = spMap
+	return candidate, nil
+}
+
+func parseAsTSV(interState any, message *models.Message) (map[string]any, error) {
 	parsedEvent, err := transform.IntermediateAsSpEnrichedParsed(interState, message)
 	if err != nil {
 		// if spMode, error for non Snowplow enriched event data
@@ -346,8 +370,7 @@ func mkJSEngineInput(e *JSEngine, message *models.Message, interState any) (*eng
 		return nil, err
 	}
 
-	candidate.Data = spMap
-	return candidate, nil
+	return spMap, nil
 }
 
 // validateJSEngineOut validates the value returned by the js engine.