From cbf1f0c86f91e628ebccfae5c0d5c3ce713170ff Mon Sep 17 00:00:00 2001 From: Ben White Date: Tue, 16 Jul 2024 15:55:05 +0200 Subject: [PATCH] feat(cdp): Added kinesis template (#23635) Co-authored-by: Marius Andra --- .../hogfunctions/HogFunctionInputs.tsx | 2 +- hogvm/__tests__/__snapshots__/arrays.hoge | 12 +- hogvm/__tests__/__snapshots__/arrays.stdout | 18 +++ hogvm/__tests__/__snapshots__/crypto.hoge | 4 + hogvm/__tests__/__snapshots__/crypto.stdout | 5 + hogvm/__tests__/__snapshots__/dateFormat.hoge | 60 ++++---- .../__tests__/__snapshots__/dateFormat.stdout | 1 + hogvm/__tests__/__snapshots__/loops.hoge | 37 +++-- hogvm/__tests__/__snapshots__/strings.hoge | 7 + hogvm/__tests__/__snapshots__/strings.stdout | 10 ++ hogvm/__tests__/arrays.hog | 29 ++++ hogvm/__tests__/crypto.hog | 8 ++ hogvm/__tests__/dateFormat.hog | 1 + hogvm/__tests__/strings.hog | 10 ++ hogvm/python/execute.py | 26 ++-- hogvm/python/stl/__init__.py | 116 +++++++++++++++ hogvm/python/stl/crypto.py | 21 +++ hogvm/typescript/package.json | 2 +- hogvm/typescript/src/stl/crypto.ts | 22 +++ hogvm/typescript/src/stl/date.ts | 2 +- hogvm/typescript/src/stl/stl.ts | 108 +++++++++++++- plugin-server/package.json | 2 +- plugin-server/pnpm-lock.yaml | 8 +- posthog/cdp/templates/__init__.py | 4 +- .../aws_kinesis/template_aws_kinesis.py | 134 ++++++++++++++++++ .../aws_kinesis/test_template_aws_kinesis.py | 36 +++++ posthog/hogql/bytecode.py | 44 +++--- posthog/hogql/test/test_metadata.py | 6 +- 28 files changed, 635 insertions(+), 100 deletions(-) create mode 100644 hogvm/__tests__/__snapshots__/crypto.hoge create mode 100644 hogvm/__tests__/__snapshots__/crypto.stdout create mode 100644 hogvm/__tests__/__snapshots__/strings.hoge create mode 100644 hogvm/__tests__/__snapshots__/strings.stdout create mode 100644 hogvm/__tests__/crypto.hog create mode 100644 hogvm/__tests__/strings.hog create mode 100644 hogvm/python/stl/crypto.py create mode 100644 hogvm/typescript/src/stl/crypto.ts create mode 100644 posthog/cdp/templates/aws_kinesis/template_aws_kinesis.py create mode 100644 posthog/cdp/templates/aws_kinesis/test_template_aws_kinesis.py diff --git a/frontend/src/scenes/pipeline/hogfunctions/HogFunctionInputs.tsx b/frontend/src/scenes/pipeline/hogfunctions/HogFunctionInputs.tsx index 663d1b7f944..1ece727c35e 100644 --- a/frontend/src/scenes/pipeline/hogfunctions/HogFunctionInputs.tsx +++ b/frontend/src/scenes/pipeline/hogfunctions/HogFunctionInputs.tsx @@ -328,7 +328,7 @@ function HogFunctionInputSchemaControls({ value, onChange, onDone }: HogFunction />
} size="small" onClick={() => onChange(null)} /> - onDone()}> + onDone()}> Done
diff --git a/hogvm/__tests__/__snapshots__/arrays.hoge b/hogvm/__tests__/__snapshots__/arrays.hoge index f7be67dafd2..7f082d8fdf5 100644 --- a/hogvm/__tests__/__snapshots__/arrays.hoge +++ b/hogvm/__tests__/__snapshots__/arrays.hoge @@ -8,4 +8,14 @@ 33, 1, 48, 2, "print", 1, 35, 33, 1, 33, 2, 33, 3, 33, 4, 43, 2, 43, 2, 33, 5, 43, 3, 33, 6, 48, 33, 3, 48, 33, 1, 48, 2, "print", 1, 35, 33, 1, 33, 2, 33, 3, 33, 4, 43, 2, 43, 2, 33, 5, 43, 3, 33, 6, 48, 33, 3, 48, 33, 1, 48, 2, "print", 1, 35, 33, 1, 33, 1, 33, 2, 33, 3, 33, 4, 43, 2, 43, 2, 33, 5, 43, 3, 33, 1, 45, 33, 1, 45, 33, 1, 45, 6, 2, "print", 1, -35, 33, 1, 33, 2, 33, 3, 33, 4, 43, 2, 43, 2, 33, 5, 43, 3, 33, 1, 45, 33, 1, 45, 33, 1, 45, 2, "print", 1, 35, 35] +35, 33, 1, 33, 2, 33, 3, 33, 4, 43, 2, 43, 2, 33, 5, 43, 3, 33, 1, 45, 33, 1, 45, 33, 1, 45, 2, "print", 1, 35, 32, +"------", 2, "print", 1, 35, 33, 4, 33, 1, 33, 2, 33, 3, 43, 3, 2, "arrayPushBack", 2, 2, "print", 1, 35, 33, 0, 33, 1, +33, 2, 33, 3, 43, 3, 2, "arrayPushFront", 2, 2, "print", 1, 35, 33, 1, 33, 2, 33, 3, 43, 3, 2, "arrayPopBack", 1, 2, +"print", 1, 35, 33, 1, 33, 2, 33, 3, 43, 3, 2, "arrayPopFront", 1, 2, "print", 1, 35, 33, 3, 33, 2, 33, 1, 43, 3, 2, +"arraySort", 1, 2, "print", 1, 35, 33, 1, 33, 2, 33, 3, 43, 3, 2, "arrayReverse", 1, 2, "print", 1, 35, 33, 3, 33, 2, +33, 1, 43, 3, 2, "arrayReverseSort", 1, 2, "print", 1, 35, 32, ",", 33, 1, 33, 2, 33, 3, 43, 3, 2, "arrayStringConcat", +2, 2, "print", 1, 35, 32, "-----", 2, "print", 1, 35, 33, 1, 33, 2, 33, 3, 33, 4, 43, 4, 36, 1, 2, "print", 1, 35, 33, +5, 36, 1, 2, "arrayPushBack", 2, 35, 36, 1, 2, "print", 1, 35, 33, 0, 36, 1, 2, "arrayPushFront", 2, 35, 36, 1, 2, +"print", 1, 35, 36, 1, 2, "arrayPopBack", 1, 35, 36, 1, 2, "print", 1, 35, 36, 1, 2, "arrayPopFront", 1, 35, 36, 1, 2, +"print", 1, 35, 36, 1, 2, "arraySort", 1, 35, 36, 1, 2, "print", 1, 35, 36, 1, 2, "arrayReverse", 1, 35, 36, 1, 2, +"print", 1, 35, 36, 1, 2, "arrayReverseSort", 1, 35, 36, 1, 2, "print", 1, 35, 35, 35] diff --git a/hogvm/__tests__/__snapshots__/arrays.stdout b/hogvm/__tests__/__snapshots__/arrays.stdout index cc9e9b46db3..bac9eea0c6d 100644 --- a/hogvm/__tests__/__snapshots__/arrays.stdout +++ b/hogvm/__tests__/__snapshots__/arrays.stdout @@ -16,3 +16,21 @@ null null 5 4 +------ +[1, 2, 3, 4] +[0, 1, 2, 3] +[1, 2] +[2, 3] +[1, 2, 3] +[3, 2, 1] +[3, 2, 1] +1,2,3 +----- +[1, 2, 3, 4] +[1, 2, 3, 4] +[1, 2, 3, 4] +[1, 2, 3, 4] +[1, 2, 3, 4] +[1, 2, 3, 4] +[1, 2, 3, 4] +[1, 2, 3, 4] diff --git a/hogvm/__tests__/__snapshots__/crypto.hoge b/hogvm/__tests__/__snapshots__/crypto.hoge new file mode 100644 index 00000000000..5100cc86000 --- /dev/null +++ b/hogvm/__tests__/__snapshots__/crypto.hoge @@ -0,0 +1,4 @@ +["_h", 32, "this is a secure string", 36, 0, 32, "string:", 2, "print", 2, 35, 36, 0, 2, "md5Hex", 1, 32, +"md5Hex(string):", 2, "print", 2, 35, 36, 0, 2, "sha256Hex", 1, 32, "sha256Hex(string):", 2, "print", 2, 35, 32, "1", +32, "string", 32, "more", 32, "keys", 43, 4, 36, 1, 32, "data:", 2, "print", 2, 35, 36, 1, 2, "sha256HmacChainHex", 1, +32, "sha256HmacChainHex(data):", 2, "print", 2, 35, 35, 35] diff --git a/hogvm/__tests__/__snapshots__/crypto.stdout b/hogvm/__tests__/__snapshots__/crypto.stdout new file mode 100644 index 00000000000..dc98f5fe5e8 --- /dev/null +++ b/hogvm/__tests__/__snapshots__/crypto.stdout @@ -0,0 +1,5 @@ +string: this is a secure string +md5Hex(string): e7b466647ea215dbe59b00c756560911 +sha256Hex(string): 5216c0931310b31737ef30353830c234901283544e934f54eb75f622cfb86c9d +data: ['1', 'string', 'more', 'keys'] +sha256HmacChainHex(data): 826820d7eeca97f26ca18096be85fed346f6fd9cc18d64e72c935bea3450dbd9 diff --git a/hogvm/__tests__/__snapshots__/dateFormat.hoge b/hogvm/__tests__/__snapshots__/dateFormat.hoge index 283b1026684..eea4f091c7f 100644 --- a/hogvm/__tests__/__snapshots__/dateFormat.hoge +++ b/hogvm/__tests__/__snapshots__/dateFormat.hoge @@ -1,32 +1,32 @@ ["_h", 34, 1234377543.123456, 2, "fromUnixTimestamp", 1, 32, "%Y-%m-%d %H:%i:%S", 36, 0, 2, "formatDateTime", 2, 2, "print", 1, 35, 32, "Europe/Brussels", 32, "%Y-%m-%d %H:%i:%S", 36, 0, 2, "formatDateTime", 3, 2, "print", 1, 35, 32, -"America/New_York", 32, "%Y-%m-%d %H:%i:%S", 36, 0, 2, "formatDateTime", 3, 2, "print", 1, 35, 32, "-----", 2, "print", -1, 35, 32, "%a", 36, 0, 2, "formatDateTime", 2, 32, "%a: ", 2, "concat", 2, 2, "print", 1, 35, 32, "%b", 36, 0, 2, -"formatDateTime", 2, 32, "%b: ", 2, "concat", 2, 2, "print", 1, 35, 32, "%c", 36, 0, 2, "formatDateTime", 2, 32, "%c: ", -2, "concat", 2, 2, "print", 1, 35, 32, "%C", 36, 0, 2, "formatDateTime", 2, 32, "%C: ", 2, "concat", 2, 2, "print", 1, -35, 32, "%d", 36, 0, 2, "formatDateTime", 2, 32, "%d: ", 2, "concat", 2, 2, "print", 1, 35, 32, "%D", 36, 0, 2, -"formatDateTime", 2, 32, "%D: ", 2, "concat", 2, 2, "print", 1, 35, 32, "%e", 36, 0, 2, "formatDateTime", 2, 32, "%e: ", -2, "concat", 2, 2, "print", 1, 35, 32, "%F", 36, 0, 2, "formatDateTime", 2, 32, "%F: ", 2, "concat", 2, 2, "print", 1, -35, 32, "%g", 36, 0, 2, "formatDateTime", 2, 32, "%g: ", 2, "concat", 2, 2, "print", 1, 35, 32, "%G", 36, 0, 2, -"formatDateTime", 2, 32, "%G: ", 2, "concat", 2, 2, "print", 1, 35, 32, "%h", 36, 0, 2, "formatDateTime", 2, 32, "%h: ", -2, "concat", 2, 2, "print", 1, 35, 32, "%H", 36, 0, 2, "formatDateTime", 2, 32, "%H: ", 2, "concat", 2, 2, "print", 1, -35, 32, "%i", 36, 0, 2, "formatDateTime", 2, 32, "%i: ", 2, "concat", 2, 2, "print", 1, 35, 32, "%I", 36, 0, 2, -"formatDateTime", 2, 32, "%I: ", 2, "concat", 2, 2, "print", 1, 35, 32, "%j", 36, 0, 2, "formatDateTime", 2, 32, "%j: ", -2, "concat", 2, 2, "print", 1, 35, 32, "%k", 36, 0, 2, "formatDateTime", 2, 32, "%k: ", 2, "concat", 2, 2, "print", 1, -35, 32, "%l", 36, 0, 2, "formatDateTime", 2, 32, "%l: ", 2, "concat", 2, 2, "print", 1, 35, 32, "%m", 36, 0, 2, -"formatDateTime", 2, 32, "%m: ", 2, "concat", 2, 2, "print", 1, 35, 32, "%M", 36, 0, 2, "formatDateTime", 2, 32, "%M: ", -2, "concat", 2, 2, "print", 1, 35, 32, "%n", 36, 0, 2, "formatDateTime", 2, 32, "%n: ", 2, "concat", 2, 2, "print", 1, -35, 32, "%p", 36, 0, 2, "formatDateTime", 2, 32, "%p: ", 2, "concat", 2, 2, "print", 1, 35, 32, "%r", 36, 0, 2, -"formatDateTime", 2, 32, "%r: ", 2, "concat", 2, 2, "print", 1, 35, 32, "%R", 36, 0, 2, "formatDateTime", 2, 32, "%R: ", -2, "concat", 2, 2, "print", 1, 35, 32, "%s", 36, 0, 2, "formatDateTime", 2, 32, "%s: ", 2, "concat", 2, 2, "print", 1, -35, 32, "%S", 36, 0, 2, "formatDateTime", 2, 32, "%S: ", 2, "concat", 2, 2, "print", 1, 35, 32, "%t", 36, 0, 2, -"formatDateTime", 2, 32, "%t: ", 2, "concat", 2, 2, "print", 1, 35, 32, "%T", 36, 0, 2, "formatDateTime", 2, 32, "%T: ", -2, "concat", 2, 2, "print", 1, 35, 32, "%u", 36, 0, 2, "formatDateTime", 2, 32, "%u: ", 2, "concat", 2, 2, "print", 1, -35, 32, "%V", 36, 0, 2, "formatDateTime", 2, 32, "%V: ", 2, "concat", 2, 2, "print", 1, 35, 32, "%w", 36, 0, 2, -"formatDateTime", 2, 32, "%w: ", 2, "concat", 2, 2, "print", 1, 35, 32, "%W", 36, 0, 2, "formatDateTime", 2, 32, "%W: ", -2, "concat", 2, 2, "print", 1, 35, 32, "%y", 36, 0, 2, "formatDateTime", 2, 32, "%y: ", 2, "concat", 2, 2, "print", 1, -35, 32, "%Y", 36, 0, 2, "formatDateTime", 2, 32, "%Y: ", 2, "concat", 2, 2, "print", 1, 35, 32, "%z", 36, 0, 2, -"formatDateTime", 2, 32, "%z: ", 2, "concat", 2, 2, "print", 1, 35, 32, "%%", 36, 0, 2, "formatDateTime", 2, 32, "%%: ", -2, "concat", 2, 2, "print", 1, 35, 32, "-----", 2, "print", 1, 35, 32, "one banana", 36, 0, 2, "formatDateTime", 2, 2, -"print", 1, 35, 32, "%Y no way %m is this %d a %H real %i time %S", 36, 0, 2, "formatDateTime", 2, 2, "print", 1, 35, -35] +"America/New_York", 32, "%Y-%m-%d %H:%i:%S", 36, 0, 2, "formatDateTime", 3, 2, "print", 1, 35, 32, "%Y%m%dT%H%i%sZ", 36, +0, 2, "formatDateTime", 2, 2, "print", 1, 35, 32, "-----", 2, "print", 1, 35, 32, "%a", 36, 0, 2, "formatDateTime", 2, +32, "%a: ", 2, "concat", 2, 2, "print", 1, 35, 32, "%b", 36, 0, 2, "formatDateTime", 2, 32, "%b: ", 2, "concat", 2, 2, +"print", 1, 35, 32, "%c", 36, 0, 2, "formatDateTime", 2, 32, "%c: ", 2, "concat", 2, 2, "print", 1, 35, 32, "%C", 36, 0, +2, "formatDateTime", 2, 32, "%C: ", 2, "concat", 2, 2, "print", 1, 35, 32, "%d", 36, 0, 2, "formatDateTime", 2, 32, +"%d: ", 2, "concat", 2, 2, "print", 1, 35, 32, "%D", 36, 0, 2, "formatDateTime", 2, 32, "%D: ", 2, "concat", 2, 2, +"print", 1, 35, 32, "%e", 36, 0, 2, "formatDateTime", 2, 32, "%e: ", 2, "concat", 2, 2, "print", 1, 35, 32, "%F", 36, 0, +2, "formatDateTime", 2, 32, "%F: ", 2, "concat", 2, 2, "print", 1, 35, 32, "%g", 36, 0, 2, "formatDateTime", 2, 32, +"%g: ", 2, "concat", 2, 2, "print", 1, 35, 32, "%G", 36, 0, 2, "formatDateTime", 2, 32, "%G: ", 2, "concat", 2, 2, +"print", 1, 35, 32, "%h", 36, 0, 2, "formatDateTime", 2, 32, "%h: ", 2, "concat", 2, 2, "print", 1, 35, 32, "%H", 36, 0, +2, "formatDateTime", 2, 32, "%H: ", 2, "concat", 2, 2, "print", 1, 35, 32, "%i", 36, 0, 2, "formatDateTime", 2, 32, +"%i: ", 2, "concat", 2, 2, "print", 1, 35, 32, "%I", 36, 0, 2, "formatDateTime", 2, 32, "%I: ", 2, "concat", 2, 2, +"print", 1, 35, 32, "%j", 36, 0, 2, "formatDateTime", 2, 32, "%j: ", 2, "concat", 2, 2, "print", 1, 35, 32, "%k", 36, 0, +2, "formatDateTime", 2, 32, "%k: ", 2, "concat", 2, 2, "print", 1, 35, 32, "%l", 36, 0, 2, "formatDateTime", 2, 32, +"%l: ", 2, "concat", 2, 2, "print", 1, 35, 32, "%m", 36, 0, 2, "formatDateTime", 2, 32, "%m: ", 2, "concat", 2, 2, +"print", 1, 35, 32, "%M", 36, 0, 2, "formatDateTime", 2, 32, "%M: ", 2, "concat", 2, 2, "print", 1, 35, 32, "%n", 36, 0, +2, "formatDateTime", 2, 32, "%n: ", 2, "concat", 2, 2, "print", 1, 35, 32, "%p", 36, 0, 2, "formatDateTime", 2, 32, +"%p: ", 2, "concat", 2, 2, "print", 1, 35, 32, "%r", 36, 0, 2, "formatDateTime", 2, 32, "%r: ", 2, "concat", 2, 2, +"print", 1, 35, 32, "%R", 36, 0, 2, "formatDateTime", 2, 32, "%R: ", 2, "concat", 2, 2, "print", 1, 35, 32, "%s", 36, 0, +2, "formatDateTime", 2, 32, "%s: ", 2, "concat", 2, 2, "print", 1, 35, 32, "%S", 36, 0, 2, "formatDateTime", 2, 32, +"%S: ", 2, "concat", 2, 2, "print", 1, 35, 32, "%t", 36, 0, 2, "formatDateTime", 2, 32, "%t: ", 2, "concat", 2, 2, +"print", 1, 35, 32, "%T", 36, 0, 2, "formatDateTime", 2, 32, "%T: ", 2, "concat", 2, 2, "print", 1, 35, 32, "%u", 36, 0, +2, "formatDateTime", 2, 32, "%u: ", 2, "concat", 2, 2, "print", 1, 35, 32, "%V", 36, 0, 2, "formatDateTime", 2, 32, +"%V: ", 2, "concat", 2, 2, "print", 1, 35, 32, "%w", 36, 0, 2, "formatDateTime", 2, 32, "%w: ", 2, "concat", 2, 2, +"print", 1, 35, 32, "%W", 36, 0, 2, "formatDateTime", 2, 32, "%W: ", 2, "concat", 2, 2, "print", 1, 35, 32, "%y", 36, 0, +2, "formatDateTime", 2, 32, "%y: ", 2, "concat", 2, 2, "print", 1, 35, 32, "%Y", 36, 0, 2, "formatDateTime", 2, 32, +"%Y: ", 2, "concat", 2, 2, "print", 1, 35, 32, "%z", 36, 0, 2, "formatDateTime", 2, 32, "%z: ", 2, "concat", 2, 2, +"print", 1, 35, 32, "%%", 36, 0, 2, "formatDateTime", 2, 32, "%%: ", 2, "concat", 2, 2, "print", 1, 35, 32, "-----", 2, +"print", 1, 35, 32, "one banana", 36, 0, 2, "formatDateTime", 2, 2, "print", 1, 35, 32, +"%Y no way %m is this %d a %H real %i time %S", 36, 0, 2, "formatDateTime", 2, 2, "print", 1, 35, 35] diff --git a/hogvm/__tests__/__snapshots__/dateFormat.stdout b/hogvm/__tests__/__snapshots__/dateFormat.stdout index d5e9a380670..c1610cb6b58 100644 --- a/hogvm/__tests__/__snapshots__/dateFormat.stdout +++ b/hogvm/__tests__/__snapshots__/dateFormat.stdout @@ -1,6 +1,7 @@ 2009-02-11 18:39:03 2009-02-11 19:39:03 2009-02-11 13:39:03 +20090211T183903Z ----- %a: Wed %b: Feb diff --git a/hogvm/__tests__/__snapshots__/loops.hoge b/hogvm/__tests__/__snapshots__/loops.hoge index 01d7b736667..349f7f4cbb3 100644 --- a/hogvm/__tests__/__snapshots__/loops.hoge +++ b/hogvm/__tests__/__snapshots__/loops.hoge @@ -3,23 +3,20 @@ 36, 0, 15, 40, 15, 36, 0, 2, "print", 1, 35, 33, 1, 36, 0, 6, 37, 0, 39, -22, 35, 32, "i", 1, 1, 2, "print", 1, 35, 32, "-- test emptier for loop --", 2, "print", 1, 35, 33, 0, 33, 3, 36, 0, 15, 40, 15, 32, "woo", 2, "print", 1, 35, 33, 1, 36, 0, 6, 37, 0, 39, -22, 32, "hoo", 2, "print", 1, 35, 35, 32, "-- for in loop with arrays --", 2, "print", 1, 35, 33, -1, 33, 2, 33, 3, 43, 3, 31, 31, 31, 31, 31, 31, 36, 0, 37, 1, 36, 1, 2, "values", 1, 37, 3, 33, 0, 37, 4, 36, 3, 2, -"length", 1, 37, 5, 36, 5, 36, 4, 15, 40, 22, 36, 3, 36, 4, 45, 37, 6, 36, 6, 2, "print", 1, 35, 36, 4, 33, 1, 6, 37, 4, -39, -29, 35, 35, 35, 35, 35, 35, 35, 32, "-- for in loop with arrays and keys --", 2, "print", 1, 35, 33, 1, 33, 2, 33, -3, 43, 3, 31, 31, 31, 31, 31, 31, 31, 36, 0, 37, 1, 36, 1, 2, "keys", 1, 37, 2, 36, 1, 2, "values", 1, 37, 3, 33, 0, 37, -4, 36, 3, 2, "length", 1, 37, 5, 36, 5, 36, 4, 15, 40, 31, 36, 2, 36, 4, 45, 37, 6, 36, 3, 36, 4, 45, 37, 7, 36, 7, 36, -6, 2, "print", 2, 35, 36, 4, 33, 1, 6, 37, 4, 39, -38, 35, 35, 35, 35, 35, 35, 35, 35, 32, -"-- for in loop with tuples --", 2, "print", 1, 35, 33, 1, 33, 2, 33, 3, 44, 3, 31, 31, 31, 31, 31, 31, 36, 0, 37, 1, -36, 1, 2, "values", 1, 37, 3, 33, 0, 37, 4, 36, 3, 2, "length", 1, 37, 5, 36, 5, 36, 4, 15, 40, 22, 36, 3, 36, 4, 45, -37, 6, 36, 6, 2, "print", 1, 35, 36, 4, 33, 1, 6, 37, 4, 39, -29, 35, 35, 35, 35, 35, 35, 35, 32, -"-- for in loop with tuples and keys --", 2, "print", 1, 35, 33, 1, 33, 2, 33, 3, 44, 3, 31, 31, 31, 31, 31, 31, 31, 36, -0, 37, 1, 36, 1, 2, "keys", 1, 37, 2, 36, 1, 2, "values", 1, 37, 3, 33, 0, 37, 4, 36, 3, 2, "length", 1, 37, 5, 36, 5, -36, 4, 15, 40, 31, 36, 2, 36, 4, 45, 37, 6, 36, 3, 36, 4, 45, 37, 7, 36, 7, 36, 6, 2, "print", 2, 35, 36, 4, 33, 1, 6, -37, 4, 39, -38, 35, 35, 35, 35, 35, 35, 35, 35, 32, "-- for in loop with dicts --", 2, "print", 1, 35, 32, "first", 32, -"v1", 32, "second", 32, "v2", 32, "third", 32, "v3", 42, 3, 31, 31, 31, 31, 31, 31, 36, 0, 37, 1, 36, 1, 2, "values", 1, -37, 3, 33, 0, 37, 4, 36, 3, 2, "length", 1, 37, 5, 36, 5, 36, 4, 15, 40, 22, 36, 3, 36, 4, 45, 37, 6, 36, 6, 2, "print", -1, 35, 36, 4, 33, 1, 6, 37, 4, 39, -29, 35, 35, 35, 35, 35, 35, 35, 32, "-- for in loop with dicts and keys --", 2, -"print", 1, 35, 32, "first", 32, "v1", 32, "second", 32, "v2", 32, "third", 32, "v3", 42, 3, 31, 31, 31, 31, 31, 31, 31, -36, 0, 37, 1, 36, 1, 2, "keys", 1, 37, 2, 36, 1, 2, "values", 1, 37, 3, 33, 0, 37, 4, 36, 3, 2, "length", 1, 37, 5, 36, -5, 36, 4, 15, 40, 31, 36, 2, 36, 4, 45, 37, 6, 36, 3, 36, 4, 45, 37, 7, 36, 7, 36, 6, 2, "print", 2, 35, 36, 4, 33, 1, -6, 37, 4, 39, -38, 35, 35, 35, 35, 35, 35, 35, 35] +1, 33, 2, 33, 3, 43, 3, 36, 0, 36, 1, 2, "values", 1, 33, 0, 36, 2, 2, "length", 1, 31, 36, 4, 36, 3, 15, 40, 22, 36, 2, +36, 3, 45, 37, 5, 36, 5, 2, "print", 1, 35, 36, 3, 33, 1, 6, 37, 3, 39, -29, 35, 35, 35, 35, 35, 35, 32, +"-- for in loop with arrays and keys --", 2, "print", 1, 35, 33, 1, 33, 2, 33, 3, 43, 3, 36, 0, 36, 1, 2, "keys", 1, 36, +1, 2, "values", 1, 33, 0, 36, 3, 2, "length", 1, 31, 31, 36, 5, 36, 4, 15, 40, 31, 36, 2, 36, 4, 45, 37, 6, 36, 3, 36, +4, 45, 37, 7, 36, 7, 36, 6, 2, "print", 2, 35, 36, 4, 33, 1, 6, 37, 4, 39, -38, 35, 35, 35, 35, 35, 35, 35, 35, 32, +"-- for in loop with tuples --", 2, "print", 1, 35, 33, 1, 33, 2, 33, 3, 44, 3, 36, 0, 36, 1, 2, "values", 1, 33, 0, 36, +2, 2, "length", 1, 31, 36, 4, 36, 3, 15, 40, 22, 36, 2, 36, 3, 45, 37, 5, 36, 5, 2, "print", 1, 35, 36, 3, 33, 1, 6, 37, +3, 39, -29, 35, 35, 35, 35, 35, 35, 32, "-- for in loop with tuples and keys --", 2, "print", 1, 35, 33, 1, 33, 2, 33, +3, 44, 3, 36, 0, 36, 1, 2, "keys", 1, 36, 1, 2, "values", 1, 33, 0, 36, 3, 2, "length", 1, 31, 31, 36, 5, 36, 4, 15, 40, +31, 36, 2, 36, 4, 45, 37, 6, 36, 3, 36, 4, 45, 37, 7, 36, 7, 36, 6, 2, "print", 2, 35, 36, 4, 33, 1, 6, 37, 4, 39, -38, +35, 35, 35, 35, 35, 35, 35, 35, 32, "-- for in loop with dicts --", 2, "print", 1, 35, 32, "first", 32, "v1", 32, +"second", 32, "v2", 32, "third", 32, "v3", 42, 3, 36, 0, 36, 1, 2, "values", 1, 33, 0, 36, 2, 2, "length", 1, 31, 36, 4, +36, 3, 15, 40, 22, 36, 2, 36, 3, 45, 37, 5, 36, 5, 2, "print", 1, 35, 36, 3, 33, 1, 6, 37, 3, 39, -29, 35, 35, 35, 35, +35, 35, 32, "-- for in loop with dicts and keys --", 2, "print", 1, 35, 32, "first", 32, "v1", 32, "second", 32, "v2", +32, "third", 32, "v3", 42, 3, 36, 0, 36, 1, 2, "keys", 1, 36, 1, 2, "values", 1, 33, 0, 36, 3, 2, "length", 1, 31, 31, +36, 5, 36, 4, 15, 40, 31, 36, 2, 36, 4, 45, 37, 6, 36, 3, 36, 4, 45, 37, 7, 36, 7, 36, 6, 2, "print", 2, 35, 36, 4, 33, +1, 6, 37, 4, 39, -38, 35, 35, 35, 35, 35, 35, 35, 35] diff --git a/hogvm/__tests__/__snapshots__/strings.hoge b/hogvm/__tests__/__snapshots__/strings.hoge new file mode 100644 index 00000000000..0a379fd0821 --- /dev/null +++ b/hogvm/__tests__/__snapshots__/strings.hoge @@ -0,0 +1,7 @@ +["_h", 32, " hello world ", 2, "trim", 1, 2, "print", 1, 35, 32, " hello world ", 2, "trimLeft", 1, 2, "print", 1, +35, 32, " hello world ", 2, "trimRight", 1, 2, "print", 1, 35, 32, "x", 32, "xxxx hello world xx", 2, "trim", 2, 2, +"print", 1, 35, 32, "x", 32, "xxxx hello world xx", 2, "trimLeft", 2, 2, "print", 1, 35, 32, "x", 32, +"xxxx hello world xx", 2, "trimRight", 2, 2, "print", 1, 35, 32, "hello world and more", 32, " ", 2, "splitByString", +2, 2, "print", 1, 35, 33, 1, 32, "hello world and more", 32, " ", 2, "splitByString", 3, 2, "print", 1, 35, 33, 2, 32, +"hello world and more", 32, " ", 2, "splitByString", 3, 2, "print", 1, 35, 33, 10, 32, "hello world and more", 32, " ", +2, "splitByString", 3, 2, "print", 1, 35] diff --git a/hogvm/__tests__/__snapshots__/strings.stdout b/hogvm/__tests__/__snapshots__/strings.stdout new file mode 100644 index 00000000000..cc6b8a731f8 --- /dev/null +++ b/hogvm/__tests__/__snapshots__/strings.stdout @@ -0,0 +1,10 @@ +hello world +hello world + hello world + hello world + hello world xx +xxxx hello world +['hello', 'world', 'and', 'more'] +['hello'] +['hello', 'world'] +['hello', 'world', 'and', 'more'] diff --git a/hogvm/__tests__/arrays.hog b/hogvm/__tests__/arrays.hog index 2e2c67d2287..04cc57153ba 100644 --- a/hogvm/__tests__/arrays.hog +++ b/hogvm/__tests__/arrays.hog @@ -18,3 +18,32 @@ print([1, [2, [3, 4], ], 5]?.6?.3?.1) print([1, [2, [3, 4], ], 5]?.[6]?.[3]?.[1]) print([1, [2, [3, 4]], 5][1][1][1] + 1) print([1, [2, [3, 4, ], ], 5, ].1.1.1) + +print('------') + +print(arrayPushBack([1,2,3], 4)) +print(arrayPushFront([1,2,3], 0)) +print(arrayPopBack([1,2,3])) +print(arrayPopFront([1,2,3])) +print(arraySort([3,2,1])) +print(arrayReverse([1,2,3])) +print(arrayReverseSort([3,2,1])) +print(arrayStringConcat([1,2,3], ',')) + +print('-----') +let arr := [1,2,3,4] // we don't modify arr +print(arr) +arrayPushBack(arr, 5) +print(arr) +arrayPushFront(arr, 0) +print(arr) +arrayPopBack(arr) +print(arr) +arrayPopFront(arr) +print(arr) +arraySort(arr) +print(arr) +arrayReverse(arr) +print(arr) +arrayReverseSort(arr) +print(arr) diff --git a/hogvm/__tests__/crypto.hog b/hogvm/__tests__/crypto.hog new file mode 100644 index 00000000000..10e98eb688b --- /dev/null +++ b/hogvm/__tests__/crypto.hog @@ -0,0 +1,8 @@ +let string := 'this is a secure string' +print('string:', string) +print('md5Hex(string):', md5Hex(string)) +print('sha256Hex(string):', sha256Hex(string)) + +let data := ['1', 'string', 'more', 'keys'] +print('data:', data) +print('sha256HmacChainHex(data):', sha256HmacChainHex(data)) diff --git a/hogvm/__tests__/dateFormat.hog b/hogvm/__tests__/dateFormat.hog index 706063a17e0..27cafbeb708 100644 --- a/hogvm/__tests__/dateFormat.hog +++ b/hogvm/__tests__/dateFormat.hog @@ -2,6 +2,7 @@ let dt := fromUnixTimestamp(1234377543.123456) print(formatDateTime(dt, '%Y-%m-%d %H:%i:%S')) print(formatDateTime(dt, '%Y-%m-%d %H:%i:%S', 'Europe/Brussels')) print(formatDateTime(dt, '%Y-%m-%d %H:%i:%S', 'America/New_York')) +print(formatDateTime(dt, '%Y%m%dT%H%i%sZ')) print('-----') print('%a: ' || formatDateTime(dt, '%a')) diff --git a/hogvm/__tests__/strings.hog b/hogvm/__tests__/strings.hog new file mode 100644 index 00000000000..f77be3808ec --- /dev/null +++ b/hogvm/__tests__/strings.hog @@ -0,0 +1,10 @@ +print(trim(' hello world ')) +print(trimLeft(' hello world ')) +print(trimRight(' hello world ')) +print(trim('xxxx hello world xx', 'x')) +print(trimLeft('xxxx hello world xx', 'x')) +print(trimRight('xxxx hello world xx', 'x')) +print(splitByString(' ', 'hello world and more')) +print(splitByString(' ', 'hello world and more', 1)) +print(splitByString(' ', 'hello world and more', 2)) +print(splitByString(' ', 'hello world and more', 10)) diff --git a/hogvm/python/execute.py b/hogvm/python/execute.py index e5527c3bf57..909223cbb59 100644 --- a/hogvm/python/execute.py +++ b/hogvm/python/execute.py @@ -208,18 +208,24 @@ def execute_bytecode( push_stack({}) case Operation.ARRAY: count = next_token() - elems = stack[-count:] - stack = stack[:-count] - mem_used -= sum(mem_stack[-count:]) - mem_stack = mem_stack[:-count] - push_stack(elems) + if count > 0: + elems = stack[-count:] + stack = stack[:-count] + mem_used -= sum(mem_stack[-count:]) + mem_stack = mem_stack[:-count] + push_stack(elems) + else: + push_stack([]) case Operation.TUPLE: count = next_token() - elems = stack[-count:] - stack = stack[:-count] - mem_used -= sum(mem_stack[-count:]) - mem_stack = mem_stack[:-count] - push_stack(tuple(elems)) + if count > 0: + elems = stack[-count:] + stack = stack[:-count] + mem_used -= sum(mem_stack[-count:]) + mem_stack = mem_stack[:-count] + push_stack(tuple(elems)) + else: + push_stack(()) case Operation.JUMP: count = next_token() ip += count diff --git a/hogvm/python/stl/__init__.py b/hogvm/python/stl/__init__.py index fcbfd213aaf..ef4a2b71441 100644 --- a/hogvm/python/stl/__init__.py +++ b/hogvm/python/stl/__init__.py @@ -21,6 +21,7 @@ from .date import ( is_hog_datetime, is_hog_date, ) +from .crypto import sha256Hex, md5Hex, sha256HmacChainHex if TYPE_CHECKING: from posthog.models import Team @@ -210,12 +211,53 @@ def replaceAll(args: list[Any], team: Optional["Team"], stdout: Optional[list[st return args[0].replace(args[1], args[2]) +def trim(args: list[Any], team: Optional["Team"], stdout: Optional[list[str]], timeout: int) -> str: + if len(args) > 1 and len(args[1]) > 1: + return "" + return args[0].strip(args[1] if len(args) > 1 else None) + + +def trimLeft(args: list[Any], team: Optional["Team"], stdout: Optional[list[str]], timeout: int) -> str: + if len(args) > 1 and len(args[1]) > 1: + return "" + return args[0].lstrip(args[1] if len(args) > 1 else None) + + +def trimRight(args: list[Any], team: Optional["Team"], stdout: Optional[list[str]], timeout: int) -> str: + if len(args) > 1 and len(args[1]) > 1: + return "" + return args[0].rstrip(args[1] if len(args) > 1 else None) + + +def splitByString(args: list[Any], team: Optional["Team"], stdout: Optional[list[str]], timeout: int) -> list: + separator = args[0] + string = args[1] + if len(args) > 2: + parts = string.split(separator, args[2]) + if len(parts) > args[2]: + return parts[: args[2]] + return parts + return string.split(separator) + + def generateUUIDv4(args: list[Any], team: Optional["Team"], stdout: Optional[list[str]], timeout: int) -> str: import uuid return str(uuid.uuid4()) +def _sha256Hex(args: list[Any], team: Optional["Team"], stdout: Optional[list[str]], timeout: int) -> str: + return sha256Hex(args[0]) + + +def _md5Hex(args: list[Any], team: Optional["Team"], stdout: Optional[list[str]], timeout: int) -> str: + return md5Hex(args[0]) + + +def _sha256HmacChainHex(args: list[Any], team: Optional["Team"], stdout: Optional[list[str]], timeout: int) -> str: + return sha256HmacChainHex(args[0]) + + def keys(args: list[Any], team: Optional["Team"], stdout: Optional[list[str]], timeout: int) -> list: obj = args[0] if isinstance(obj, dict): @@ -234,6 +276,65 @@ def values(args: list[Any], team: Optional["Team"], stdout: Optional[list[str]], return [] +def arrayPushBack(args: list[Any], team: Optional["Team"], stdout: Optional[list[str]], timeout: int) -> list: + arr = args[0] + item = args[1] + if not isinstance(arr, list): + return [item] + return [*arr, item] + + +def arrayPushFront(args: list[Any], team: Optional["Team"], stdout: Optional[list[str]], timeout: int) -> list: + arr = args[0] + item = args[1] + if not isinstance(arr, list): + return [item] + return [item, *arr] + + +def arrayPopBack(args: list[Any], team: Optional["Team"], stdout: Optional[list[str]], timeout: int) -> list: + arr = args[0] + if not isinstance(arr, list): + return [] + return arr[:-1] + + +def arrayPopFront(args: list[Any], team: Optional["Team"], stdout: Optional[list[str]], timeout: int) -> list: + arr = args[0] + if not isinstance(arr, list): + return [] + return arr[1:] + + +def arraySort(args: list[Any], team: Optional["Team"], stdout: Optional[list[str]], timeout: int) -> list: + arr = args[0] + if not isinstance(arr, list): + return [] + return sorted(arr) + + +def arrayReverse(args: list[Any], team: Optional["Team"], stdout: Optional[list[str]], timeout: int) -> list: + arr = args[0] + if not isinstance(arr, list): + return [] + return arr[::-1] + + +def arrayReverseSort(args: list[Any], team: Optional["Team"], stdout: Optional[list[str]], timeout: int) -> list: + arr = args[0] + if not isinstance(arr, list): + return [] + return sorted(arr, reverse=True) + + +def arrayStringConcat(args: list[Any], team: Optional["Team"], stdout: Optional[list[str]], timeout: int) -> str: + arr = args[0] + sep = args[1] if len(args) > 1 else "" + if not isinstance(arr, list): + return "" + return sep.join([str(s) for s in arr]) + + def _now(args: list[Any], team: Optional["Team"], stdout: Optional[list[str]], timeout: int) -> Any: return now() @@ -298,9 +399,24 @@ STL: dict[str, Callable[[list[Any], Optional["Team"], list[str] | None, int], An "decodeURLComponent": decodeURLComponent, "replaceOne": replaceOne, "replaceAll": replaceAll, + "trim": trim, + "trimLeft": trimLeft, + "trimRight": trimRight, + "splitByString": splitByString, "generateUUIDv4": generateUUIDv4, + "sha256Hex": _sha256Hex, + "md5Hex": _md5Hex, + "sha256HmacChainHex": _sha256HmacChainHex, "keys": keys, "values": values, + "arrayPushBack": arrayPushBack, + "arrayPushFront": arrayPushFront, + "arrayPopBack": arrayPopBack, + "arrayPopFront": arrayPopFront, + "arraySort": arraySort, + "arrayReverse": arrayReverse, + "arrayReverseSort": arrayReverseSort, + "arrayStringConcat": arrayStringConcat, "now": _now, "toUnixTimestamp": _toUnixTimestamp, "fromUnixTimestamp": _fromUnixTimestamp, diff --git a/hogvm/python/stl/crypto.py b/hogvm/python/stl/crypto.py new file mode 100644 index 00000000000..e399c5821e0 --- /dev/null +++ b/hogvm/python/stl/crypto.py @@ -0,0 +1,21 @@ +import hashlib +import hmac + + +def md5Hex(data: str) -> str: + return hashlib.md5(data.encode()).hexdigest() + + +def sha256Hex(data: str) -> str: + return hashlib.sha256(data.encode()).hexdigest() + + +def sha256HmacChainHex(data: list) -> str: + if len(data) < 2: + raise ValueError("Data array must contain at least two elements.") + + hmac_obj = hmac.new(data[0].encode(), data[1].encode(), hashlib.sha256) + for i in range(2, len(data)): + hmac_obj = hmac.new(hmac_obj.digest(), data[i].encode(), hashlib.sha256) + + return hmac_obj.hexdigest() diff --git a/hogvm/typescript/package.json b/hogvm/typescript/package.json index 7d000c06947..3b8142b03ba 100644 --- a/hogvm/typescript/package.json +++ b/hogvm/typescript/package.json @@ -1,6 +1,6 @@ { "name": "@posthog/hogvm", - "version": "1.0.25", + "version": "1.0.26", "description": "PostHog Hog Virtual Machine", "types": "dist/index.d.ts", "main": "dist/index.js", diff --git a/hogvm/typescript/src/stl/crypto.ts b/hogvm/typescript/src/stl/crypto.ts new file mode 100644 index 00000000000..035bad46d2d --- /dev/null +++ b/hogvm/typescript/src/stl/crypto.ts @@ -0,0 +1,22 @@ +import * as crypto from 'crypto' + +export function sha256Hex(data: string): string { + return crypto.createHash('sha256').update(data).digest('hex') +} + +export function md5Hex(data: string): string { + return crypto.createHash('md5').update(data).digest('hex') +} + +export function sha256HmacChainHex(data: string[]): string { + if (data.length < 2) { + throw new Error('Data array must contain at least two elements.') + } + let hmac = crypto.createHmac('sha256', data[0]) + hmac.update(data[1]) + for (let i = 2; i < data.length; i++) { + hmac = crypto.createHmac('sha256', hmac.digest()) + hmac.update(data[i]) + } + return hmac.digest('hex') +} diff --git a/hogvm/typescript/src/stl/date.ts b/hogvm/typescript/src/stl/date.ts index 4290dae35ae..3c9f6d2e771 100644 --- a/hogvm/typescript/src/stl/date.ts +++ b/hogvm/typescript/src/stl/date.ts @@ -57,7 +57,7 @@ export function toHogDateTime(timestamp: number | HogDate, zone?: string): HogDa // EXPORTED STL functions export function now(zone?: string): HogDateTime { - return toHogDateTime(Date.now(), zone) + return toHogDateTime(Date.now() / 1000, zone) } export function toUnixTimestamp(input: HogDateTime | HogDate | string, zone?: string): number { diff --git a/hogvm/typescript/src/stl/stl.ts b/hogvm/typescript/src/stl/stl.ts index 81680cd2fad..e6b97e8d6b6 100644 --- a/hogvm/typescript/src/stl/stl.ts +++ b/hogvm/typescript/src/stl/stl.ts @@ -1,5 +1,4 @@ import { DateTime } from 'luxon' - import { fromUnixTimestamp, fromUnixTimestampMilli, @@ -15,6 +14,7 @@ import { toUnixTimestampMilli, formatDateTime, } from './date' +import { sha256Hex, sha256HmacChainHex, md5Hex } from './crypto' import { printHogStringOutput } from './print' export const STL: Record any> = { @@ -189,6 +189,49 @@ export const STL: Record replaceAll(args) { return args[0].replaceAll(args[1], args[2]) }, + trim([str, char = ' ']) { + if (char.length !== 1) { + return '' + } + let start = 0 + while (str[start] === char) { + start++ + } + let end = str.length + while (str[end - 1] === char) { + end-- + } + if (start >= end) { + return '' + } + return str.slice(start, end) + }, + trimLeft([str, char = ' ']) { + if (char.length !== 1) { + return '' + } + let start = 0 + while (str[start] === char) { + start++ + } + return str.slice(start) + }, + trimRight([str, char = ' ']) { + if (char.length !== 1) { + return '' + } + let end = str.length + while (str[end - 1] === char) { + end-- + } + return str.slice(0, end) + }, + splitByString([separator, str, maxSplits = undefined]) { + if (maxSplits === undefined) { + return str.split(separator) + } + return str.split(separator, maxSplits) + }, generateUUIDv4() { return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function (c) { const r = (Math.random() * 16) | 0 @@ -196,8 +239,16 @@ export const STL: Record return v.toString(16) }) }, - keys(args) { - const obj = args[0] + sha256Hex([str]) { + return sha256Hex(str) + }, + md5Hex([str]) { + return md5Hex(str) + }, + sha256HmacChainHex([data]) { + return sha256HmacChainHex(data) + }, + keys([obj]) { if (typeof obj === 'object') { if (Array.isArray(obj)) { return Array.from(obj.keys()) @@ -208,8 +259,7 @@ export const STL: Record } return [] }, - values(args) { - const obj = args[0] + values([obj]) { if (typeof obj === 'object') { if (Array.isArray(obj)) { return [...obj] @@ -220,6 +270,54 @@ export const STL: Record } return [] }, + arrayPushBack([arr, item]) { + if (!Array.isArray(arr)) { + return [item] + } + return [...arr, item] + }, + arrayPushFront([arr, item]) { + if (!Array.isArray(arr)) { + return [item] + } + return [item, ...arr] + }, + arrayPopBack([arr]) { + if (!Array.isArray(arr)) { + return [] + } + return arr.slice(0, arr.length - 1) + }, + arrayPopFront([arr]) { + if (!Array.isArray(arr)) { + return [] + } + return arr.slice(1) + }, + arraySort([arr]) { + if (!Array.isArray(arr)) { + return [] + } + return [...arr].sort() + }, + arrayReverse([arr]) { + if (!Array.isArray(arr)) { + return [] + } + return [...arr].reverse() + }, + arrayReverseSort([arr]) { + if (!Array.isArray(arr)) { + return [] + } + return [...arr].sort().reverse() + }, + arrayStringConcat([arr, separator = '']) { + if (!Array.isArray(arr)) { + return '' + } + return arr.join(separator) + }, now() { return now() }, diff --git a/plugin-server/package.json b/plugin-server/package.json index 8249c274e12..ce544dad494 100644 --- a/plugin-server/package.json +++ b/plugin-server/package.json @@ -50,7 +50,7 @@ "@google-cloud/storage": "^5.8.5", "@maxmind/geoip2-node": "^3.4.0", "@posthog/clickhouse": "^1.7.0", - "@posthog/hogvm": "^1.0.25", + "@posthog/hogvm": "^1.0.26", "@posthog/plugin-scaffold": "1.4.4", "@sentry/node": "^7.49.0", "@sentry/profiling-node": "^0.3.0", diff --git a/plugin-server/pnpm-lock.yaml b/plugin-server/pnpm-lock.yaml index dcb2e3d8b62..9cc6059e343 100644 --- a/plugin-server/pnpm-lock.yaml +++ b/plugin-server/pnpm-lock.yaml @@ -44,8 +44,8 @@ dependencies: specifier: ^1.7.0 version: 1.7.0 '@posthog/hogvm': - specifier: ^1.0.25 - version: 1.0.25(luxon@3.4.4)(re2@1.20.3) + specifier: ^1.0.26 + version: 1.0.26(luxon@3.4.4)(re2@1.20.3) '@posthog/plugin-scaffold': specifier: 1.4.4 version: 1.4.4 @@ -3110,8 +3110,8 @@ packages: engines: {node: '>=12'} dev: false - /@posthog/hogvm@1.0.25(luxon@3.4.4)(re2@1.20.3): - resolution: {integrity: sha512-q8j/vN/OXcRKYseCZ6veHoKcGqP9bdM+S3ECs0MGRrr7iyEKF6I7hUtu+jS5mD3Ubo+ceo8DQfTj/Me+s/4cBA==} + /@posthog/hogvm@1.0.26(luxon@3.4.4)(re2@1.20.3): + resolution: {integrity: sha512-lkJflxu/LP9TnXXW1FCePr2iaKKD7XKq68ipmtFY7v43km3ma0Se4NnuftiSX9evGIpQVXT/9JB5LY9waM/lJw==} peerDependencies: luxon: ^3.4.4 re2: ^1.21.3 diff --git a/posthog/cdp/templates/__init__.py b/posthog/cdp/templates/__init__.py index 3ba1bbd58f0..97c1fac6ceb 100644 --- a/posthog/cdp/templates/__init__.py +++ b/posthog/cdp/templates/__init__.py @@ -5,10 +5,10 @@ from .hubspot.template_hubspot import template as hubspot from .customerio.template_customerio import template as customerio from .intercom.template_intercom import template as intercom from .clearbit.template_clearbit import template as clearbit +from .aws_kinesis.template_aws_kinesis import template as aws_kinesis -HOG_FUNCTION_TEMPLATES = [webhook, hello_world, slack, hubspot, customerio, intercom, clearbit] - +HOG_FUNCTION_TEMPLATES = [webhook, hello_world, slack, hubspot, customerio, intercom, clearbit, aws_kinesis] HOG_FUNCTION_TEMPLATES_BY_ID = {template.id: template for template in HOG_FUNCTION_TEMPLATES} __all__ = ["HOG_FUNCTION_TEMPLATES", "HOG_FUNCTION_TEMPLATES_BY_ID"] diff --git a/posthog/cdp/templates/aws_kinesis/template_aws_kinesis.py b/posthog/cdp/templates/aws_kinesis/template_aws_kinesis.py new file mode 100644 index 00000000000..afd1cd2ca6e --- /dev/null +++ b/posthog/cdp/templates/aws_kinesis/template_aws_kinesis.py @@ -0,0 +1,134 @@ +from posthog.cdp.templates.hog_function_template import HogFunctionTemplate + + +template: HogFunctionTemplate = HogFunctionTemplate( + status="beta", + id="template-aws-kinesis", + name="AWS Kinesis", + description="Put data to an AWS Kinesis stream", + # icon_url="/api/projects/@current/hog_functions/icon/?id=posthog.com&temp=true", + hog=""" +fn uploadToKinesis(data) { + let region := inputs.aws_region + let endpoint := f'https://kinesis.{region}.amazonaws.com' + let service := 'kinesis' + let amzDate := formatDateTime(now(), '%Y%m%dT%H%i%sZ') + let date := formatDateTime(now(), '%Y%m%d') + + let payload := jsonStringify({ + 'StreamName': inputs.aws_kinesis_stream_arn, + 'PartitionKey': inputs.aws_kinesis_partition_key, + 'Data': base64Encode(data), + }) + + let requestHeaders := { + 'Content-Type': 'application/x-amz-json-1.1', + 'X-Amz-Target': 'Kinesis_20131202.PutRecord', + 'X-Amz-Date': amzDate, + 'Host': f'kinesis.{region}.amazonaws.com', + } + + let canonicalHeaderParts := [] + for (let key, value in requestHeaders) { + let val := replaceAll(trim(value), '\\\\s+', ' ') + canonicalHeaderParts := arrayPushBack(canonicalHeaderParts, f'{lower(key)}:{val}') + } + let canonicalHeaders := arrayStringConcat(arraySort(canonicalHeaderParts), '\\n') || '\\n' + + let signedHeaderParts := [] + for (let key, value in requestHeaders) { + signedHeaderParts := arrayPushBack(signedHeaderParts, lower(key)) + } + let signedHeaders := arrayStringConcat(arraySort(signedHeaderParts), ';') + + let canonicalRequest := arrayStringConcat([ + 'POST', + '/', + '', + canonicalHeaders, + signedHeaders, + sha256Hex(payload), + ], '\\n') + + let credentialScope := f'{date}/{region}/{service}/aws4_request' + let stringToSign := arrayStringConcat([ + 'AWS4-HMAC-SHA256', + amzDate, + credentialScope, + sha256Hex(canonicalRequest), + ], '\\n') + + let signature := sha256HmacChainHex([ + f'AWS4{inputs.aws_secret_access_key}', date, region, service, 'aws4_request', stringToSign + ]) + + let authorizationHeader := + f'AWS4-HMAC-SHA256 Credential={inputs.aws_access_key_id}/{credentialScope}, ' || + f'SignedHeaders={signedHeaders}, ' || + f'Signature={signature}' + + requestHeaders['Authorization'] := authorizationHeader + + let res := fetch(endpoint, { + 'headers': requestHeaders, + 'body': payload, + 'method': 'POST' + }) + + if (res.status >= 200 and res.status < 300) { + print('Event sent successfully!') + return + } + + print('Error sending event:', res.status, res.body) +} + +uploadToKinesis(jsonStringify(inputs.payload)) +""".strip(), + inputs_schema=[ + { + "key": "aws_access_key_id", + "type": "string", + "label": "AWS Access Key ID", + "secret": True, + "required": True, + }, + { + "key": "aws_secret_access_key", + "type": "string", + "label": "AWS Secret Access Key", + "secret": True, + "required": True, + }, + { + "key": "aws_region", + "type": "string", + "label": "AWS Region", + "secret": False, + "required": True, + "default": "us-east-1", + }, + { + "key": "aws_kinesis_stream_arn", + "type": "string", + "label": "Kinesis Stream ARN", + "secret": False, + "required": True, + }, + { + "key": "aws_kinesis_partition_key", + "type": "string", + "label": "Kinesis Partition Key", + "secret": False, + "required": False, + }, + { + "key": "payload", + "type": "json", + "label": "Message Payload", + "default": {"event": "{event}", "person": "{person}"}, + "secret": False, + "required": False, + }, + ], +) diff --git a/posthog/cdp/templates/aws_kinesis/test_template_aws_kinesis.py b/posthog/cdp/templates/aws_kinesis/test_template_aws_kinesis.py new file mode 100644 index 00000000000..5c4adee71ee --- /dev/null +++ b/posthog/cdp/templates/aws_kinesis/test_template_aws_kinesis.py @@ -0,0 +1,36 @@ +from freezegun import freeze_time +from posthog.cdp.templates.helpers import BaseHogFunctionTemplateTest +from posthog.cdp.templates.aws_kinesis.template_aws_kinesis import template as template_aws_kinesis + + +class TestTemplateAwsKinesis(BaseHogFunctionTemplateTest): + template = template_aws_kinesis + + @freeze_time("2024-04-16T12:34:51Z") + def test_function_works(self): + res = self.run_function( + inputs={ + "aws_access_key_id": "aws_access_key_id", + "aws_secret_access_key": "aws_secret_access_key", + "aws_region": "aws_region", + "aws_kinesis_stream_arn": "aws_kinesis_stream_arn", + "aws_kinesis_partition_key": "1", + "payload": {"hello": "world"}, + } + ) + + assert res.result is None + assert self.get_mock_fetch_calls()[0] == ( + "https://kinesis.aws_region.amazonaws.com", + { + "headers": { + "Content-Type": "application/x-amz-json-1.1", + "X-Amz-Target": "Kinesis_20131202.PutRecord", + "X-Amz-Date": "20240416T123451Z", + "Host": "kinesis.aws_region.amazonaws.com", + "Authorization": "AWS4-HMAC-SHA256 Credential=aws_access_key_id/20240416/aws_region/kinesis/aws4_request, SignedHeaders=content-type;host;x-amz-date;x-amz-target, Signature=65b18913b42d8a7a1d33c0711da192d5a2e99eb79fb08ab3e5eefb6488b903ff", + }, + "body": '{"StreamName": "aws_kinesis_stream_arn", "PartitionKey": "1", "Data": "eyJoZWxsbyI6ICJ3b3JsZCJ9"}', + "method": "POST", + }, + ) diff --git a/posthog/hogql/bytecode.py b/posthog/hogql/bytecode.py index 33f209b6b94..682bf9d0dd6 100644 --- a/posthog/hogql/bytecode.py +++ b/posthog/hogql/bytecode.py @@ -248,7 +248,7 @@ class BytecodeBuilder(Visitor): return response if node.name not in STL and node.name not in self.functions and node.name not in self.supported_functions: - raise QueryError(f"HogQL function `{node.name}` is not implemented") + raise QueryError(f"Hog function `{node.name}` is not implemented") if node.name in self.functions and len(node.args) != len(self.functions[node.name].params): raise QueryError( f"Function `{node.name}` expects {len(self.functions[node.name].params)} arguments, got {len(node.args)}" @@ -346,27 +346,31 @@ class BytecodeBuilder(Visitor): # set up a bunch of temporary variables expr_local = self._declare_local("__H_expr_H__") # the obj/array itself - expr_keys_local = self._declare_local("__H_keys_H__") # keys - expr_values_local = self._declare_local("__H_values_H__") # values - loop_index_local = self._declare_local("__H_index_H__") # 0 - loop_limit_local = self._declare_local("__H_limit_H__") # length of keys - key_var_local = self._declare_local(key_var) if key_var is not None else -1 # loop key - value_var_local = self._declare_local(value_var) # loop value - response.extend([Operation.NULL] * (7 if key_var is not None else 6)) - response.extend([*self.visit(node.expr), Operation.SET_LOCAL, expr_local]) + response.extend(self.visit(node.expr)) - # populate keys, value, loop index and max loop index if key_var is not None: - response.extend( - [Operation.GET_LOCAL, expr_local, Operation.CALL, "keys", 1, Operation.SET_LOCAL, expr_keys_local] - ) - response.extend( - [Operation.GET_LOCAL, expr_local, Operation.CALL, "values", 1, Operation.SET_LOCAL, expr_values_local] - ) - response.extend([Operation.INTEGER, 0, Operation.SET_LOCAL, loop_index_local]) - response.extend( - [Operation.GET_LOCAL, expr_values_local, Operation.CALL, "length", 1, Operation.SET_LOCAL, loop_limit_local] - ) + expr_keys_local = self._declare_local("__H_keys_H__") # keys + response.extend([Operation.GET_LOCAL, expr_local, Operation.CALL, "keys", 1]) + else: + expr_keys_local = None + + expr_values_local = self._declare_local("__H_values_H__") # values + response.extend([Operation.GET_LOCAL, expr_local, Operation.CALL, "values", 1]) + + loop_index_local = self._declare_local("__H_index_H__") # 0 + response.extend([Operation.INTEGER, 0]) + + loop_limit_local = self._declare_local("__H_limit_H__") # length of keys + response.extend([Operation.GET_LOCAL, expr_values_local, Operation.CALL, "length", 1]) + + if key_var is not None: + key_var_local = self._declare_local(key_var) # loop key + response.extend([Operation.NULL]) + else: + key_var_local = None + + value_var_local = self._declare_local(value_var) # loop value + response.extend([Operation.NULL]) # check if loop_index < loop_limit condition = [Operation.GET_LOCAL, loop_limit_local, Operation.GET_LOCAL, loop_index_local, Operation.LT] diff --git a/posthog/hogql/test/test_metadata.py b/posthog/hogql/test/test_metadata.py index 5106270aa05..68b7183f7b0 100644 --- a/posthog/hogql/test/test_metadata.py +++ b/posthog/hogql/test/test_metadata.py @@ -339,7 +339,7 @@ class TestMetadata(ClickhouseTestMixin, APIBaseTest): "isValidView": False, "notices": [], "warnings": [], - "errors": [{"end": 15, "fix": None, "message": "HogQL function `NONO` is not implemented", "start": 9}], + "errors": [{"end": 15, "fix": None, "message": "Hog function `NONO` is not implemented", "start": 9}], }, ) @@ -361,8 +361,6 @@ class TestMetadata(ClickhouseTestMixin, APIBaseTest): metadata.dict() | { "isValid": False, - "errors": [ - {"end": 17, "fix": None, "message": "HogQL function `NONO` is not implemented", "start": 11} - ], + "errors": [{"end": 17, "fix": None, "message": "Hog function `NONO` is not implemented", "start": 11}], }, )