From 043aa16e5b6ac0047693eeaca05d4659fa3481b8 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Wed, 10 Jul 2024 15:11:15 +0200 Subject: [PATCH] feat(hog): memory limits (#23564) --- hogvm/__tests__/__snapshots__/printLoops.hoge | 4 + .../__tests__/__snapshots__/printLoops.stdout | 2 + .../__tests__/__snapshots__/printLoops2.hoge | 4 + .../__snapshots__/printLoops2.stdout | 2 + hogvm/__tests__/printLoops.hog | 22 +++ hogvm/__tests__/printLoops2.hog | 11 ++ hogvm/python/execute.py | 116 ++++++++----- hogvm/python/stl/__init__.py | 23 ++- hogvm/python/stl/print.py | 27 ++- hogvm/python/test/test_execute.py | 137 ++++++++++++++++ hogvm/python/utils.py | 24 +++ hogvm/test.sh | 7 +- hogvm/typescript/package.json | 2 +- .../typescript/src/__tests__/execute.test.ts | 146 ++++++++++++++++- hogvm/typescript/src/__tests__/utils.test.ts | 32 ++++ hogvm/typescript/src/execute.ts | 155 +++++++++++------- hogvm/typescript/src/stl/print.ts | 55 ++++--- hogvm/typescript/src/stl/stl.ts | 42 +++-- hogvm/typescript/src/utils.ts | 40 +++++ plugin-server/package.json | 2 +- plugin-server/pnpm-lock.yaml | 8 +- 21 files changed, 700 insertions(+), 161 deletions(-) create mode 100644 hogvm/__tests__/__snapshots__/printLoops.hoge create mode 100644 hogvm/__tests__/__snapshots__/printLoops.stdout create mode 100644 hogvm/__tests__/__snapshots__/printLoops2.hoge create mode 100644 hogvm/__tests__/__snapshots__/printLoops2.stdout create mode 100644 hogvm/__tests__/printLoops.hog create mode 100644 hogvm/__tests__/printLoops2.hog create mode 100644 hogvm/typescript/src/__tests__/utils.test.ts diff --git a/hogvm/__tests__/__snapshots__/printLoops.hoge b/hogvm/__tests__/__snapshots__/printLoops.hoge new file mode 100644 index 00000000000..a53576adf8f --- /dev/null +++ b/hogvm/__tests__/__snapshots__/printLoops.hoge @@ -0,0 +1,4 @@ +["_h", 32, "key", 32, "value", 32, "key2", 32, "value2", 42, 2, 32, "na", 33, 0, 33, 100, 36, 2, 15, 40, 45, 32, "na", +36, 1, 2, "concat", 2, 37, 1, 36, 0, 36, 2, 32, "key_", 2, "concat", 2, 32, "wasted", 32, " batman!", 36, 1, 32, +"memory: ", 2, "concat", 3, 32, "something", 36, 0, 42, 2, 46, 33, 1, 36, 2, 6, 37, 2, 39, -52, 35, 36, 0, 2, "print", +1, 35, 36, 0, 2, "jsonStringify", 1, 36, 2, 2, "jsonParse", 1, 2, "print", 1, 35, 35, 35, 35] diff --git a/hogvm/__tests__/__snapshots__/printLoops.stdout b/hogvm/__tests__/__snapshots__/printLoops.stdout new file mode 100644 index 00000000000..f5a15515e95 --- /dev/null +++ b/hogvm/__tests__/__snapshots__/printLoops.stdout @@ -0,0 +1,2 @@ +{'key': 'value', 'key2': 'value2', 'key_0': {'wasted': 'memory: nana batman!', 'something': null}, 'key_1': {'wasted': 'memory: nanana batman!', 'something': null}, 'key_2': {'wasted': 'memory: nananana batman!', 'something': null}, 'key_3': {'wasted': 'memory: nanananana batman!', 'something': null}, 'key_4': {'wasted': 'memory: nananananana batman!', 'something': null}, 'key_5': {'wasted': 'memory: nanananananana batman!', 'something': null}, 'key_6': {'wasted': 'memory: nananananananana batman!', 'something': null}, 'key_7': {'wasted': 'memory: nanananananananana batman!', 'something': null}, 'key_8': {'wasted': 'memory: nananananananananana batman!', 'something': null}, 'key_9': {'wasted': 'memory: nanananananananananana batman!', 'something': null}, 'key_10': {'wasted': 'memory: nananananananananananana batman!', 'something': null}, 'key_11': {'wasted': 'memory: nanananananananananananana batman!', 'something': null}, 'key_12': {'wasted': 'memory: nananananananananananananana batman!', 'something': null}, 'key_13': {'wasted': 'memory: nanananananananananananananana batman!', 'something': null}, 'key_14': {'wasted': 'memory: nananananananananananananananana batman!', 'something': null}, 'key_15': {'wasted': 'memory: nanananananananananananananananana batman!', 'something': null}, 'key_16': {'wasted': 'memory: nananananananananananananananananana batman!', 'something': null}, 'key_17': {'wasted': 'memory: nanananananananananananananananananana batman!', 'something': null}, 'key_18': {'wasted': 'memory: nananananananananananananananananananana batman!', 'something': null}, 'key_19': {'wasted': 'memory: nanananananananananananananananananananana batman!', 'something': null}, 'key_20': {'wasted': 'memory: nananananananananananananananananananananana batman!', 'something': null}, 'key_21': {'wasted': 'memory: nanananananananananananananananananananananana batman!', 'something': null}, 'key_22': {'wasted': 'memory: nananananananananananananananananananananananana batman!', 'something': null}, 'key_23': {'wasted': 'memory: nanananananananananananananananananananananananana batman!', 'something': null}, 'key_24': {'wasted': 'memory: nananananananananananananananananananananananananana batman!', 'something': null}, 'key_25': {'wasted': 'memory: nanananananananananananananananananananananananananana batman!', 'something': null}, 'key_26': {'wasted': 'memory: nananananananananananananananananananananananananananana batman!', 'something': null}, 'key_27': {'wasted': 'memory: nanananananananananananananananananananananananananananana batman!', 'something': null}, 'key_28': {'wasted': 'memory: nananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_29': {'wasted': 'memory: nanananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_30': {'wasted': 'memory: nananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_31': {'wasted': 'memory: nanananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_32': {'wasted': 'memory: nananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_33': {'wasted': 'memory: nanananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_34': {'wasted': 'memory: nananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_35': {'wasted': 'memory: nanananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_36': {'wasted': 'memory: nananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_37': {'wasted': 'memory: nanananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_38': {'wasted': 'memory: nananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_39': {'wasted': 'memory: nanananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_40': {'wasted': 'memory: nananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_41': {'wasted': 'memory: nanananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_42': {'wasted': 'memory: nananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_43': {'wasted': 'memory: nanananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_44': {'wasted': 'memory: nananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_45': {'wasted': 'memory: nanananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_46': {'wasted': 'memory: nananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_47': {'wasted': 'memory: nanananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_48': {'wasted': 'memory: nananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_49': {'wasted': 'memory: nanananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_50': {'wasted': 'memory: nananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_51': {'wasted': 'memory: nanananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_52': {'wasted': 'memory: nananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_53': {'wasted': 'memory: nanananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_54': {'wasted': 'memory: nananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_55': {'wasted': 'memory: nanananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_56': {'wasted': 'memory: nananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_57': {'wasted': 'memory: nanananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_58': {'wasted': 'memory: nananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_59': {'wasted': 'memory: nanananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_60': {'wasted': 'memory: nananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_61': {'wasted': 'memory: nanananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_62': {'wasted': 'memory: nananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_63': {'wasted': 'memory: nanananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_64': {'wasted': 'memory: nananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_65': {'wasted': 'memory: nanananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_66': {'wasted': 'memory: nananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_67': {'wasted': 'memory: nanananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_68': {'wasted': 'memory: nananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_69': {'wasted': 'memory: nanananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_70': {'wasted': 'memory: nananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_71': {'wasted': 'memory: nanananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_72': {'wasted': 'memory: nananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_73': {'wasted': 'memory: nanananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_74': {'wasted': 'memory: nananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_75': {'wasted': 'memory: nanananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_76': {'wasted': 'memory: nananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_77': {'wasted': 'memory: nanananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_78': {'wasted': 'memory: nananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_79': {'wasted': 'memory: nanananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_80': {'wasted': 'memory: nananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_81': {'wasted': 'memory: nanananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_82': {'wasted': 'memory: nananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_83': {'wasted': 'memory: nanananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_84': {'wasted': 'memory: nananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_85': {'wasted': 'memory: nanananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_86': {'wasted': 'memory: nananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_87': {'wasted': 'memory: nanananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_88': {'wasted': 'memory: nananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_89': {'wasted': 'memory: nanananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_90': {'wasted': 'memory: nananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_91': {'wasted': 'memory: nanananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_92': {'wasted': 'memory: nananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_93': {'wasted': 'memory: nanananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_94': {'wasted': 'memory: nananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_95': {'wasted': 'memory: nanananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_96': {'wasted': 'memory: nananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_97': {'wasted': 'memory: nanananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_98': {'wasted': 'memory: nananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_99': {'wasted': 'memory: nanananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}} +{'key': 'value', 'key2': 'value2', 'key_0': {'wasted': 'memory: nana batman!', 'something': null}, 'key_1': {'wasted': 'memory: nanana batman!', 'something': null}, 'key_2': {'wasted': 'memory: nananana batman!', 'something': null}, 'key_3': {'wasted': 'memory: nanananana batman!', 'something': null}, 'key_4': {'wasted': 'memory: nananananana batman!', 'something': null}, 'key_5': {'wasted': 'memory: nanananananana batman!', 'something': null}, 'key_6': {'wasted': 'memory: nananananananana batman!', 'something': null}, 'key_7': {'wasted': 'memory: nanananananananana batman!', 'something': null}, 'key_8': {'wasted': 'memory: nananananananananana batman!', 'something': null}, 'key_9': {'wasted': 'memory: nanananananananananana batman!', 'something': null}, 'key_10': {'wasted': 'memory: nananananananananananana batman!', 'something': null}, 'key_11': {'wasted': 'memory: nanananananananananananana batman!', 'something': null}, 'key_12': {'wasted': 'memory: nananananananananananananana batman!', 'something': null}, 'key_13': {'wasted': 'memory: nanananananananananananananana batman!', 'something': null}, 'key_14': {'wasted': 'memory: nananananananananananananananana batman!', 'something': null}, 'key_15': {'wasted': 'memory: nanananananananananananananananana batman!', 'something': null}, 'key_16': {'wasted': 'memory: nananananananananananananananananana batman!', 'something': null}, 'key_17': {'wasted': 'memory: nanananananananananananananananananana batman!', 'something': null}, 'key_18': {'wasted': 'memory: nananananananananananananananananananana batman!', 'something': null}, 'key_19': {'wasted': 'memory: nanananananananananananananananananananana batman!', 'something': null}, 'key_20': {'wasted': 'memory: nananananananananananananananananananananana batman!', 'something': null}, 'key_21': {'wasted': 'memory: nanananananananananananananananananananananana batman!', 'something': null}, 'key_22': {'wasted': 'memory: nananananananananananananananananananananananana batman!', 'something': null}, 'key_23': {'wasted': 'memory: nanananananananananananananananananananananananana batman!', 'something': null}, 'key_24': {'wasted': 'memory: nananananananananananananananananananananananananana batman!', 'something': null}, 'key_25': {'wasted': 'memory: nanananananananananananananananananananananananananana batman!', 'something': null}, 'key_26': {'wasted': 'memory: nananananananananananananananananananananananananananana batman!', 'something': null}, 'key_27': {'wasted': 'memory: nanananananananananananananananananananananananananananana batman!', 'something': null}, 'key_28': {'wasted': 'memory: nananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_29': {'wasted': 'memory: nanananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_30': {'wasted': 'memory: nananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_31': {'wasted': 'memory: nanananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_32': {'wasted': 'memory: nananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_33': {'wasted': 'memory: nanananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_34': {'wasted': 'memory: nananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_35': {'wasted': 'memory: nanananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_36': {'wasted': 'memory: nananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_37': {'wasted': 'memory: nanananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_38': {'wasted': 'memory: nananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_39': {'wasted': 'memory: nanananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_40': {'wasted': 'memory: nananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_41': {'wasted': 'memory: nanananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_42': {'wasted': 'memory: nananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_43': {'wasted': 'memory: nanananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_44': {'wasted': 'memory: nananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_45': {'wasted': 'memory: nanananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_46': {'wasted': 'memory: nananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_47': {'wasted': 'memory: nanananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_48': {'wasted': 'memory: nananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_49': {'wasted': 'memory: nanananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_50': {'wasted': 'memory: nananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_51': {'wasted': 'memory: nanananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_52': {'wasted': 'memory: nananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_53': {'wasted': 'memory: nanananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_54': {'wasted': 'memory: nananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_55': {'wasted': 'memory: nanananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_56': {'wasted': 'memory: nananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_57': {'wasted': 'memory: nanananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_58': {'wasted': 'memory: nananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_59': {'wasted': 'memory: nanananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_60': {'wasted': 'memory: nananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_61': {'wasted': 'memory: nanananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_62': {'wasted': 'memory: nananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_63': {'wasted': 'memory: nanananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_64': {'wasted': 'memory: nananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_65': {'wasted': 'memory: nanananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_66': {'wasted': 'memory: nananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_67': {'wasted': 'memory: nanananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_68': {'wasted': 'memory: nananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_69': {'wasted': 'memory: nanananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_70': {'wasted': 'memory: nananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_71': {'wasted': 'memory: nanananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_72': {'wasted': 'memory: nananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_73': {'wasted': 'memory: nanananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_74': {'wasted': 'memory: nananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_75': {'wasted': 'memory: nanananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_76': {'wasted': 'memory: nananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_77': {'wasted': 'memory: nanananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_78': {'wasted': 'memory: nananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_79': {'wasted': 'memory: nanananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_80': {'wasted': 'memory: nananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_81': {'wasted': 'memory: nanananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_82': {'wasted': 'memory: nananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_83': {'wasted': 'memory: nanananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_84': {'wasted': 'memory: nananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_85': {'wasted': 'memory: nanananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_86': {'wasted': 'memory: nananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_87': {'wasted': 'memory: nanananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_88': {'wasted': 'memory: nananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_89': {'wasted': 'memory: nanananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_90': {'wasted': 'memory: nananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_91': {'wasted': 'memory: nanananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_92': {'wasted': 'memory: nananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_93': {'wasted': 'memory: nanananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_94': {'wasted': 'memory: nananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_95': {'wasted': 'memory: nanananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_96': {'wasted': 'memory: nananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_97': {'wasted': 'memory: nanananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_98': {'wasted': 'memory: nananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}, 'key_99': {'wasted': 'memory: nanananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananananana batman!', 'something': null}} diff --git a/hogvm/__tests__/__snapshots__/printLoops2.hoge b/hogvm/__tests__/__snapshots__/printLoops2.hoge new file mode 100644 index 00000000000..487604661c0 --- /dev/null +++ b/hogvm/__tests__/__snapshots__/printLoops2.hoge @@ -0,0 +1,4 @@ +["_h", 32, "key", 32, "value", 32, "key2", 32, "value2", 42, 2, 32, "key", 32, "value", 32, "key2", 32, "value2", 42, 2, +33, 0, 33, 30, 36, 2, 15, 40, 25, 36, 0, 36, 2, 32, "key_", 2, "concat", 2, 32, "something", 36, 1, 42, 1, 46, 33, 1, +36, 2, 6, 37, 2, 39, -32, 35, 36, 0, 2, "print", 1, 35, 36, 0, 2, "jsonStringify", 1, 2, "jsonParse", 1, 2, "print", 1, +35, 35, 35] diff --git a/hogvm/__tests__/__snapshots__/printLoops2.stdout b/hogvm/__tests__/__snapshots__/printLoops2.stdout new file mode 100644 index 00000000000..8efb7c32d76 --- /dev/null +++ b/hogvm/__tests__/__snapshots__/printLoops2.stdout @@ -0,0 +1,2 @@ +{'key': 'value', 'key2': 'value2', 'key_0': {'something': {'key': 'value', 'key2': 'value2'}}, 'key_1': {'something': {'key': 'value', 'key2': 'value2'}}, 'key_2': {'something': {'key': 'value', 'key2': 'value2'}}, 'key_3': {'something': {'key': 'value', 'key2': 'value2'}}, 'key_4': {'something': {'key': 'value', 'key2': 'value2'}}, 'key_5': {'something': {'key': 'value', 'key2': 'value2'}}, 'key_6': {'something': {'key': 'value', 'key2': 'value2'}}, 'key_7': {'something': {'key': 'value', 'key2': 'value2'}}, 'key_8': {'something': {'key': 'value', 'key2': 'value2'}}, 'key_9': {'something': {'key': 'value', 'key2': 'value2'}}, 'key_10': {'something': {'key': 'value', 'key2': 'value2'}}, 'key_11': {'something': {'key': 'value', 'key2': 'value2'}}, 'key_12': {'something': {'key': 'value', 'key2': 'value2'}}, 'key_13': {'something': {'key': 'value', 'key2': 'value2'}}, 'key_14': {'something': {'key': 'value', 'key2': 'value2'}}, 'key_15': {'something': {'key': 'value', 'key2': 'value2'}}, 'key_16': {'something': {'key': 'value', 'key2': 'value2'}}, 'key_17': {'something': {'key': 'value', 'key2': 'value2'}}, 'key_18': {'something': {'key': 'value', 'key2': 'value2'}}, 'key_19': {'something': {'key': 'value', 'key2': 'value2'}}, 'key_20': {'something': {'key': 'value', 'key2': 'value2'}}, 'key_21': {'something': {'key': 'value', 'key2': 'value2'}}, 'key_22': {'something': {'key': 'value', 'key2': 'value2'}}, 'key_23': {'something': {'key': 'value', 'key2': 'value2'}}, 'key_24': {'something': {'key': 'value', 'key2': 'value2'}}, 'key_25': {'something': {'key': 'value', 'key2': 'value2'}}, 'key_26': {'something': {'key': 'value', 'key2': 'value2'}}, 'key_27': {'something': {'key': 'value', 'key2': 'value2'}}, 'key_28': {'something': {'key': 'value', 'key2': 'value2'}}, 'key_29': {'something': {'key': 'value', 'key2': 'value2'}}} +{'key': 'value', 'key2': 'value2', 'key_0': {'something': {'key': 'value', 'key2': 'value2'}}, 'key_1': {'something': {'key': 'value', 'key2': 'value2'}}, 'key_2': {'something': {'key': 'value', 'key2': 'value2'}}, 'key_3': {'something': {'key': 'value', 'key2': 'value2'}}, 'key_4': {'something': {'key': 'value', 'key2': 'value2'}}, 'key_5': {'something': {'key': 'value', 'key2': 'value2'}}, 'key_6': {'something': {'key': 'value', 'key2': 'value2'}}, 'key_7': {'something': {'key': 'value', 'key2': 'value2'}}, 'key_8': {'something': {'key': 'value', 'key2': 'value2'}}, 'key_9': {'something': {'key': 'value', 'key2': 'value2'}}, 'key_10': {'something': {'key': 'value', 'key2': 'value2'}}, 'key_11': {'something': {'key': 'value', 'key2': 'value2'}}, 'key_12': {'something': {'key': 'value', 'key2': 'value2'}}, 'key_13': {'something': {'key': 'value', 'key2': 'value2'}}, 'key_14': {'something': {'key': 'value', 'key2': 'value2'}}, 'key_15': {'something': {'key': 'value', 'key2': 'value2'}}, 'key_16': {'something': {'key': 'value', 'key2': 'value2'}}, 'key_17': {'something': {'key': 'value', 'key2': 'value2'}}, 'key_18': {'something': {'key': 'value', 'key2': 'value2'}}, 'key_19': {'something': {'key': 'value', 'key2': 'value2'}}, 'key_20': {'something': {'key': 'value', 'key2': 'value2'}}, 'key_21': {'something': {'key': 'value', 'key2': 'value2'}}, 'key_22': {'something': {'key': 'value', 'key2': 'value2'}}, 'key_23': {'something': {'key': 'value', 'key2': 'value2'}}, 'key_24': {'something': {'key': 'value', 'key2': 'value2'}}, 'key_25': {'something': {'key': 'value', 'key2': 'value2'}}, 'key_26': {'something': {'key': 'value', 'key2': 'value2'}}, 'key_27': {'something': {'key': 'value', 'key2': 'value2'}}, 'key_28': {'something': {'key': 'value', 'key2': 'value2'}}, 'key_29': {'something': {'key': 'value', 'key2': 'value2'}}} diff --git a/hogvm/__tests__/printLoops.hog b/hogvm/__tests__/printLoops.hog new file mode 100644 index 00000000000..3c7a7f03a0e --- /dev/null +++ b/hogvm/__tests__/printLoops.hog @@ -0,0 +1,22 @@ +// Printing recursive objects. +let obj := {'key': 'value', 'key2': 'value2'} +let str := 'na' +for (let i := 0; i < 100; i := i + 1) { + str := str || 'na' + obj[f'key_{i}'] := { + 'wasted': 'memory: ' || str || ' batman!', + 'something': obj, // something links to obj + } +} + +// printing works without loops +print(obj) + +// this doesn't crash +let json := jsonStringify(obj) + +// Commented out because JSON output is slightly different in python vs nodejs +// print(json) + +// Should be equal to the original printed object -> nulls instead of recursive nodes +print(jsonParse(json)) diff --git a/hogvm/__tests__/printLoops2.hog b/hogvm/__tests__/printLoops2.hog new file mode 100644 index 00000000000..44412b7733b --- /dev/null +++ b/hogvm/__tests__/printLoops2.hog @@ -0,0 +1,11 @@ +// Printing recursive objects. +let root := {'key': 'value', 'key2': 'value2'} +let leaf := {'key': 'value', 'key2': 'value2'} +for (let i := 0; i < 30; i := i + 1) { + root[f'key_{i}'] := { + 'something': leaf, + } +} +// Should NOT replace all leaves with nulls. +print(root) +print(jsonParse(jsonStringify(root))) diff --git a/hogvm/python/execute.py b/hogvm/python/execute.py index 63f82204536..4129d14406f 100644 --- a/hogvm/python/execute.py +++ b/hogvm/python/execute.py @@ -10,11 +10,13 @@ from hogvm.python.operation import Operation, HOGQL_BYTECODE_IDENTIFIER from hogvm.python.stl import STL from dataclasses import dataclass -from hogvm.python.utils import HogVMException, get_nested_value, like, set_nested_value +from hogvm.python.utils import HogVMException, get_nested_value, like, set_nested_value, calculate_cost if TYPE_CHECKING: from posthog.models import Team +MAX_MEMORY = 64 * 1024 * 1024 # 64 MB + @dataclass class BytecodeResult: @@ -35,8 +37,11 @@ def execute_bytecode( start_time = time.time() last_op = len(bytecode) - 1 stack: list = [] + mem_stack: list = [] call_stack: list[tuple[int, int, int]] = [] # (ip, stack_start, arg_len) declared_functions: dict[str, tuple[int, int]] = {} + mem_used = 0 + max_mem_used = 0 ip = -1 ops = 0 stdout: list[str] = [] @@ -52,8 +57,20 @@ def execute_bytecode( def pop_stack(): if not stack: raise HogVMException("Stack underflow") + nonlocal mem_used + mem_used -= mem_stack.pop() return stack.pop() + def push_stack(value): + stack.append(value) + mem_stack.append(calculate_cost(value)) + nonlocal mem_used + mem_used += mem_stack[-1] + nonlocal max_mem_used + max_mem_used = max(mem_used, max_mem_used) + if mem_used > MAX_MEMORY: + raise HogVMException(f"Memory limit of {MAX_MEMORY} bytes exceeded. Tried to allocate {mem_used} bytes.") + if next_token() != HOGQL_BYTECODE_IDENTIFIER: raise HogVMException(f"Invalid bytecode. Must start with '{HOGQL_BYTECODE_IDENTIFIER}'") @@ -75,72 +92,72 @@ def execute_bytecode( case None: break case Operation.STRING: - stack.append(next_token()) + push_stack(next_token()) case Operation.INTEGER: - stack.append(next_token()) + push_stack(next_token()) case Operation.FLOAT: - stack.append(next_token()) + push_stack(next_token()) case Operation.TRUE: - stack.append(True) + push_stack(True) case Operation.FALSE: - stack.append(False) + push_stack(False) case Operation.NULL: - stack.append(None) + push_stack(None) case Operation.NOT: - stack.append(not pop_stack()) + push_stack(not pop_stack()) case Operation.AND: - stack.append(all([pop_stack() for _ in range(next_token())])) # noqa: C419 + push_stack(all([pop_stack() for _ in range(next_token())])) # noqa: C419 case Operation.OR: - stack.append(any([pop_stack() for _ in range(next_token())])) # noqa: C419 + push_stack(any([pop_stack() for _ in range(next_token())])) # noqa: C419 case Operation.PLUS: - stack.append(pop_stack() + pop_stack()) + push_stack(pop_stack() + pop_stack()) case Operation.MINUS: - stack.append(pop_stack() - pop_stack()) + push_stack(pop_stack() - pop_stack()) case Operation.DIVIDE: - stack.append(pop_stack() / pop_stack()) + push_stack(pop_stack() / pop_stack()) case Operation.MULTIPLY: - stack.append(pop_stack() * pop_stack()) + push_stack(pop_stack() * pop_stack()) case Operation.MOD: - stack.append(pop_stack() % pop_stack()) + push_stack(pop_stack() % pop_stack()) case Operation.EQ: - stack.append(pop_stack() == pop_stack()) + push_stack(pop_stack() == pop_stack()) case Operation.NOT_EQ: - stack.append(pop_stack() != pop_stack()) + push_stack(pop_stack() != pop_stack()) case Operation.GT: - stack.append(pop_stack() > pop_stack()) + push_stack(pop_stack() > pop_stack()) case Operation.GT_EQ: - stack.append(pop_stack() >= pop_stack()) + push_stack(pop_stack() >= pop_stack()) case Operation.LT: - stack.append(pop_stack() < pop_stack()) + push_stack(pop_stack() < pop_stack()) case Operation.LT_EQ: - stack.append(pop_stack() <= pop_stack()) + push_stack(pop_stack() <= pop_stack()) case Operation.LIKE: - stack.append(like(pop_stack(), pop_stack())) + push_stack(like(pop_stack(), pop_stack())) case Operation.ILIKE: - stack.append(like(pop_stack(), pop_stack(), re.IGNORECASE)) + push_stack(like(pop_stack(), pop_stack(), re.IGNORECASE)) case Operation.NOT_LIKE: - stack.append(not like(pop_stack(), pop_stack())) + push_stack(not like(pop_stack(), pop_stack())) case Operation.NOT_ILIKE: - stack.append(not like(pop_stack(), pop_stack(), re.IGNORECASE)) + push_stack(not like(pop_stack(), pop_stack(), re.IGNORECASE)) case Operation.IN: - stack.append(pop_stack() in pop_stack()) + push_stack(pop_stack() in pop_stack()) case Operation.NOT_IN: - stack.append(pop_stack() not in pop_stack()) + push_stack(pop_stack() not in pop_stack()) case Operation.REGEX: args = [pop_stack(), pop_stack()] - stack.append(bool(re.search(re.compile(args[1]), args[0]))) + push_stack(bool(re.search(re.compile(args[1]), args[0]))) case Operation.NOT_REGEX: args = [pop_stack(), pop_stack()] - stack.append(not bool(re.search(re.compile(args[1]), args[0]))) + push_stack(not bool(re.search(re.compile(args[1]), args[0]))) case Operation.IREGEX: args = [pop_stack(), pop_stack()] - stack.append(bool(re.search(re.compile(args[1], re.RegexFlag.IGNORECASE), args[0]))) + push_stack(bool(re.search(re.compile(args[1], re.RegexFlag.IGNORECASE), args[0]))) case Operation.NOT_IREGEX: args = [pop_stack(), pop_stack()] - stack.append(not bool(re.search(re.compile(args[1], re.RegexFlag.IGNORECASE), args[0]))) + push_stack(not bool(re.search(re.compile(args[1], re.RegexFlag.IGNORECASE), args[0]))) case Operation.GET_GLOBAL: chain = [pop_stack() for _ in range(next_token())] - stack.append(deepcopy(get_nested_value(globals, chain))) + push_stack(deepcopy(get_nested_value(globals, chain))) case Operation.POP: pop_stack() case Operation.RETURN: @@ -148,22 +165,29 @@ def execute_bytecode( ip, stack_start, arg_len = call_stack.pop() response = pop_stack() stack = stack[0:stack_start] - stack.append(response) + mem_used -= sum(mem_stack[stack_start:]) + mem_stack = mem_stack[0:stack_start] + push_stack(response) else: return BytecodeResult(result=pop_stack(), stdout=stdout, bytecode=bytecode) case Operation.GET_LOCAL: stack_start = 0 if not call_stack else call_stack[-1][1] - stack.append(stack[next_token() + stack_start]) + push_stack(stack[next_token() + stack_start]) case Operation.SET_LOCAL: stack_start = 0 if not call_stack else call_stack[-1][1] value = pop_stack() - stack[next_token() + stack_start] = value + index = next_token() + stack_start + stack[index] = value + last_cost = mem_stack[index] + mem_stack[index] = calculate_cost(value) + mem_used += mem_stack[index] - last_cost + max_mem_used = max(mem_used, max_mem_used) case Operation.GET_PROPERTY: property = pop_stack() - stack.append(get_nested_value(pop_stack(), [property])) + push_stack(get_nested_value(pop_stack(), [property])) case Operation.GET_PROPERTY_NULLISH: property = pop_stack() - stack.append(get_nested_value(pop_stack(), [property], nullish=True)) + push_stack(get_nested_value(pop_stack(), [property], nullish=True)) case Operation.SET_PROPERTY: value = pop_stack() field = pop_stack() @@ -173,19 +197,25 @@ def execute_bytecode( if count > 0: elems = stack[-(count * 2) :] stack = stack[: -(count * 2)] - stack.append({elems[i]: elems[i + 1] for i in range(0, len(elems), 2)}) + mem_used -= sum(mem_stack[-(count * 2) :]) + mem_stack = mem_stack[: -(count * 2)] + push_stack({elems[i]: elems[i + 1] for i in range(0, len(elems), 2)}) else: - stack.append({}) + push_stack({}) case Operation.ARRAY: count = next_token() elems = stack[-count:] stack = stack[:-count] - stack.append(elems) + mem_used -= sum(mem_stack[-count:]) + mem_stack = mem_stack[:-count] + push_stack(elems) case Operation.TUPLE: count = next_token() elems = stack[-count:] stack = stack[:-count] - stack.append(tuple(elems)) + mem_used -= sum(mem_stack[-count:]) + mem_stack = mem_stack[:-count] + push_stack(tuple(elems)) case Operation.JUMP: count = next_token() ip += count @@ -214,13 +244,13 @@ def execute_bytecode( args = [pop_stack() for _ in range(next_token())] if functions is not None and name in functions: - stack.append(functions[name](*args)) + push_stack(functions[name](*args)) continue if name not in STL: raise HogVMException(f"Unsupported function call: {name}") - stack.append(STL[name](name, args, team, stdout, timeout)) + push_stack(STL[name](name, args, team, stdout, timeout)) if ip == last_op: break if debug: diff --git a/hogvm/python/stl/__init__.py b/hogvm/python/stl/__init__.py index a010ce19588..8734eea8cc7 100644 --- a/hogvm/python/stl/__init__.py +++ b/hogvm/python/stl/__init__.py @@ -107,9 +107,28 @@ def jsonParse(name: str, args: list[Any], team: Optional["Team"], stdout: Option def jsonStringify(name: str, args: list[Any], team: Optional["Team"], stdout: Optional[list[str]], timeout: int) -> str: + marked = set() + + def json_safe(obj): + if isinstance(obj, dict) or isinstance(obj, list) or isinstance(obj, tuple): + if id(obj) in marked: + return None + else: + marked.add(id(obj)) + try: + if isinstance(obj, dict): + return {json_safe(k): json_safe(v) for k, v in obj.items()} + elif isinstance(obj, list): + return [json_safe(v) for v in obj] + elif isinstance(obj, tuple): + return tuple(json_safe(v) for v in obj) + finally: + marked.remove(id(obj)) + return obj + if len(args) > 1 and isinstance(args[1], int) and args[1] > 0: - return json.dumps(args[0], indent=args[1]) - return json.dumps(args[0]) + return json.dumps(json_safe(args[0]), indent=args[1]) + return json.dumps(json_safe(args[0])) def base64Encode(name: str, args: list[Any], team: Optional["Team"], stdout: Optional[list[str]], timeout: int) -> str: diff --git a/hogvm/python/stl/print.py b/hogvm/python/stl/print.py index c6735376649..ad9f50482bb 100644 --- a/hogvm/python/stl/print.py +++ b/hogvm/python/stl/print.py @@ -31,15 +31,24 @@ def escape_identifier(identifier: str | int) -> str: return "`{}`".format("".join(backquote_escape_chars_map.get(c, c) for c in identifier)) -def print_hog_value(obj): - if isinstance(obj, list): - return f"[{', '.join(map(print_hog_value, obj))}]" - if isinstance(obj, dict): - return f"{{{', '.join([f'{print_hog_value(key)}: {print_hog_value(value)}' for key, value in obj.items()])}}}" - if isinstance(obj, tuple): - if len(obj) < 2: - return f"tuple({', '.join(map(print_hog_value, obj))})" - return f"({', '.join(map(print_hog_value, obj))})" +def print_hog_value(obj, marked: set | None = None): + if marked is None: + marked = set() + if isinstance(obj, list) or isinstance(obj, dict) or isinstance(obj, tuple): + if id(obj) in marked: + return "null" + marked.add(id(obj)) + try: + if isinstance(obj, list): + return f"[{', '.join([print_hog_value(o, marked) for o in obj])}]" + if isinstance(obj, dict): + return f"{{{', '.join([f'{print_hog_value(key, marked)}: {print_hog_value(value, marked)}' for key, value in obj.items()])}}}" + if isinstance(obj, tuple): + if len(obj) < 2: + return f"tuple({', '.join([print_hog_value(o, marked) for o in obj])})" + return f"({', '.join([print_hog_value(o, marked) for o in obj])})" + finally: + marked.remove(id(obj)) if obj is True: return "true" if obj is False: diff --git a/hogvm/python/test/test_execute.py b/hogvm/python/test/test_execute.py index 0069d797027..4d7814439aa 100644 --- a/hogvm/python/test/test_execute.py +++ b/hogvm/python/test/test_execute.py @@ -134,6 +134,143 @@ class TestBytecodeExecute: else: raise AssertionError("Expected Exception not raised") + def test_memory_limits_1(self): + # let string := 'banana' + # for (let i := 0; i < 100; i := i + 1) { + # string := string || string + # } + bytecode = [ + "_h", + 32, + "banana", + 33, + 0, + 33, + 100, + 36, + 1, + 15, + 40, + 18, + 36, + 0, + 36, + 0, + 2, + "concat", + 2, + 37, + 0, + 33, + 1, + 36, + 1, + 6, + 37, + 1, + 39, + -25, + 35, + 35, + ] + try: + execute_bytecode(bytecode, {}) + except Exception as e: + assert str(e) == "Memory limit of 67108864 bytes exceeded. Tried to allocate 75497504 bytes." + else: + raise AssertionError("Expected Exception not raised") + + def test_memory_limits_2(self): + # let string := 'banana' + # for (let i := 0; i < 100; i := i + 1) { + # string := string || string + # } + bytecode = [ + "_h", + 32, + "key", + 32, + "value", + 32, + "key2", + 32, + "value2", + 42, + 2, + 32, + "na", + 33, + 0, + 33, + 10000, + 36, + 2, + 15, + 40, + 52, + 33, + 16, + 36, + 2, + 15, + 40, + 9, + 36, + 1, + 36, + 1, + 2, + "concat", + 2, + 37, + 1, + 36, + 0, + 36, + 2, + 32, + "key_", + 2, + "concat", + 2, + 32, + "wasted", + 32, + " batman!", + 36, + 1, + 32, + "memory: ", + 2, + "concat", + 3, + 32, + "something", + 36, + 0, + 42, + 2, + 46, + 33, + 1, + 36, + 2, + 6, + 37, + 2, + 39, + -59, + 35, + 35, + 35, + ] + try: + execute_bytecode(bytecode, {}) + except Exception as e: + assert str(e) == "Memory limit of 67108864 bytes exceeded. Tried to allocate 67155164 bytes." + else: + raise AssertionError("Expected Exception not raised") + def test_functions(self): def stringify(*args): if args[0] == 1: diff --git a/hogvm/python/utils.py b/hogvm/python/utils.py index e48b4cb979a..5fd7e35b806 100644 --- a/hogvm/python/utils.py +++ b/hogvm/python/utils.py @@ -2,6 +2,9 @@ import re from typing import Any +COST_PER_UNIT = 8 + + class HogVMException(Exception): pass @@ -46,3 +49,24 @@ def set_nested_value(obj, chain, value) -> Any: raise HogVMException(f'Can not set property "{chain[-1]}" on object of type "{type(obj).__name__}"') return obj + + +def calculate_cost(object, marked: set | None = None) -> int: + if marked is None: + marked = set() + if isinstance(object, dict) or isinstance(object, list) or isinstance(object, tuple): + if id(object) in marked: + return COST_PER_UNIT + marked.add(id(object)) + try: + if isinstance(object, dict): + return COST_PER_UNIT + sum( + [calculate_cost(key, marked) + calculate_cost(value, marked) for key, value in object.items()] + ) + elif isinstance(object, list) or isinstance(object, tuple): + return COST_PER_UNIT + sum([calculate_cost(val, marked) for val in object]) + finally: + marked.remove(id(object)) + elif isinstance(object, str): + return COST_PER_UNIT + len(object) + return COST_PER_UNIT diff --git a/hogvm/test.sh b/hogvm/test.sh index dd767aecad4..99b3f0dda37 100755 --- a/hogvm/test.sh +++ b/hogvm/test.sh @@ -1,8 +1,14 @@ #!/bin/bash set -e +cd typescript +pnpm run build +cd .. cd .. +rm -f hogvm/__tests__/__snapshots__/*.stdout.nodejs +rm -f hogvm/__tests__/__snapshots__/*.stdout.python + for file in hogvm/__tests__/*.hog; do echo "Testing $file" @@ -21,7 +27,6 @@ for file in hogvm/__tests__/*.hog; do rm $basename.stdout.python else echo "Test failed" - rm $basename.stdout.nodejs $basename.stdout.python fi set -e done diff --git a/hogvm/typescript/package.json b/hogvm/typescript/package.json index 1ff026fb571..93037a855d9 100644 --- a/hogvm/typescript/package.json +++ b/hogvm/typescript/package.json @@ -1,6 +1,6 @@ { "name": "@posthog/hogvm", - "version": "1.0.18", + "version": "1.0.20", "description": "PostHog Hog Virtual Machine", "types": "dist/index.d.ts", "main": "dist/index.js", diff --git a/hogvm/typescript/src/__tests__/execute.test.ts b/hogvm/typescript/src/__tests__/execute.test.ts index 62cda07057c..ed836decc27 100644 --- a/hogvm/typescript/src/__tests__/execute.test.ts +++ b/hogvm/typescript/src/__tests__/execute.test.ts @@ -13,7 +13,7 @@ const tuple = (array: any[]): any[] => { return array } -describe('HogQL Bytecode', () => { +describe('hogvm execute', () => { test('execution results', async () => { const globals = { properties: { foo: 'bar', nullValue: null } } const options = { globals } @@ -133,6 +133,149 @@ describe('HogQL Bytecode', () => { expect(() => execSync(bytecode2)).toThrow('Too many arguments') }) + test('memory limits 1', async () => { + // let string := 'banana' + // for (let i := 0; i < 100; i := i + 1) { + // string := string || string + // } + const bytecode: any[] = [ + '_h', + 32, + 'banana', + 33, + 0, + 33, + 100, + 36, + 1, + 15, + 40, + 18, + 36, + 0, + 36, + 0, + 2, + 'concat', + 2, + 37, + 0, + 33, + 1, + 36, + 1, + 6, + 37, + 1, + 39, + -25, + 35, + 35, + ] + + await expect(execAsync(bytecode)).rejects.toThrow( + 'Memory limit of 67108864 bytes exceeded. Tried to allocate 75497504 bytes.' + ) + }) + + test('memory limits 2', async () => { + // // Printing recursive objects. + // let obj := {'key': 'value', 'key2': 'value2'} + // let str := 'na' + // for (let i := 0; i < 10000; i := i + 1) { + // if (i < 16) { + // str := str || str + // } + // obj[f'key_{i}'] := { + // 'wasted': 'memory: ' || str || ' batman!', + // 'something': obj, // something links to obj + // } + // } + const bytecode: any[] = [ + '_h', + 32, + 'key', + 32, + 'value', + 32, + 'key2', + 32, + 'value2', + 42, + 2, + 32, + 'na', + 33, + 0, + 33, + 10000, + 36, + 2, + 15, + 40, + 52, + 33, + 16, + 36, + 2, + 15, + 40, + 9, + 36, + 1, + 36, + 1, + 2, + 'concat', + 2, + 37, + 1, + 36, + 0, + 36, + 2, + 32, + 'key_', + 2, + 'concat', + 2, + 32, + 'wasted', + 32, + ' batman!', + 36, + 1, + 32, + 'memory: ', + 2, + 'concat', + 3, + 32, + 'something', + 36, + 0, + 42, + 2, + 46, + 33, + 1, + 36, + 2, + 6, + 37, + 2, + 39, + -59, + 35, + 35, + 35, + ] + + await expect(execAsync(bytecode)).rejects.toThrow( + 'Memory limit of 67108864 bytes exceeded. Tried to allocate 67155164 bytes.' + ) + }) + test('should execute user-defined stringify function correctly', async () => { const functions = { stringify: (arg: any) => { @@ -383,6 +526,7 @@ describe('HogQL Bytecode', () => { callStack: [], declaredFunctions: {}, ip: 8, + maxMemUsed: 16, ops: 3, stack: [4.2], syncDuration: 0, diff --git a/hogvm/typescript/src/__tests__/utils.test.ts b/hogvm/typescript/src/__tests__/utils.test.ts new file mode 100644 index 00000000000..b02f6dd16fd --- /dev/null +++ b/hogvm/typescript/src/__tests__/utils.test.ts @@ -0,0 +1,32 @@ +import { calculateCost } from '../utils' + +const PTR_COST = 8 + +describe('hogvm utils', () => { + test('calculateCost', async () => { + expect(calculateCost(1)).toBe(PTR_COST) + expect(calculateCost('hello')).toBe(PTR_COST + 5) + expect(calculateCost(true)).toBe(PTR_COST) + expect(calculateCost(null)).toBe(PTR_COST) + expect(calculateCost([])).toBe(PTR_COST) + expect(calculateCost([1])).toBe(PTR_COST * 2) + expect(calculateCost(['hello'])).toBe(PTR_COST * 2 + 5) + expect(calculateCost({})).toBe(PTR_COST) + expect(calculateCost({ key: 'value' })).toBe(PTR_COST * 3 + 3 + 5) + expect(calculateCost(new Map([['key', 'value']]))).toBe(PTR_COST * 3 + 3 + 5) + expect( + calculateCost( + new Map([ + ['key', 'value'], + ['key2', new Map([['key', 'value']])], + ]) + ) + ).toBe(PTR_COST * 7 + 3 + 5 + 4 + 3 + 5) + }) + + test('calculateCost with cycles', async () => { + const obj: Record = {} + obj['key'] = obj + expect(calculateCost(obj)).toBe(PTR_COST * 3 + 3) + }) +}) diff --git a/hogvm/typescript/src/execute.ts b/hogvm/typescript/src/execute.ts index c5398ff4864..d02da8ef31f 100644 --- a/hogvm/typescript/src/execute.ts +++ b/hogvm/typescript/src/execute.ts @@ -2,10 +2,12 @@ import RE2 from 're2' import { Operation } from './operation' import { ASYNC_STL, STL } from './stl/stl' -import { convertHogToJS, convertJSToHog, getNestedValue, like, setNestedValue } from './utils' +import { calculateCost, convertHogToJS, convertJSToHog, getNestedValue, like, setNestedValue } from './utils' const DEFAULT_MAX_ASYNC_STEPS = 100 +const DEFAULT_MAX_MEMORY = 64 * 1024 * 1024 // 64 MB const DEFAULT_TIMEOUT_MS = 5000 // ms +const MAX_FUNCTION_ARGS_LENGTH = 300 export interface VMState { /** Bytecode running in the VM */ @@ -24,6 +26,8 @@ export interface VMState { asyncSteps: number /** Combined duration of sync steps */ syncDuration: number + /** Max memory used */ + maxMemUsed: number } export interface ExecOptions { @@ -35,6 +39,8 @@ export interface ExecOptions { timeout?: number /** Max number of async function that can happen. When reached the function will throw */ maxAsyncSteps?: number + /** Memory limit in bytes. This is calculated based on the size of the VM stack. */ + memoryLimit?: number } export interface ExecResult { @@ -45,9 +51,6 @@ export interface ExecResult { state?: VMState } -/** Maximum function arguments allowed */ -const MAX_ARGS_LENGTH = 300 - export function execSync(bytecode: any[], options?: ExecOptions): any { const response = exec(bytecode, options) if (response.finished) { @@ -109,8 +112,12 @@ export function exec(code: any[] | VMState, options?: ExecOptions): ExecResult { const asyncSteps = vmState ? vmState.asyncSteps : 0 const syncDuration = vmState ? vmState.syncDuration : 0 const stack: any[] = vmState ? vmState.stack : [] + const memStack: number[] = stack.map((s) => calculateCost(s)) const callStack: [number, number, number][] = vmState ? vmState.callStack : [] const declaredFunctions: Record = vmState ? vmState.declaredFunctions : {} + let memUsed = memStack.reduce((acc, val) => acc + val, 0) + let maxMemUsed = Math.max(vmState ? vmState.maxMemUsed : 0, memUsed) + const memLimit = options?.memoryLimit ?? DEFAULT_MAX_MEMORY let ip = vmState ? vmState.ip : 1 let ops = vmState ? vmState.ops : 0 const timeout = options?.timeout ?? DEFAULT_TIMEOUT_MS @@ -120,15 +127,36 @@ export function exec(code: any[] | VMState, options?: ExecOptions): ExecResult { if (stack.length === 0) { throw new Error('Invalid HogQL bytecode, stack is empty') } + memUsed -= memStack.pop() ?? 0 return stack.pop() } + function pushStack(value: any): any { + memStack.push(calculateCost(value)) + memUsed += memStack[memStack.length - 1] + maxMemUsed = Math.max(maxMemUsed, memUsed) + if (memUsed > memLimit && memLimit > 0) { + throw new Error(`Memory limit of ${memLimit} bytes exceeded. Tried to allocate ${memUsed} bytes.`) + } + return stack.push(value) + } + + function spliceStack2(start: number, deleteCount?: number): any[] { + memUsed -= memStack.splice(start, deleteCount).reduce((acc, val) => acc + val, 0) + return stack.splice(start, deleteCount) + } + function spliceStack1(start: number): any[] { + memUsed -= memStack.splice(start).reduce((acc, val) => acc + val, 0) + return stack.splice(start) + } + function next(): any { if (ip >= bytecode!.length - 1) { throw new Error('Unexpected end of bytecode') } return bytecode![++ip] } + function checkTimeout(): void { if (syncDuration + Date.now() - startTime > timeout) { throw new Error(`Execution timed out after ${timeout / 1000} seconds. Performed ${ops} ops.`) @@ -144,110 +172,110 @@ export function exec(code: any[] | VMState, options?: ExecOptions): ExecResult { case null: break case Operation.STRING: - stack.push(next()) + pushStack(next()) break case Operation.FLOAT: - stack.push(next()) + pushStack(next()) break case Operation.INTEGER: - stack.push(next()) + pushStack(next()) break case Operation.TRUE: - stack.push(true) + pushStack(true) break case Operation.FALSE: - stack.push(false) + pushStack(false) break case Operation.NULL: - stack.push(null) + pushStack(null) break case Operation.NOT: - stack.push(!popStack()) + pushStack(!popStack()) break case Operation.AND: - stack.push( - Array(next()) - .fill(null) - .map(() => popStack()) - .every(Boolean) - ) + temp = next() + temp2 = true + for (let i = 0; i < temp; i++) { + temp2 = !!popStack() && temp2 + } + pushStack(temp2) break case Operation.OR: - stack.push( - Array(next()) - .fill(null) - .map(() => popStack()) - .some(Boolean) - ) + temp = next() + temp2 = false + for (let i = 0; i < temp; i++) { + temp2 = !!popStack() || temp2 + } + pushStack(temp2) break case Operation.PLUS: - stack.push(Number(popStack()) + Number(popStack())) + pushStack(Number(popStack()) + Number(popStack())) break case Operation.MINUS: - stack.push(Number(popStack()) - Number(popStack())) + pushStack(Number(popStack()) - Number(popStack())) break case Operation.DIVIDE: - stack.push(Number(popStack()) / Number(popStack())) + pushStack(Number(popStack()) / Number(popStack())) break case Operation.MULTIPLY: - stack.push(Number(popStack()) * Number(popStack())) + pushStack(Number(popStack()) * Number(popStack())) break case Operation.MOD: - stack.push(Number(popStack()) % Number(popStack())) + pushStack(Number(popStack()) % Number(popStack())) break case Operation.EQ: - stack.push(popStack() === popStack()) + pushStack(popStack() === popStack()) break case Operation.NOT_EQ: - stack.push(popStack() !== popStack()) + pushStack(popStack() !== popStack()) break case Operation.GT: - stack.push(popStack() > popStack()) + pushStack(popStack() > popStack()) break case Operation.GT_EQ: - stack.push(popStack() >= popStack()) + pushStack(popStack() >= popStack()) break case Operation.LT: - stack.push(popStack() < popStack()) + pushStack(popStack() < popStack()) break case Operation.LT_EQ: - stack.push(popStack() <= popStack()) + pushStack(popStack() <= popStack()) break case Operation.LIKE: - stack.push(like(popStack(), popStack())) + pushStack(like(popStack(), popStack())) break case Operation.ILIKE: - stack.push(like(popStack(), popStack(), true)) + pushStack(like(popStack(), popStack(), true)) break case Operation.NOT_LIKE: - stack.push(!like(popStack(), popStack())) + pushStack(!like(popStack(), popStack())) break case Operation.NOT_ILIKE: - stack.push(!like(popStack(), popStack(), true)) + pushStack(!like(popStack(), popStack(), true)) break case Operation.IN: temp = popStack() - stack.push(popStack().includes(temp)) + pushStack(popStack().includes(temp)) break case Operation.NOT_IN: temp = popStack() - stack.push(!popStack().includes(temp)) + pushStack(!popStack().includes(temp)) break case Operation.REGEX: temp = popStack() - stack.push(new RE2(popStack()).test(temp)) + pushStack(new RE2(popStack()).test(temp)) break case Operation.NOT_REGEX: temp = popStack() - stack.push(!new RE2(popStack()).test(temp)) + pushStack(!new RE2(popStack()).test(temp)) break case Operation.IREGEX: temp = popStack() - stack.push(new RE2(popStack(), 'i').test(temp)) + pushStack(new RE2(popStack(), 'i').test(temp)) break case Operation.NOT_IREGEX: temp = popStack() - stack.push(!new RE2(popStack(), 'i').test(temp)) + pushStack(!new RE2(popStack(), 'i').test(temp)) break case Operation.GET_GLOBAL: { const count = next() @@ -255,7 +283,7 @@ export function exec(code: any[] | VMState, options?: ExecOptions): ExecResult { for (let i = 0; i < count; i++) { chain.push(popStack()) } - stack.push(options?.globals ? convertJSToHog(getNestedValue(options.globals, chain)) : null) + pushStack(options?.globals ? convertJSToHog(getNestedValue(options.globals, chain)) : null) break } case Operation.POP: @@ -265,8 +293,8 @@ export function exec(code: any[] | VMState, options?: ExecOptions): ExecResult { if (callStack.length > 0) { const [newIp, stackStart, _] = callStack.pop()! const response = popStack() - stack.splice(stackStart) - stack.push(response) + spliceStack1(stackStart) + pushStack(response) ip = newIp break } else { @@ -277,19 +305,23 @@ export function exec(code: any[] | VMState, options?: ExecOptions): ExecResult { } case Operation.GET_LOCAL: temp = callStack.length > 0 ? callStack[callStack.length - 1][1] : 0 - stack.push(stack[next() + temp]) + pushStack(stack[next() + temp]) break case Operation.SET_LOCAL: - temp = callStack.length > 0 ? callStack[callStack.length - 1][1] : 0 - stack[next() + temp] = popStack() + temp = (callStack.length > 0 ? callStack[callStack.length - 1][1] : 0) + next() + stack[temp] = popStack() + temp2 = memStack[temp] + memStack[temp] = calculateCost(stack[temp]) + memUsed += memStack[temp] - temp2 + maxMemUsed = Math.max(maxMemUsed, memUsed) break case Operation.GET_PROPERTY: temp = popStack() // property - stack.push(getNestedValue(popStack(), [temp])) + pushStack(getNestedValue(popStack(), [temp])) break case Operation.GET_PROPERTY_NULLISH: temp = popStack() // property - stack.push(getNestedValue(popStack(), [temp], true)) + pushStack(getNestedValue(popStack(), [temp], true)) break case Operation.SET_PROPERTY: temp = popStack() // value @@ -298,23 +330,23 @@ export function exec(code: any[] | VMState, options?: ExecOptions): ExecResult { break case Operation.DICT: temp = next() * 2 // number of elements to remove from the stack - tempArray = stack.splice(stack.length - temp, temp) + tempArray = spliceStack2(stack.length - temp, temp) tempMap = new Map() for (let i = 0; i < tempArray.length; i += 2) { tempMap.set(tempArray[i], tempArray[i + 1]) } - stack.push(tempMap) + pushStack(tempMap) break case Operation.ARRAY: temp = next() - tempArray = stack.splice(stack.length - temp, temp) - stack.push(tempArray) + tempArray = spliceStack2(stack.length - temp, temp) + pushStack(tempArray) break case Operation.TUPLE: temp = next() - tempArray = stack.splice(stack.length - temp, temp) + tempArray = spliceStack2(stack.length - temp, temp) ;(tempArray as any).__isHogTuple = true - stack.push(tempArray) + pushStack(tempArray) break case Operation.JUMP: temp = next() @@ -353,14 +385,14 @@ export function exec(code: any[] | VMState, options?: ExecOptions): ExecResult { if (temp > stack.length) { throw new Error('Not enough arguments on the stack') } - if (temp > MAX_ARGS_LENGTH) { + if (temp > MAX_FUNCTION_ARGS_LENGTH) { throw new Error('Too many arguments') } const args = Array(temp) .fill(null) .map(() => popStack()) if (options?.functions && options.functions.hasOwnProperty(name) && options.functions[name]) { - stack.push(convertJSToHog(options.functions[name](...args.map(convertHogToJS)))) + pushStack(convertJSToHog(options.functions[name](...args.map(convertHogToJS)))) } else if ( name !== 'toString' && ((options?.asyncFunctions && @@ -386,10 +418,11 @@ export function exec(code: any[] | VMState, options?: ExecOptions): ExecResult { ops, asyncSteps: asyncSteps + 1, syncDuration: syncDuration + (Date.now() - startTime), + maxMemUsed, }, } satisfies ExecResult } else if (name in STL) { - stack.push(STL[name](args, name, timeout)) + pushStack(STL[name](args, name, timeout)) } else { throw new Error(`Unsupported function call: ${name}`) } diff --git a/hogvm/typescript/src/stl/print.ts b/hogvm/typescript/src/stl/print.ts index b70e554a725..a22552f7cc8 100644 --- a/hogvm/typescript/src/stl/print.ts +++ b/hogvm/typescript/src/stl/print.ts @@ -39,34 +39,41 @@ export function escapeIdentifier(identifier: string | number): string { .join('')}\`` } -export function printHogValue(obj: any): string { - if (Array.isArray(obj)) { - if ((obj as any).__isHogTuple) { - if (obj.length < 2) { - return `tuple(${obj.map(printHogValue).join(', ')})` - } - return `(${obj.map(printHogValue).join(', ')})` - } else { - return `[${obj.map(printHogValue).join(', ')}]` - } - } - if (obj instanceof Map) { - return `{${Array.from(obj.entries()) - .map(([key, value]) => `${printHogValue(key)}: ${printHogValue(value)}`) - .join(', ')}}` +export function printHogValue(obj: any, marked: Set | undefined = undefined): string { + if (!marked) { + marked = new Set() } if (typeof obj === 'object' && obj !== null) { - return `{${Object.entries(obj) - .map(([key, value]) => `${printHogValue(key)}: ${printHogValue(value)}`) - .join(', ')}}` - } - if (typeof obj === 'boolean') { + if (marked.has(obj)) { + return 'null' + } + marked.add(obj) + try { + if (Array.isArray(obj)) { + if ((obj as any).__isHogTuple) { + if (obj.length < 2) { + return `tuple(${obj.map((o) => printHogValue(o, marked)).join(', ')})` + } + return `(${obj.map((o) => printHogValue(o, marked)).join(', ')})` + } + return `[${obj.map((o) => printHogValue(o, marked)).join(', ')}]` + } + if (obj instanceof Map) { + return `{${Array.from(obj.entries()) + .map(([key, value]) => `${printHogValue(key, marked)}: ${printHogValue(value, marked)}`) + .join(', ')}}` + } + return `{${Object.entries(obj) + .map(([key, value]) => `${printHogValue(key, marked)}: ${printHogValue(value, marked)}`) + .join(', ')}}` + } finally { + marked.delete(obj) + } + } else if (typeof obj === 'boolean') { return obj ? 'true' : 'false' - } - if (obj === null) { + } else if (obj === null) { return 'null' - } - if (typeof obj === 'string') { + } else if (typeof obj === 'string') { return escapeString(obj) } return obj.toString() diff --git a/hogvm/typescript/src/stl/stl.ts b/hogvm/typescript/src/stl/stl.ts index 9f915d85304..3e58deb0f7a 100644 --- a/hogvm/typescript/src/stl/stl.ts +++ b/hogvm/typescript/src/stl/stl.ts @@ -80,21 +80,35 @@ export const STL: Record }, jsonStringify: (args) => { // Recursively convert maps to objects - function convert(x: any): any { - if (x instanceof Map) { - const obj: Record = {} - x.forEach((value, key) => { - obj[key] = convert(value) - }) - return obj - } else if (typeof x === 'object' && Array.isArray(x)) { - return x.map(convert) - } else if (typeof x === 'object' && x !== null) { - const obj: Record = {} - for (const key in x) { - obj[key] = convert(x[key]) + function convert(x: any, marked?: Set): any { + if (!marked) { + marked = new Set() + } + if (typeof x === 'object' && x !== null) { + if (marked.has(x)) { + return null + } + marked.add(x) + try { + if (x instanceof Map) { + const obj: Record = {} + x.forEach((value, key) => { + obj[convert(key, marked)] = convert(value, marked) + }) + return obj + } + if (typeof x === 'object' && Array.isArray(x)) { + return x.map((v) => convert(v, marked)) + } + + const obj: Record = {} + for (const key in x) { + obj[key] = convert(x[key], marked) + } + return obj + } finally { + marked.delete(x) } - return obj } return x } diff --git a/hogvm/typescript/src/utils.ts b/hogvm/typescript/src/utils.ts index 1a0814d67c0..9e526245314 100644 --- a/hogvm/typescript/src/utils.ts +++ b/hogvm/typescript/src/utils.ts @@ -1,3 +1,6 @@ +/** Fixed cost per object in memory */ +const COST_PER_UNIT = 8 + export function like(string: string, pattern: string, caseInsensitive = false): boolean { pattern = String(pattern) .replaceAll(/[-/\\^$*+?.()|[\]{}]/g, '\\$&') @@ -79,3 +82,40 @@ export function convertHogToJS(x: any): any { } return x } + +export function calculateCost(object: any, marked: Set | undefined = undefined): any { + if (!marked) { + marked = new Set() + } + if (typeof object === 'object' && object !== null) { + if (marked.has(object)) { + return COST_PER_UNIT + } + marked.add(object) + try { + if (object instanceof Map) { + return ( + COST_PER_UNIT + + Array.from(object.keys()).reduce( + (acc, key) => acc + calculateCost(key, marked) + calculateCost(object.get(key), marked), + 0 + ) + ) + } else if (Array.isArray(object)) { + return COST_PER_UNIT + object.reduce((acc, val) => acc + calculateCost(val, marked), 0) + } + return ( + COST_PER_UNIT + + Object.keys(object).reduce( + (acc, key) => acc + calculateCost(key, marked) + calculateCost(object[key], marked), + 0 + ) + ) + } finally { + marked.delete(object) + } + } else if (typeof object === 'string') { + return COST_PER_UNIT + object.length + } + return COST_PER_UNIT +} diff --git a/plugin-server/package.json b/plugin-server/package.json index 2b535cdaaf2..e7e286c0008 100644 --- a/plugin-server/package.json +++ b/plugin-server/package.json @@ -50,7 +50,7 @@ "@google-cloud/storage": "^5.8.5", "@maxmind/geoip2-node": "^3.4.0", "@posthog/clickhouse": "^1.7.0", - "@posthog/hogvm": "^1.0.18", + "@posthog/hogvm": "^1.0.20", "@posthog/plugin-scaffold": "1.4.4", "@sentry/node": "^7.49.0", "@sentry/profiling-node": "^0.3.0", diff --git a/plugin-server/pnpm-lock.yaml b/plugin-server/pnpm-lock.yaml index 466733d3071..96a760b6055 100644 --- a/plugin-server/pnpm-lock.yaml +++ b/plugin-server/pnpm-lock.yaml @@ -44,8 +44,8 @@ dependencies: specifier: ^1.7.0 version: 1.7.0 '@posthog/hogvm': - specifier: ^1.0.18 - version: 1.0.18(re2@1.20.3) + specifier: ^1.0.20 + version: 1.0.20(re2@1.20.3) '@posthog/plugin-scaffold': specifier: 1.4.4 version: 1.4.4 @@ -3110,8 +3110,8 @@ packages: engines: {node: '>=12'} dev: false - /@posthog/hogvm@1.0.18(re2@1.20.3): - resolution: {integrity: sha512-h0C9AlpfDRYlSzmXFOZKXR5x5UD+sgXkiQ6CUCoBQX0TjxMYE7hU1lZ7cgWQTYWJwSHeLp5RTUN0BjRDEFhj/Q==} + /@posthog/hogvm@1.0.20(re2@1.20.3): + resolution: {integrity: sha512-NSy4EbjR0SyNCkHHA2wU7psI9oVQtEkxl1Tr8NBYxa2QBnmAS+yWcST0MhFIX38GrkYVWTsgrZUeSnEQTsvMxg==} peerDependencies: re2: ^1.21.3 dependencies: