From 06e993818d014eded0d4168ffe1080f0fce93525 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Tue, 7 Nov 2023 21:56:59 +0000 Subject: [PATCH] feat(apps): transpile via django (#18201) --- .dockerignore | 2 + .github/actions/run-backend-tests/action.yml | 18 + .github/workflows/ci-backend.yml | 1 + .gitignore | 1 + plugin-transpiler/README.md | 3 + plugin-transpiler/build.mjs | 13 + plugin-transpiler/package.json | 21 ++ plugin-transpiler/pnpm-lock.yaml | 340 +++++++++++++++++++ plugin-transpiler/src/index.ts | 46 +++ plugin-transpiler/src/presets.ts | 25 ++ plugin-transpiler/tsconfig.json | 20 ++ posthog/api/plugin.py | 64 +++- posthog/api/test/test_plugin.py | 150 ++++---- production.Dockerfile | 18 +- unit.json | 16 +- 15 files changed, 659 insertions(+), 79 deletions(-) create mode 100644 plugin-transpiler/README.md create mode 100755 plugin-transpiler/build.mjs create mode 100644 plugin-transpiler/package.json create mode 100644 plugin-transpiler/pnpm-lock.yaml create mode 100644 plugin-transpiler/src/index.ts create mode 100644 plugin-transpiler/src/presets.ts create mode 100644 plugin-transpiler/tsconfig.json diff --git a/.dockerignore b/.dockerignore index 79a794a0401..14f0bba5c33 100644 --- a/.dockerignore +++ b/.dockerignore @@ -34,3 +34,5 @@ !share/GeoLite2-City.mmdb !hogvm/python !unit.json +!plugin-transpiler/src +!plugin-transpiler/*.* diff --git a/.github/actions/run-backend-tests/action.yml b/.github/actions/run-backend-tests/action.yml index 9109dfc4eb9..edd36992a66 100644 --- a/.github/actions/run-backend-tests/action.yml +++ b/.github/actions/run-backend-tests/action.yml @@ -62,6 +62,24 @@ runs: run: | sudo apt-get update && sudo apt-get install libxml2-dev libxmlsec1-dev libxmlsec1-openssl + - name: Install pnpm + uses: pnpm/action-setup@v2 + with: + version: 8.x.x + + - name: Set up Node.js + uses: actions/setup-node@v3 + with: + node-version: 18 + cache: pnpm + + - name: Install plugin-transpiler + shell: bash + run: | + cd plugin-transpiler + pnpm install + pnpm run build + - uses: syphar/restore-virtualenv@v1 id: cache-backend-tests with: diff --git a/.github/workflows/ci-backend.yml b/.github/workflows/ci-backend.yml index 845c4a60a46..f80300fca0a 100644 --- a/.github/workflows/ci-backend.yml +++ b/.github/workflows/ci-backend.yml @@ -68,6 +68,7 @@ jobs: - mypy.ini - pytest.ini - frontend/src/queries/schema.json # Used for generating schema.py + - plugin-transpiler/src # Used for transpiling plugins # Make sure we run if someone is explicitly change the workflow - .github/workflows/ci-backend.yml - .github/actions/run-backend-tests/action.yml diff --git a/.gitignore b/.gitignore index 8f32f6b56d3..b3ee2cc9213 100644 --- a/.gitignore +++ b/.gitignore @@ -55,3 +55,4 @@ gen/ upgrade/ hogvm/typescript/dist .wokeignore +plugin-transpiler/dist diff --git a/plugin-transpiler/README.md b/plugin-transpiler/README.md new file mode 100644 index 00000000000..eab9d78b4c4 --- /dev/null +++ b/plugin-transpiler/README.md @@ -0,0 +1,3 @@ +## Plugin Transpiler + +This project transpiles frontend plugins and site apps. diff --git a/plugin-transpiler/build.mjs b/plugin-transpiler/build.mjs new file mode 100755 index 00000000000..c748c3bddc7 --- /dev/null +++ b/plugin-transpiler/build.mjs @@ -0,0 +1,13 @@ +#!/usr/bin/env node +import * as esbuild from 'esbuild' +;(async function build() { + let result = await esbuild.build({ + entryPoints: ['src/index.ts'], + bundle: true, + outdir: 'dist', + }) + if (!result.errors.length) { + // eslint-disable-next-line no-console + console.log('Build succeeded') + } +})() diff --git a/plugin-transpiler/package.json b/plugin-transpiler/package.json new file mode 100644 index 00000000000..a36609f09fc --- /dev/null +++ b/plugin-transpiler/package.json @@ -0,0 +1,21 @@ +{ + "name": "plugin-transpiler", + "version": "1.0.0", + "description": "Transpiles site apps TSX to browser JS via stdin/stdout", + "main": "transpile.mjs", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1", + "build": "tsc -b && node build.mjs", + "start:dist": "node dist/index.js", + "start": "npm run build && npm run start:dist" + }, + "author": "PostHog Inc.", + "license": "MIT", + "devDependencies": { + "@babel/standalone": "^7.23.2", + "@types/babel__standalone": "^7.1.6", + "@types/node": "^20.8.9", + "esbuild": "^0.19.5", + "typescript": "^5.2.2" + } +} diff --git a/plugin-transpiler/pnpm-lock.yaml b/plugin-transpiler/pnpm-lock.yaml new file mode 100644 index 00000000000..4363b1cc475 --- /dev/null +++ b/plugin-transpiler/pnpm-lock.yaml @@ -0,0 +1,340 @@ +lockfileVersion: '6.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +devDependencies: + '@babel/standalone': + specifier: ^7.23.2 + version: 7.23.2 + '@types/babel__standalone': + specifier: ^7.1.6 + version: 7.1.6 + '@types/node': + specifier: ^20.8.9 + version: 20.8.9 + esbuild: + specifier: ^0.19.5 + version: 0.19.5 + typescript: + specifier: ^5.2.2 + version: 5.2.2 + +packages: + + /@babel/helper-string-parser@7.22.5: + resolution: {integrity: sha512-mM4COjgZox8U+JcXQwPijIZLElkgEpO5rsERVDJTc2qfCDfERyob6k5WegS14SX18IIjv+XD+GrqNumY5JRCDw==} + engines: {node: '>=6.9.0'} + dev: true + + /@babel/helper-validator-identifier@7.22.20: + resolution: {integrity: sha512-Y4OZ+ytlatR8AI+8KZfKuL5urKp7qey08ha31L8b3BwewJAoJamTzyvxPR/5D+KkdJCGPq/+8TukHBlY10FX9A==} + engines: {node: '>=6.9.0'} + dev: true + + /@babel/parser@7.23.0: + resolution: {integrity: sha512-vvPKKdMemU85V9WE/l5wZEmImpCtLqbnTvqDS2U1fJ96KrxoW7KrXhNsNCblQlg8Ck4b85yxdTyelsMUgFUXiw==} + engines: {node: '>=6.0.0'} + hasBin: true + dependencies: + '@babel/types': 7.23.0 + dev: true + + /@babel/standalone@7.23.2: + resolution: {integrity: sha512-VJNw7OS26JvB6rE9XpbT6uQeQIEBWU5eeHGS4VR/+/4ZoKdLBXLcy66ZVJ/9IBkK1RMp8B0cohvhzdKWtJAGmg==} + engines: {node: '>=6.9.0'} + dev: true + + /@babel/types@7.23.0: + resolution: {integrity: sha512-0oIyUfKoI3mSqMvsxBdclDwxXKXAUA8v/apZbc+iSyARYou1o8ZGDxbUYyLFoW2arqS2jDGqJuZvv1d/io1axg==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/helper-string-parser': 7.22.5 + '@babel/helper-validator-identifier': 7.22.20 + to-fast-properties: 2.0.0 + dev: true + + /@esbuild/android-arm64@0.19.5: + resolution: {integrity: sha512-5d1OkoJxnYQfmC+Zd8NBFjkhyCNYwM4n9ODrycTFY6Jk1IGiZ+tjVJDDSwDt77nK+tfpGP4T50iMtVi4dEGzhQ==} + engines: {node: '>=12'} + cpu: [arm64] + os: [android] + requiresBuild: true + dev: true + optional: true + + /@esbuild/android-arm@0.19.5: + resolution: {integrity: sha512-bhvbzWFF3CwMs5tbjf3ObfGqbl/17ict2/uwOSfr3wmxDE6VdS2GqY/FuzIPe0q0bdhj65zQsvqfArI9MY6+AA==} + engines: {node: '>=12'} + cpu: [arm] + os: [android] + requiresBuild: true + dev: true + optional: true + + /@esbuild/android-x64@0.19.5: + resolution: {integrity: sha512-9t+28jHGL7uBdkBjL90QFxe7DVA+KGqWlHCF8ChTKyaKO//VLuoBricQCgwhOjA1/qOczsw843Fy4cbs4H3DVA==} + engines: {node: '>=12'} + cpu: [x64] + os: [android] + requiresBuild: true + dev: true + optional: true + + /@esbuild/darwin-arm64@0.19.5: + resolution: {integrity: sha512-mvXGcKqqIqyKoxq26qEDPHJuBYUA5KizJncKOAf9eJQez+L9O+KfvNFu6nl7SCZ/gFb2QPaRqqmG0doSWlgkqw==} + engines: {node: '>=12'} + cpu: [arm64] + os: [darwin] + requiresBuild: true + dev: true + optional: true + + /@esbuild/darwin-x64@0.19.5: + resolution: {integrity: sha512-Ly8cn6fGLNet19s0X4unjcniX24I0RqjPv+kurpXabZYSXGM4Pwpmf85WHJN3lAgB8GSth7s5A0r856S+4DyiA==} + engines: {node: '>=12'} + cpu: [x64] + os: [darwin] + requiresBuild: true + dev: true + optional: true + + /@esbuild/freebsd-arm64@0.19.5: + resolution: {integrity: sha512-GGDNnPWTmWE+DMchq1W8Sd0mUkL+APvJg3b11klSGUDvRXh70JqLAO56tubmq1s2cgpVCSKYywEiKBfju8JztQ==} + engines: {node: '>=12'} + cpu: [arm64] + os: [freebsd] + requiresBuild: true + dev: true + optional: true + + /@esbuild/freebsd-x64@0.19.5: + resolution: {integrity: sha512-1CCwDHnSSoA0HNwdfoNY0jLfJpd7ygaLAp5EHFos3VWJCRX9DMwWODf96s9TSse39Br7oOTLryRVmBoFwXbuuQ==} + engines: {node: '>=12'} + cpu: [x64] + os: [freebsd] + requiresBuild: true + dev: true + optional: true + + /@esbuild/linux-arm64@0.19.5: + resolution: {integrity: sha512-o3vYippBmSrjjQUCEEiTZ2l+4yC0pVJD/Dl57WfPwwlvFkrxoSO7rmBZFii6kQB3Wrn/6GwJUPLU5t52eq2meA==} + engines: {node: '>=12'} + cpu: [arm64] + os: [linux] + requiresBuild: true + dev: true + optional: true + + /@esbuild/linux-arm@0.19.5: + resolution: {integrity: sha512-lrWXLY/vJBzCPC51QN0HM71uWgIEpGSjSZZADQhq7DKhPcI6NH1IdzjfHkDQws2oNpJKpR13kv7/pFHBbDQDwQ==} + engines: {node: '>=12'} + cpu: [arm] + os: [linux] + requiresBuild: true + dev: true + optional: true + + /@esbuild/linux-ia32@0.19.5: + resolution: {integrity: sha512-MkjHXS03AXAkNp1KKkhSKPOCYztRtK+KXDNkBa6P78F8Bw0ynknCSClO/ztGszILZtyO/lVKpa7MolbBZ6oJtQ==} + engines: {node: '>=12'} + cpu: [ia32] + os: [linux] + requiresBuild: true + dev: true + optional: true + + /@esbuild/linux-loong64@0.19.5: + resolution: {integrity: sha512-42GwZMm5oYOD/JHqHska3Jg0r+XFb/fdZRX+WjADm3nLWLcIsN27YKtqxzQmGNJgu0AyXg4HtcSK9HuOk3v1Dw==} + engines: {node: '>=12'} + cpu: [loong64] + os: [linux] + requiresBuild: true + dev: true + optional: true + + /@esbuild/linux-mips64el@0.19.5: + resolution: {integrity: sha512-kcjndCSMitUuPJobWCnwQ9lLjiLZUR3QLQmlgaBfMX23UEa7ZOrtufnRds+6WZtIS9HdTXqND4yH8NLoVVIkcg==} + engines: {node: '>=12'} + cpu: [mips64el] + os: [linux] + requiresBuild: true + dev: true + optional: true + + /@esbuild/linux-ppc64@0.19.5: + resolution: {integrity: sha512-yJAxJfHVm0ZbsiljbtFFP1BQKLc8kUF6+17tjQ78QjqjAQDnhULWiTA6u0FCDmYT1oOKS9PzZ2z0aBI+Mcyj7Q==} + engines: {node: '>=12'} + cpu: [ppc64] + os: [linux] + requiresBuild: true + dev: true + optional: true + + /@esbuild/linux-riscv64@0.19.5: + resolution: {integrity: sha512-5u8cIR/t3gaD6ad3wNt1MNRstAZO+aNyBxu2We8X31bA8XUNyamTVQwLDA1SLoPCUehNCymhBhK3Qim1433Zag==} + engines: {node: '>=12'} + cpu: [riscv64] + os: [linux] + requiresBuild: true + dev: true + optional: true + + /@esbuild/linux-s390x@0.19.5: + resolution: {integrity: sha512-Z6JrMyEw/EmZBD/OFEFpb+gao9xJ59ATsoTNlj39jVBbXqoZm4Xntu6wVmGPB/OATi1uk/DB+yeDPv2E8PqZGw==} + engines: {node: '>=12'} + cpu: [s390x] + os: [linux] + requiresBuild: true + dev: true + optional: true + + /@esbuild/linux-x64@0.19.5: + resolution: {integrity: sha512-psagl+2RlK1z8zWZOmVdImisMtrUxvwereIdyJTmtmHahJTKb64pAcqoPlx6CewPdvGvUKe2Jw+0Z/0qhSbG1A==} + engines: {node: '>=12'} + cpu: [x64] + os: [linux] + requiresBuild: true + dev: true + optional: true + + /@esbuild/netbsd-x64@0.19.5: + resolution: {integrity: sha512-kL2l+xScnAy/E/3119OggX8SrWyBEcqAh8aOY1gr4gPvw76la2GlD4Ymf832UCVbmuWeTf2adkZDK+h0Z/fB4g==} + engines: {node: '>=12'} + cpu: [x64] + os: [netbsd] + requiresBuild: true + dev: true + optional: true + + /@esbuild/openbsd-x64@0.19.5: + resolution: {integrity: sha512-sPOfhtzFufQfTBgRnE1DIJjzsXukKSvZxloZbkJDG383q0awVAq600pc1nfqBcl0ice/WN9p4qLc39WhBShRTA==} + engines: {node: '>=12'} + cpu: [x64] + os: [openbsd] + requiresBuild: true + dev: true + optional: true + + /@esbuild/sunos-x64@0.19.5: + resolution: {integrity: sha512-dGZkBXaafuKLpDSjKcB0ax0FL36YXCvJNnztjKV+6CO82tTYVDSH2lifitJ29jxRMoUhgkg9a+VA/B03WK5lcg==} + engines: {node: '>=12'} + cpu: [x64] + os: [sunos] + requiresBuild: true + dev: true + optional: true + + /@esbuild/win32-arm64@0.19.5: + resolution: {integrity: sha512-dWVjD9y03ilhdRQ6Xig1NWNgfLtf2o/STKTS+eZuF90fI2BhbwD6WlaiCGKptlqXlURVB5AUOxUj09LuwKGDTg==} + engines: {node: '>=12'} + cpu: [arm64] + os: [win32] + requiresBuild: true + dev: true + optional: true + + /@esbuild/win32-ia32@0.19.5: + resolution: {integrity: sha512-4liggWIA4oDgUxqpZwrDhmEfAH4d0iljanDOK7AnVU89T6CzHon/ony8C5LeOdfgx60x5cnQJFZwEydVlYx4iw==} + engines: {node: '>=12'} + cpu: [ia32] + os: [win32] + requiresBuild: true + dev: true + optional: true + + /@esbuild/win32-x64@0.19.5: + resolution: {integrity: sha512-czTrygUsB/jlM8qEW5MD8bgYU2Xg14lo6kBDXW6HdxKjh8M5PzETGiSHaz9MtbXBYDloHNUAUW2tMiKW4KM9Mw==} + engines: {node: '>=12'} + cpu: [x64] + os: [win32] + requiresBuild: true + dev: true + optional: true + + /@types/babel__core@7.20.3: + resolution: {integrity: sha512-54fjTSeSHwfan8AyHWrKbfBWiEUrNTZsUwPTDSNaaP1QDQIZbeNUg3a59E9D+375MzUw/x1vx2/0F5LBz+AeYA==} + dependencies: + '@babel/parser': 7.23.0 + '@babel/types': 7.23.0 + '@types/babel__generator': 7.6.6 + '@types/babel__template': 7.4.3 + '@types/babel__traverse': 7.20.3 + dev: true + + /@types/babel__generator@7.6.6: + resolution: {integrity: sha512-66BXMKb/sUWbMdBNdMvajU7i/44RkrA3z/Yt1c7R5xejt8qh84iU54yUWCtm0QwGJlDcf/gg4zd/x4mpLAlb/w==} + dependencies: + '@babel/types': 7.23.0 + dev: true + + /@types/babel__standalone@7.1.6: + resolution: {integrity: sha512-JRVq2H02irW6GDxkHepsdZVjor87SZo4eDsxoBtSbswf7HWtVYNxCmJDBkJbFNFWGOuzsxATZV8xZ2PI9sBiIw==} + dependencies: + '@types/babel__core': 7.20.3 + dev: true + + /@types/babel__template@7.4.3: + resolution: {integrity: sha512-ciwyCLeuRfxboZ4isgdNZi/tkt06m8Tw6uGbBSBgWrnnZGNXiEyM27xc/PjXGQLqlZ6ylbgHMnm7ccF9tCkOeQ==} + dependencies: + '@babel/parser': 7.23.0 + '@babel/types': 7.23.0 + dev: true + + /@types/babel__traverse@7.20.3: + resolution: {integrity: sha512-Lsh766rGEFbaxMIDH7Qa+Yha8cMVI3qAK6CHt3OR0YfxOIn5Z54iHiyDRycHrBqeIiqGa20Kpsv1cavfBKkRSw==} + dependencies: + '@babel/types': 7.23.0 + dev: true + + /@types/node@20.8.9: + resolution: {integrity: sha512-UzykFsT3FhHb1h7yD4CA4YhBHq545JC0YnEz41xkipN88eKQtL6rSgocL5tbAP6Ola9Izm/Aw4Ora8He4x0BHg==} + dependencies: + undici-types: 5.26.5 + dev: true + + /esbuild@0.19.5: + resolution: {integrity: sha512-bUxalY7b1g8vNhQKdB24QDmHeY4V4tw/s6Ak5z+jJX9laP5MoQseTOMemAr0gxssjNcH0MCViG8ONI2kksvfFQ==} + engines: {node: '>=12'} + hasBin: true + requiresBuild: true + optionalDependencies: + '@esbuild/android-arm': 0.19.5 + '@esbuild/android-arm64': 0.19.5 + '@esbuild/android-x64': 0.19.5 + '@esbuild/darwin-arm64': 0.19.5 + '@esbuild/darwin-x64': 0.19.5 + '@esbuild/freebsd-arm64': 0.19.5 + '@esbuild/freebsd-x64': 0.19.5 + '@esbuild/linux-arm': 0.19.5 + '@esbuild/linux-arm64': 0.19.5 + '@esbuild/linux-ia32': 0.19.5 + '@esbuild/linux-loong64': 0.19.5 + '@esbuild/linux-mips64el': 0.19.5 + '@esbuild/linux-ppc64': 0.19.5 + '@esbuild/linux-riscv64': 0.19.5 + '@esbuild/linux-s390x': 0.19.5 + '@esbuild/linux-x64': 0.19.5 + '@esbuild/netbsd-x64': 0.19.5 + '@esbuild/openbsd-x64': 0.19.5 + '@esbuild/sunos-x64': 0.19.5 + '@esbuild/win32-arm64': 0.19.5 + '@esbuild/win32-ia32': 0.19.5 + '@esbuild/win32-x64': 0.19.5 + dev: true + + /to-fast-properties@2.0.0: + resolution: {integrity: sha512-/OaKK0xYrs3DmxRYqL/yDc+FxFUVYhDlXMhRmv3z915w2HF1tnN1omB354j8VUGO/hbRzyD6Y3sA7v7GS/ceog==} + engines: {node: '>=4'} + dev: true + + /typescript@5.2.2: + resolution: {integrity: sha512-mI4WrpHsbCIcwT9cF4FZvr80QUeKvsUsUvKDoR+X/7XHQH98xYD8YHZg7ANtz2GtZt/CBq2QJ0thkGJMHfqc1w==} + engines: {node: '>=14.17'} + hasBin: true + dev: true + + /undici-types@5.26.5: + resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==} + dev: true diff --git a/plugin-transpiler/src/index.ts b/plugin-transpiler/src/index.ts new file mode 100644 index 00000000000..f5c910f3456 --- /dev/null +++ b/plugin-transpiler/src/index.ts @@ -0,0 +1,46 @@ +import { transform } from '@babel/standalone' +import { presets } from './presets' + +process.stdin.setEncoding('utf8') + +let type: 'site' | 'frontend' = 'site' + +for (let i = 2; i < process.argv.length; i++) { + const arg = process.argv[i] + if (arg === '--type' && process.argv[i + 1]) { + type = process.argv[++i] as any + if (type !== 'site' && type !== 'frontend') { + console.error(`Unknown app type: ${type}`) + process.exit(1) + } + } else { + console.error(`Unknown argument: ${arg}`) + process.exit(1) + } +} +const { wrapper, ...options } = presets[type] + +let code = '' +process.stdin.on('readable', () => { + let chunk: string | Buffer + while ((chunk = process.stdin.read())) { + code += chunk + } +}) + +process.stdin.on('end', () => { + try { + let output = transform(code, options).code + if (output) { + if (wrapper) { + output = wrapper(output) + } + process.stdout.write(output, 'utf8') + } else { + throw new Error('Could not transpile code') + } + } catch (error: any) { + console.error(error.message) + process.exit(1) + } +}) diff --git a/plugin-transpiler/src/presets.ts b/plugin-transpiler/src/presets.ts new file mode 100644 index 00000000000..8cc9d051cc1 --- /dev/null +++ b/plugin-transpiler/src/presets.ts @@ -0,0 +1,25 @@ +export const presets = { + site: { + envName: 'production', + code: true, + babelrc: false, + configFile: false, + filename: 'site.ts', + presets: [['typescript', { isTSX: false, allExtensions: true }], 'env'], + wrapper: (code: string): string => `(function () {let exports={};${code};return exports;})`, + }, + frontend: { + envName: 'production', + code: true, + babelrc: false, + configFile: false, + filename: 'frontend.tsx', + plugins: ['transform-react-jsx'], + presets: [ + ['typescript', { isTSX: true, allExtensions: true }], + ['env', { targets: { esmodules: false } }], + ], + wrapper: (code: string): string => + `"use strict";\nexport function getFrontendApp (require) { let exports = {}; ${code}; return exports; }`, + }, +} diff --git a/plugin-transpiler/tsconfig.json b/plugin-transpiler/tsconfig.json new file mode 100644 index 00000000000..b13f56c7d6e --- /dev/null +++ b/plugin-transpiler/tsconfig.json @@ -0,0 +1,20 @@ +{ + "compilerOptions": { + "target": "ES2020", // Target a recent version of ECMAScript. + "module": "CommonJS", // Use CommonJS module system for Node.js. + "outDir": "./dist", // Output directory for the compiled JavaScript files. + "strict": true, // Enable all strict type-checking options. + "esModuleInterop": true, // Allows default imports from modules with no default export. + "skipLibCheck": true, // Skip type checking of declaration files. + "forceConsistentCasingInFileNames": true, // Disallow inconsistently-cased references to the same file. + "resolveJsonModule": true, // Allow importing .json files as modules. + "moduleResolution": "node" // Use Node.js-style module resolution. + }, + "include": [ + "src/**/*.ts", // Include all TypeScript files in the 'src' directory and its subdirectories. + "src/**/*.tsx" // Include all TSX files, if you have any. + ], + "exclude": [ + "node_modules" // Exclude node_modules from the compilation. + ] +} diff --git a/posthog/api/plugin.py b/posthog/api/plugin.py index dc08147c430..9311b756f44 100644 --- a/posthog/api/plugin.py +++ b/posthog/api/plugin.py @@ -1,6 +1,8 @@ import json +import os import re -from typing import Any, Dict, List, Optional, Set, cast +import subprocess +from typing import Any, Dict, List, Optional, Set, cast, Literal import requests from dateutil.relativedelta import relativedelta @@ -180,6 +182,24 @@ def _fix_formdata_config_json(request: request.Request, validated_data: dict): validated_data["config"] = json.loads(request.POST["config"]) +def transpile(input_string: str, type: Literal["site", "frontend"] = "site") -> Optional[str]: + from posthog.settings.base_variables import BASE_DIR + + transpiler_path = os.path.join(BASE_DIR, "plugin-transpiler/dist/index.js") + if type not in ["site", "frontend"]: + raise Exception('Invalid type. Must be "site" or "frontend".') + + process = subprocess.Popen( + ["node", transpiler_path, "--type", type], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + stdout, stderr = process.communicate(input=input_string.encode()) + + if process.returncode != 0: + error = stderr.decode() + raise Exception(error) + return stdout.decode() + + class PlainRenderer(renderers.BaseRenderer): format = "txt" @@ -355,25 +375,47 @@ class PluginViewSet(StructuredViewSetMixin, viewsets.ModelViewSet): plugin = self.get_plugin_with_permissions(reason="source editing") sources: Dict[str, PluginSourceFile] = {} performed_changes = False - for source in PluginSourceFile.objects.filter(plugin=plugin): - sources[source.filename] = source - for key, value in request.data.items(): + for plugin_source_file in PluginSourceFile.objects.filter(plugin=plugin): + sources[plugin_source_file.filename] = plugin_source_file + for key, source in request.data.items(): + transpiled = None + error = None + status = None + try: + if key == "site.ts": + transpiled = transpile(source, type="site") + status = PluginSourceFile.Status.TRANSPILED + elif key == "frontend.tsx": + transpiled = transpile(source, type="frontend") + status = PluginSourceFile.Status.TRANSPILED + except Exception as e: + error = str(e) + status = PluginSourceFile.Status.ERROR + if key not in sources: performed_changes = True sources[key], created = PluginSourceFile.objects.update_or_create( - plugin=plugin, filename=key, defaults={"source": value} + plugin=plugin, + filename=key, + defaults={ + "source": source, + "transpiled": transpiled, + "status": status, + "error": error, + }, ) - elif sources[key].source != value: + elif sources[key].source != source or sources[key].transpiled != transpiled or sources[key].error != error: performed_changes = True - if value is None: + if source is None: sources[key].delete() del sources[key] else: - sources[key].source = value - sources[key].status = None - sources[key].transpiled = None - sources[key].error = None + sources[key].source = source + sources[key].transpiled = transpiled + sources[key].status = status + sources[key].error = error sources[key].save() + response: Dict[str, str] = {} for _, source in sources.items(): response[source.filename] = source.source diff --git a/posthog/api/test/test_plugin.py b/posthog/api/test/test_plugin.py index 5c6a3344d4d..1854c32e060 100644 --- a/posthog/api/test/test_plugin.py +++ b/posthog/api/test/test_plugin.py @@ -37,6 +37,8 @@ def mocked_plugin_reload(*args, **kwargs): @mock.patch("posthog.models.plugin.reload_plugins_on_workers", side_effect=mocked_plugin_reload) @mock.patch("requests.get", side_effect=mocked_plugin_requests_get) class TestPluginAPI(APIBaseTest, QueryMatchingTest): + maxDiff = None + @classmethod def setUpTestData(cls): super().setUpTestData() @@ -703,88 +705,122 @@ class TestPluginAPI(APIBaseTest, QueryMatchingTest): self.assertEqual(response.json(), {"plugin.json": '{"name":"my plugin"}'}) self.assertEqual(mock_reload.call_count, 3) - def test_create_plugin_frontend_source(self, mock_get, mock_reload): - self.assertEqual(mock_reload.call_count, 0) + def test_transpile_plugin_frontend_source(self, mock_get, mock_reload): + # Setup + assert mock_reload.call_count == 0 response = self.client.post( "/api/organizations/@current/plugins/", {"plugin_type": "source", "name": "myplugin"}, ) - self.assertEqual(response.status_code, 201) + assert response.status_code == 201 id = response.json()["id"] - self.assertEqual( - response.json(), - { - "id": id, - "plugin_type": "source", - "name": "myplugin", - "description": None, - "url": None, - "config_schema": {}, - "tag": None, - "icon": None, - "latest_tag": None, - "is_global": False, - "organization_id": response.json()["organization_id"], - "organization_name": self.CONFIG_ORGANIZATION_NAME, - "capabilities": {}, - "metrics": {}, - "public_jobs": {}, - }, - ) - self.assertEqual(Plugin.objects.count(), 1) - self.assertEqual(mock_reload.call_count, 0) + assert response.json() == { + "id": id, + "plugin_type": "source", + "name": "myplugin", + "description": None, + "url": None, + "config_schema": {}, + "tag": None, + "icon": None, + "latest_tag": None, + "is_global": False, + "organization_id": response.json()["organization_id"], + "organization_name": self.CONFIG_ORGANIZATION_NAME, + "capabilities": {}, + "metrics": {}, + "public_jobs": {}, + } - response = self.client.patch( + assert Plugin.objects.count() == 1 + assert mock_reload.call_count == 0 + + # Add first source file, frontend.tsx + self.client.patch( f"/api/organizations/@current/plugins/{id}/update_source", {"frontend.tsx": "export const scene = {}"}, ) + assert Plugin.objects.count() == 1 + assert PluginSourceFile.objects.count() == 1 + assert mock_reload.call_count == 1 - self.assertEqual(Plugin.objects.count(), 1) - self.assertEqual(PluginSourceFile.objects.count(), 1) - self.assertEqual(mock_reload.call_count, 1) - + # Fetch transpiled source via API call plugin = Plugin.objects.get(pk=id) plugin_config = PluginConfig.objects.create(plugin=plugin, team=self.team, enabled=True, order=1) - - # no frontend, since no pluginserver transpiles the code response = self.client.get(f"/api/plugin_config/{plugin_config.id}/frontend") self.assertEqual(response.status_code, 200) self.assertEqual( - response.content, - b'export function getFrontendApp () { return {"transpiling": true} }', + response.content.decode("utf-8"), + '"use strict";\nexport function getFrontendApp (require) { let exports = {}; ' + '"use strict";\n\nObject.defineProperty(exports, "__esModule", {\n value: true\n});\nexports.scene = void 0;\n' + "var scene = exports.scene = {};" # this is it + "; return exports; }", ) - # mock the plugin server's transpilation + # Check in the database plugin_source = PluginSourceFile.objects.get(plugin_id=id) - self.assertEqual(plugin_source.status, None) - self.assertEqual(plugin_source.transpiled, None) - plugin_source.status = PluginSourceFile.Status.TRANSPILED - plugin_source.transpiled = "'random transpiled frontend'" - plugin_source.save() + assert plugin_source.source == "export const scene = {}" + assert plugin_source.error is None + assert plugin_source.transpiled == response.content.decode("utf-8") + assert plugin_source.status == PluginSourceFile.Status.TRANSPILED - # Can get the transpiled frontend - response = self.client.get(f"/api/plugin_config/{plugin_config.id}/frontend") - self.assertEqual(response.status_code, 200) - self.assertEqual(response.content, b"'random transpiled frontend'") - - # Update the source frontend + # Updates work self.client.patch( f"/api/organizations/@current/plugins/{id}/update_source", {"frontend.tsx": "export const scene = { name: 'new' }"}, ) - - # It will clear the transpiled frontend plugin_source = PluginSourceFile.objects.get(plugin_id=id) - self.assertEqual(plugin_source.source, "export const scene = { name: 'new' }") - self.assertEqual(plugin_source.transpiled, None) - - # And reply that it's transpiling - response = self.client.get(f"/api/plugin_config/{plugin_config.id}/frontend") - self.assertEqual(response.status_code, 200) - self.assertEqual( - response.content, - b'export function getFrontendApp () { return {"transpiling": true} }', + assert plugin_source.source == "export const scene = { name: 'new' }" + assert plugin_source.error is None + assert ( + plugin_source.transpiled + == ( + '"use strict";\nexport function getFrontendApp (require) { let exports = {}; "use strict";\n\n' + 'Object.defineProperty(exports, "__esModule", {\n value: true\n});\nexports.scene = void 0;\n' + "var scene = exports.scene = {\n name: 'new'\n};" # this is it + "; return exports; }" + ) ) + assert plugin_source.status == PluginSourceFile.Status.TRANSPILED + + # Errors as well + self.client.patch( + f"/api/organizations/@current/plugins/{id}/update_source", + {"frontend.tsx": "export const scene = { nam broken code foobar"}, + ) + plugin_source = PluginSourceFile.objects.get(plugin_id=id) + assert plugin_source.source == "export const scene = { nam broken code foobar" + assert plugin_source.transpiled is None + assert plugin_source.status == PluginSourceFile.Status.ERROR + assert ( + plugin_source.error + == '/frontend.tsx: Unexpected token, expected "," (1:27)\n\n> 1 | export const scene = { nam broken code foobar\n | ^\n' + ) + + # Deletes work + self.client.patch( + f"/api/organizations/@current/plugins/{id}/update_source", + {"frontend.tsx": None}, + ) + try: + PluginSourceFile.objects.get(plugin_id=id) + assert False, "Should have thrown DoesNotExist" + except PluginSourceFile.DoesNotExist: + assert True + + # Check that the syntax for "site.ts" is slightly different + self.client.patch( + f"/api/organizations/@current/plugins/{id}/update_source", + {"site.ts": "console.log('hello')"}, + ) + plugin_source = PluginSourceFile.objects.get(plugin_id=id) + assert plugin_source.source == "console.log('hello')" + assert plugin_source.error is None + assert ( + plugin_source.transpiled + == "(function () {let exports={};\"use strict\";\n\nconsole.log('hello');;return exports;})" + ) + assert plugin_source.status == PluginSourceFile.Status.TRANSPILED def test_plugin_repository(self, mock_get, mock_reload): response = self.client.get("/api/organizations/@current/plugins/repository/") diff --git a/production.Dockerfile b/production.Dockerfile index b3d27e2861e..d147edb7da7 100644 --- a/production.Dockerfile +++ b/production.Dockerfile @@ -26,7 +26,7 @@ WORKDIR /code SHELL ["/bin/bash", "-o", "pipefail", "-c"] COPY package.json pnpm-lock.yaml ./ -RUN corepack enable && pnpm --version && \ +RUN corepack enable && \ mkdir /tmp/pnpm-store && \ pnpm install --frozen-lockfile --store-dir /tmp/pnpm-store --prod && \ rm -rf /tmp/pnpm-store @@ -37,6 +37,22 @@ COPY babel.config.js tsconfig.json webpack.config.js ./ RUN pnpm build +# +# --------------------------------------------------------- +# +FROM node:18.12.1-bullseye-slim AS plugin-transpiler-build +WORKDIR /code +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +COPY plugin-transpiler/ plugin-transpiler/ +WORKDIR /code/plugin-transpiler +RUN corepack enable && \ + mkdir /tmp/pnpm-store && \ + pnpm install --frozen-lockfile --store-dir /tmp/pnpm-store && \ + pnpm build && \ + rm -rf /tmp/pnpm-store + + # # --------------------------------------------------------- # diff --git a/unit.json b/unit.json index 2cb730ff1d4..3d80936e14f 100644 --- a/unit.json +++ b/unit.json @@ -14,27 +14,23 @@ "metrics": [ { "match": { - "uri": [ - "/metrics" - ] + "uri": ["/metrics"] }, "action": { "pass": "applications/metrics" } - }, + } ], "status": [ { "match": { - "uri": [ - "/status" - ] + "uri": ["/status"] }, "action": { "proxy": "http://unix:/var/run/control.unit.sock" } - }, - ], + } + ] }, "applications": { "posthog": { @@ -52,6 +48,6 @@ "path": ".", "module": "unit_metrics", "user": "nobody" - }, + } } }