feat(apps): transpile via django (#18201)

.dockerignore

@@ -34,3 +34,5 @@
 !share/GeoLite2-City.mmdb
 !hogvm/python
 !unit.json
+!plugin-transpiler/src
+!plugin-transpiler/*.*

.github/actions/run-backend-tests/action.yml

@@ -62,6 +62,24 @@ runs:
           run: |
               sudo apt-get update && sudo apt-get install libxml2-dev libxmlsec1-dev libxmlsec1-openssl
 
+        - name: Install pnpm
+          uses: pnpm/action-setup@v2
+          with:
+              version: 8.x.x
+
+        - name: Set up Node.js
+          uses: actions/setup-node@v3
+          with:
+              node-version: 18
+              cache: pnpm
+
+        - name: Install plugin-transpiler
+          shell: bash
+          run: |
+              cd plugin-transpiler
+              pnpm install
+              pnpm run build
+
         - uses: syphar/restore-virtualenv@v1
           id: cache-backend-tests
           with:

.github/workflows/ci-backend.yml

@@ -68,6 +68,7 @@ jobs:
                         - mypy.ini
                         - pytest.ini
                         - frontend/src/queries/schema.json # Used for generating schema.py
+                        - plugin-transpiler/src # Used for transpiling plugins
                         # Make sure we run if someone is explicitly change the workflow
                         - .github/workflows/ci-backend.yml
                         - .github/actions/run-backend-tests/action.yml

.gitignore

@@ -55,3 +55,4 @@ gen/
 upgrade/
 hogvm/typescript/dist
 .wokeignore
+plugin-transpiler/dist

plugin-transpiler/README.md

@@ -0,0 +1,3 @@
+## Plugin Transpiler
+
+This project transpiles frontend plugins and site apps.

plugin-transpiler/build.mjs

@@ -0,0 +1,13 @@
+#!/usr/bin/env node
+import * as esbuild from 'esbuild'
+;(async function build() {
+    let result = await esbuild.build({
+        entryPoints: ['src/index.ts'],
+        bundle: true,
+        outdir: 'dist',
+    })
+    if (!result.errors.length) {
+        // eslint-disable-next-line no-console
+        console.log('Build succeeded')
+    }
+})()

plugin-transpiler/package.json

@@ -0,0 +1,21 @@
+{
+    "name": "plugin-transpiler",
+    "version": "1.0.0",
+    "description": "Transpiles site apps TSX to browser JS via stdin/stdout",
+    "main": "transpile.mjs",
+    "scripts": {
+        "test": "echo \"Error: no test specified\" && exit 1",
+        "build": "tsc -b && node build.mjs",
+        "start:dist": "node dist/index.js",
+        "start": "npm run build && npm run start:dist"
+    },
+    "author": "PostHog Inc.",
+    "license": "MIT",
+    "devDependencies": {
+        "@babel/standalone": "^7.23.2",
+        "@types/babel__standalone": "^7.1.6",
+        "@types/node": "^20.8.9",
+        "esbuild": "^0.19.5",
+        "typescript": "^5.2.2"
+    }
+}

plugin-transpiler/pnpm-lock.yaml

@@ -0,0 +1,340 @@
+lockfileVersion: '6.0'
+
+settings:
+  autoInstallPeers: true
+  excludeLinksFromLockfile: false
+
+devDependencies:
+  '@babel/standalone':
+    specifier: ^7.23.2
+    version: 7.23.2
+  '@types/babel__standalone':
+    specifier: ^7.1.6
+    version: 7.1.6
+  '@types/node':
+    specifier: ^20.8.9
+    version: 20.8.9
+  esbuild:
+    specifier: ^0.19.5
+    version: 0.19.5
+  typescript:
+    specifier: ^5.2.2
+    version: 5.2.2
+
+packages:
+
+  /@babel/helper-string-parser@7.22.5:
+    resolution: {integrity: sha512-mM4COjgZox8U+JcXQwPijIZLElkgEpO5rsERVDJTc2qfCDfERyob6k5WegS14SX18IIjv+XD+GrqNumY5JRCDw==}
+    engines: {node: '>=6.9.0'}
+    dev: true
+
+  /@babel/helper-validator-identifier@7.22.20:
+    resolution: {integrity: sha512-Y4OZ+ytlatR8AI+8KZfKuL5urKp7qey08ha31L8b3BwewJAoJamTzyvxPR/5D+KkdJCGPq/+8TukHBlY10FX9A==}
+    engines: {node: '>=6.9.0'}
+    dev: true
+
+  /@babel/parser@7.23.0:
+    resolution: {integrity: sha512-vvPKKdMemU85V9WE/l5wZEmImpCtLqbnTvqDS2U1fJ96KrxoW7KrXhNsNCblQlg8Ck4b85yxdTyelsMUgFUXiw==}
+    engines: {node: '>=6.0.0'}
+    hasBin: true
+    dependencies:
+      '@babel/types': 7.23.0
+    dev: true
+
+  /@babel/standalone@7.23.2:
+    resolution: {integrity: sha512-VJNw7OS26JvB6rE9XpbT6uQeQIEBWU5eeHGS4VR/+/4ZoKdLBXLcy66ZVJ/9IBkK1RMp8B0cohvhzdKWtJAGmg==}
+    engines: {node: '>=6.9.0'}
+    dev: true
+
+  /@babel/types@7.23.0:
+    resolution: {integrity: sha512-0oIyUfKoI3mSqMvsxBdclDwxXKXAUA8v/apZbc+iSyARYou1o8ZGDxbUYyLFoW2arqS2jDGqJuZvv1d/io1axg==}
+    engines: {node: '>=6.9.0'}
+    dependencies:
+      '@babel/helper-string-parser': 7.22.5
+      '@babel/helper-validator-identifier': 7.22.20
+      to-fast-properties: 2.0.0
+    dev: true
+
+  /@esbuild/android-arm64@0.19.5:
+    resolution: {integrity: sha512-5d1OkoJxnYQfmC+Zd8NBFjkhyCNYwM4n9ODrycTFY6Jk1IGiZ+tjVJDDSwDt77nK+tfpGP4T50iMtVi4dEGzhQ==}
+    engines: {node: '>=12'}
+    cpu: [arm64]
+    os: [android]
+    requiresBuild: true
+    dev: true
+    optional: true
+
+  /@esbuild/android-arm@0.19.5:
+    resolution: {integrity: sha512-bhvbzWFF3CwMs5tbjf3ObfGqbl/17ict2/uwOSfr3wmxDE6VdS2GqY/FuzIPe0q0bdhj65zQsvqfArI9MY6+AA==}
+    engines: {node: '>=12'}
+    cpu: [arm]
+    os: [android]
+    requiresBuild: true
+    dev: true
+    optional: true
+
+  /@esbuild/android-x64@0.19.5:
+    resolution: {integrity: sha512-9t+28jHGL7uBdkBjL90QFxe7DVA+KGqWlHCF8ChTKyaKO//VLuoBricQCgwhOjA1/qOczsw843Fy4cbs4H3DVA==}
+    engines: {node: '>=12'}
+    cpu: [x64]
+    os: [android]
+    requiresBuild: true
+    dev: true
+    optional: true
+
+  /@esbuild/darwin-arm64@0.19.5:
+    resolution: {integrity: sha512-mvXGcKqqIqyKoxq26qEDPHJuBYUA5KizJncKOAf9eJQez+L9O+KfvNFu6nl7SCZ/gFb2QPaRqqmG0doSWlgkqw==}
+    engines: {node: '>=12'}
+    cpu: [arm64]
+    os: [darwin]
+    requiresBuild: true
+    dev: true
+    optional: true
+
+  /@esbuild/darwin-x64@0.19.5:
+    resolution: {integrity: sha512-Ly8cn6fGLNet19s0X4unjcniX24I0RqjPv+kurpXabZYSXGM4Pwpmf85WHJN3lAgB8GSth7s5A0r856S+4DyiA==}
+    engines: {node: '>=12'}
+    cpu: [x64]
+    os: [darwin]
+    requiresBuild: true
+    dev: true
+    optional: true
+
+  /@esbuild/freebsd-arm64@0.19.5:
+    resolution: {integrity: sha512-GGDNnPWTmWE+DMchq1W8Sd0mUkL+APvJg3b11klSGUDvRXh70JqLAO56tubmq1s2cgpVCSKYywEiKBfju8JztQ==}
+    engines: {node: '>=12'}
+    cpu: [arm64]
+    os: [freebsd]
+    requiresBuild: true
+    dev: true
+    optional: true
+
+  /@esbuild/freebsd-x64@0.19.5:
+    resolution: {integrity: sha512-1CCwDHnSSoA0HNwdfoNY0jLfJpd7ygaLAp5EHFos3VWJCRX9DMwWODf96s9TSse39Br7oOTLryRVmBoFwXbuuQ==}
+    engines: {node: '>=12'}
+    cpu: [x64]
+    os: [freebsd]
+    requiresBuild: true
+    dev: true
+    optional: true
+
+  /@esbuild/linux-arm64@0.19.5:
+    resolution: {integrity: sha512-o3vYippBmSrjjQUCEEiTZ2l+4yC0pVJD/Dl57WfPwwlvFkrxoSO7rmBZFii6kQB3Wrn/6GwJUPLU5t52eq2meA==}
+    engines: {node: '>=12'}
+    cpu: [arm64]
+    os: [linux]
+    requiresBuild: true
+    dev: true
+    optional: true
+
+  /@esbuild/linux-arm@0.19.5:
+    resolution: {integrity: sha512-lrWXLY/vJBzCPC51QN0HM71uWgIEpGSjSZZADQhq7DKhPcI6NH1IdzjfHkDQws2oNpJKpR13kv7/pFHBbDQDwQ==}
+    engines: {node: '>=12'}
+    cpu: [arm]
+    os: [linux]
+    requiresBuild: true
+    dev: true
+    optional: true
+
+  /@esbuild/linux-ia32@0.19.5:
+    resolution: {integrity: sha512-MkjHXS03AXAkNp1KKkhSKPOCYztRtK+KXDNkBa6P78F8Bw0ynknCSClO/ztGszILZtyO/lVKpa7MolbBZ6oJtQ==}
+    engines: {node: '>=12'}
+    cpu: [ia32]
+    os: [linux]
+    requiresBuild: true
+    dev: true
+    optional: true
+
+  /@esbuild/linux-loong64@0.19.5:
+    resolution: {integrity: sha512-42GwZMm5oYOD/JHqHska3Jg0r+XFb/fdZRX+WjADm3nLWLcIsN27YKtqxzQmGNJgu0AyXg4HtcSK9HuOk3v1Dw==}
+    engines: {node: '>=12'}
+    cpu: [loong64]
+    os: [linux]
+    requiresBuild: true
+    dev: true
+    optional: true
+
+  /@esbuild/linux-mips64el@0.19.5:
+    resolution: {integrity: sha512-kcjndCSMitUuPJobWCnwQ9lLjiLZUR3QLQmlgaBfMX23UEa7ZOrtufnRds+6WZtIS9HdTXqND4yH8NLoVVIkcg==}
+    engines: {node: '>=12'}
+    cpu: [mips64el]
+    os: [linux]
+    requiresBuild: true
+    dev: true
+    optional: true
+
+  /@esbuild/linux-ppc64@0.19.5:
+    resolution: {integrity: sha512-yJAxJfHVm0ZbsiljbtFFP1BQKLc8kUF6+17tjQ78QjqjAQDnhULWiTA6u0FCDmYT1oOKS9PzZ2z0aBI+Mcyj7Q==}
+    engines: {node: '>=12'}
+    cpu: [ppc64]
+    os: [linux]
+    requiresBuild: true
+    dev: true
+    optional: true
+
+  /@esbuild/linux-riscv64@0.19.5:
+    resolution: {integrity: sha512-5u8cIR/t3gaD6ad3wNt1MNRstAZO+aNyBxu2We8X31bA8XUNyamTVQwLDA1SLoPCUehNCymhBhK3Qim1433Zag==}
+    engines: {node: '>=12'}
+    cpu: [riscv64]
+    os: [linux]
+    requiresBuild: true
+    dev: true
+    optional: true
+
+  /@esbuild/linux-s390x@0.19.5:
+    resolution: {integrity: sha512-Z6JrMyEw/EmZBD/OFEFpb+gao9xJ59ATsoTNlj39jVBbXqoZm4Xntu6wVmGPB/OATi1uk/DB+yeDPv2E8PqZGw==}
+    engines: {node: '>=12'}
+    cpu: [s390x]
+    os: [linux]
+    requiresBuild: true
+    dev: true
+    optional: true
+
+  /@esbuild/linux-x64@0.19.5:
+    resolution: {integrity: sha512-psagl+2RlK1z8zWZOmVdImisMtrUxvwereIdyJTmtmHahJTKb64pAcqoPlx6CewPdvGvUKe2Jw+0Z/0qhSbG1A==}
+    engines: {node: '>=12'}
+    cpu: [x64]
+    os: [linux]
+    requiresBuild: true
+    dev: true
+    optional: true
+
+  /@esbuild/netbsd-x64@0.19.5:
+    resolution: {integrity: sha512-kL2l+xScnAy/E/3119OggX8SrWyBEcqAh8aOY1gr4gPvw76la2GlD4Ymf832UCVbmuWeTf2adkZDK+h0Z/fB4g==}
+    engines: {node: '>=12'}
+    cpu: [x64]
+    os: [netbsd]
+    requiresBuild: true
+    dev: true
+    optional: true
+
+  /@esbuild/openbsd-x64@0.19.5:
+    resolution: {integrity: sha512-sPOfhtzFufQfTBgRnE1DIJjzsXukKSvZxloZbkJDG383q0awVAq600pc1nfqBcl0ice/WN9p4qLc39WhBShRTA==}
+    engines: {node: '>=12'}
+    cpu: [x64]
+    os: [openbsd]
+    requiresBuild: true
+    dev: true
+    optional: true
+
+  /@esbuild/sunos-x64@0.19.5:
+    resolution: {integrity: sha512-dGZkBXaafuKLpDSjKcB0ax0FL36YXCvJNnztjKV+6CO82tTYVDSH2lifitJ29jxRMoUhgkg9a+VA/B03WK5lcg==}
+    engines: {node: '>=12'}
+    cpu: [x64]
+    os: [sunos]
+    requiresBuild: true
+    dev: true
+    optional: true
+
+  /@esbuild/win32-arm64@0.19.5:
+    resolution: {integrity: sha512-dWVjD9y03ilhdRQ6Xig1NWNgfLtf2o/STKTS+eZuF90fI2BhbwD6WlaiCGKptlqXlURVB5AUOxUj09LuwKGDTg==}
+    engines: {node: '>=12'}
+    cpu: [arm64]
+    os: [win32]
+    requiresBuild: true
+    dev: true
+    optional: true
+
+  /@esbuild/win32-ia32@0.19.5:
+    resolution: {integrity: sha512-4liggWIA4oDgUxqpZwrDhmEfAH4d0iljanDOK7AnVU89T6CzHon/ony8C5LeOdfgx60x5cnQJFZwEydVlYx4iw==}
+    engines: {node: '>=12'}
+    cpu: [ia32]
+    os: [win32]
+    requiresBuild: true
+    dev: true
+    optional: true
+
+  /@esbuild/win32-x64@0.19.5:
+    resolution: {integrity: sha512-czTrygUsB/jlM8qEW5MD8bgYU2Xg14lo6kBDXW6HdxKjh8M5PzETGiSHaz9MtbXBYDloHNUAUW2tMiKW4KM9Mw==}
+    engines: {node: '>=12'}
+    cpu: [x64]
+    os: [win32]
+    requiresBuild: true
+    dev: true
+    optional: true
+
+  /@types/babel__core@7.20.3:
+    resolution: {integrity: sha512-54fjTSeSHwfan8AyHWrKbfBWiEUrNTZsUwPTDSNaaP1QDQIZbeNUg3a59E9D+375MzUw/x1vx2/0F5LBz+AeYA==}
+    dependencies:
+      '@babel/parser': 7.23.0
+      '@babel/types': 7.23.0
+      '@types/babel__generator': 7.6.6
+      '@types/babel__template': 7.4.3
+      '@types/babel__traverse': 7.20.3
+    dev: true
+
+  /@types/babel__generator@7.6.6:
+    resolution: {integrity: sha512-66BXMKb/sUWbMdBNdMvajU7i/44RkrA3z/Yt1c7R5xejt8qh84iU54yUWCtm0QwGJlDcf/gg4zd/x4mpLAlb/w==}
+    dependencies:
+      '@babel/types': 7.23.0
+    dev: true
+
+  /@types/babel__standalone@7.1.6:
+    resolution: {integrity: sha512-JRVq2H02irW6GDxkHepsdZVjor87SZo4eDsxoBtSbswf7HWtVYNxCmJDBkJbFNFWGOuzsxATZV8xZ2PI9sBiIw==}
+    dependencies:
+      '@types/babel__core': 7.20.3
+    dev: true
+
+  /@types/babel__template@7.4.3:
+    resolution: {integrity: sha512-ciwyCLeuRfxboZ4isgdNZi/tkt06m8Tw6uGbBSBgWrnnZGNXiEyM27xc/PjXGQLqlZ6ylbgHMnm7ccF9tCkOeQ==}
+    dependencies:
+      '@babel/parser': 7.23.0
+      '@babel/types': 7.23.0
+    dev: true
+
+  /@types/babel__traverse@7.20.3:
+    resolution: {integrity: sha512-Lsh766rGEFbaxMIDH7Qa+Yha8cMVI3qAK6CHt3OR0YfxOIn5Z54iHiyDRycHrBqeIiqGa20Kpsv1cavfBKkRSw==}
+    dependencies:
+      '@babel/types': 7.23.0
+    dev: true
+
+  /@types/node@20.8.9:
+    resolution: {integrity: sha512-UzykFsT3FhHb1h7yD4CA4YhBHq545JC0YnEz41xkipN88eKQtL6rSgocL5tbAP6Ola9Izm/Aw4Ora8He4x0BHg==}
+    dependencies:
+      undici-types: 5.26.5
+    dev: true
+
+  /esbuild@0.19.5:
+    resolution: {integrity: sha512-bUxalY7b1g8vNhQKdB24QDmHeY4V4tw/s6Ak5z+jJX9laP5MoQseTOMemAr0gxssjNcH0MCViG8ONI2kksvfFQ==}
+    engines: {node: '>=12'}
+    hasBin: true
+    requiresBuild: true
+    optionalDependencies:
+      '@esbuild/android-arm': 0.19.5
+      '@esbuild/android-arm64': 0.19.5
+      '@esbuild/android-x64': 0.19.5
+      '@esbuild/darwin-arm64': 0.19.5
+      '@esbuild/darwin-x64': 0.19.5
+      '@esbuild/freebsd-arm64': 0.19.5
+      '@esbuild/freebsd-x64': 0.19.5
+      '@esbuild/linux-arm': 0.19.5
+      '@esbuild/linux-arm64': 0.19.5
+      '@esbuild/linux-ia32': 0.19.5
+      '@esbuild/linux-loong64': 0.19.5
+      '@esbuild/linux-mips64el': 0.19.5
+      '@esbuild/linux-ppc64': 0.19.5
+      '@esbuild/linux-riscv64': 0.19.5
+      '@esbuild/linux-s390x': 0.19.5
+      '@esbuild/linux-x64': 0.19.5
+      '@esbuild/netbsd-x64': 0.19.5
+      '@esbuild/openbsd-x64': 0.19.5
+      '@esbuild/sunos-x64': 0.19.5
+      '@esbuild/win32-arm64': 0.19.5
+      '@esbuild/win32-ia32': 0.19.5
+      '@esbuild/win32-x64': 0.19.5
+    dev: true
+
+  /to-fast-properties@2.0.0:
+    resolution: {integrity: sha512-/OaKK0xYrs3DmxRYqL/yDc+FxFUVYhDlXMhRmv3z915w2HF1tnN1omB354j8VUGO/hbRzyD6Y3sA7v7GS/ceog==}
+    engines: {node: '>=4'}
+    dev: true
+
+  /typescript@5.2.2:
+    resolution: {integrity: sha512-mI4WrpHsbCIcwT9cF4FZvr80QUeKvsUsUvKDoR+X/7XHQH98xYD8YHZg7ANtz2GtZt/CBq2QJ0thkGJMHfqc1w==}
+    engines: {node: '>=14.17'}
+    hasBin: true
+    dev: true
+
+  /undici-types@5.26.5:
+    resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==}
+    dev: true

plugin-transpiler/src/index.ts

@@ -0,0 +1,46 @@
+import { transform } from '@babel/standalone'
+import { presets } from './presets'
+
+process.stdin.setEncoding('utf8')
+
+let type: 'site' | 'frontend' = 'site'
+
+for (let i = 2; i < process.argv.length; i++) {
+    const arg = process.argv[i]
+    if (arg === '--type' && process.argv[i + 1]) {
+        type = process.argv[++i] as any
+        if (type !== 'site' && type !== 'frontend') {
+            console.error(`Unknown app type: ${type}`)
+            process.exit(1)
+        }
+    } else {
+        console.error(`Unknown argument: ${arg}`)
+        process.exit(1)
+    }
+}
+const { wrapper, ...options } = presets[type]
+
+let code = ''
+process.stdin.on('readable', () => {
+    let chunk: string | Buffer
+    while ((chunk = process.stdin.read())) {
+        code += chunk
+    }
+})
+
+process.stdin.on('end', () => {
+    try {
+        let output = transform(code, options).code
+        if (output) {
+            if (wrapper) {
+                output = wrapper(output)
+            }
+            process.stdout.write(output, 'utf8')
+        } else {
+            throw new Error('Could not transpile code')
+        }
+    } catch (error: any) {
+        console.error(error.message)
+        process.exit(1)
+    }
+})

plugin-transpiler/src/presets.ts

@@ -0,0 +1,25 @@
+export const presets = {
+    site: {
+        envName: 'production',
+        code: true,
+        babelrc: false,
+        configFile: false,
+        filename: 'site.ts',
+        presets: [['typescript', { isTSX: false, allExtensions: true }], 'env'],
+        wrapper: (code: string): string => `(function () {let exports={};${code};return exports;})`,
+    },
+    frontend: {
+        envName: 'production',
+        code: true,
+        babelrc: false,
+        configFile: false,
+        filename: 'frontend.tsx',
+        plugins: ['transform-react-jsx'],
+        presets: [
+            ['typescript', { isTSX: true, allExtensions: true }],
+            ['env', { targets: { esmodules: false } }],
+        ],
+        wrapper: (code: string): string =>
+            `"use strict";\nexport function getFrontendApp (require) { let exports = {}; ${code}; return exports; }`,
+    },
+}

plugin-transpiler/tsconfig.json

@@ -0,0 +1,20 @@
+{
+    "compilerOptions": {
+        "target": "ES2020", // Target a recent version of ECMAScript.
+        "module": "CommonJS", // Use CommonJS module system for Node.js.
+        "outDir": "./dist", // Output directory for the compiled JavaScript files.
+        "strict": true, // Enable all strict type-checking options.
+        "esModuleInterop": true, // Allows default imports from modules with no default export.
+        "skipLibCheck": true, // Skip type checking of declaration files.
+        "forceConsistentCasingInFileNames": true, // Disallow inconsistently-cased references to the same file.
+        "resolveJsonModule": true, // Allow importing .json files as modules.
+        "moduleResolution": "node" // Use Node.js-style module resolution.
+    },
+    "include": [
+        "src/**/*.ts", // Include all TypeScript files in the 'src' directory and its subdirectories.
+        "src/**/*.tsx" // Include all TSX files, if you have any.
+    ],
+    "exclude": [
+        "node_modules" // Exclude node_modules from the compilation.
+    ]
+}

posthog/api/plugin.py

@@ -1,6 +1,8 @@
 import json
+import os
 import re
-from typing import Any, Dict, List, Optional, Set, cast
+import subprocess
+from typing import Any, Dict, List, Optional, Set, cast, Literal
 
 import requests
 from dateutil.relativedelta import relativedelta
@@ -180,6 +182,24 @@ def _fix_formdata_config_json(request: request.Request, validated_data: dict):
         validated_data["config"] = json.loads(request.POST["config"])
 
 
+def transpile(input_string: str, type: Literal["site", "frontend"] = "site") -> Optional[str]:
+    from posthog.settings.base_variables import BASE_DIR
+
+    transpiler_path = os.path.join(BASE_DIR, "plugin-transpiler/dist/index.js")
+    if type not in ["site", "frontend"]:
+        raise Exception('Invalid type. Must be "site" or "frontend".')
+
+    process = subprocess.Popen(
+        ["node", transpiler_path, "--type", type], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE
+    )
+    stdout, stderr = process.communicate(input=input_string.encode())
+
+    if process.returncode != 0:
+        error = stderr.decode()
+        raise Exception(error)
+    return stdout.decode()
+
+
 class PlainRenderer(renderers.BaseRenderer):
     format = "txt"
 
@@ -355,25 +375,47 @@ class PluginViewSet(StructuredViewSetMixin, viewsets.ModelViewSet):
         plugin = self.get_plugin_with_permissions(reason="source editing")
         sources: Dict[str, PluginSourceFile] = {}
         performed_changes = False
-        for source in PluginSourceFile.objects.filter(plugin=plugin):
+        for plugin_source_file in PluginSourceFile.objects.filter(plugin=plugin):
-            sources[source.filename] = source
+            sources[plugin_source_file.filename] = plugin_source_file
-        for key, value in request.data.items():
+        for key, source in request.data.items():
+            transpiled = None
+            error = None
+            status = None
+            try:
+                if key == "site.ts":
+                    transpiled = transpile(source, type="site")
+                    status = PluginSourceFile.Status.TRANSPILED
+                elif key == "frontend.tsx":
+                    transpiled = transpile(source, type="frontend")
+                    status = PluginSourceFile.Status.TRANSPILED
+            except Exception as e:
+                error = str(e)
+                status = PluginSourceFile.Status.ERROR
+
             if key not in sources:
                 performed_changes = True
                 sources[key], created = PluginSourceFile.objects.update_or_create(
-                    plugin=plugin, filename=key, defaults={"source": value}
+                    plugin=plugin,
+                    filename=key,
+                    defaults={
+                        "source": source,
+                        "transpiled": transpiled,
+                        "status": status,
+                        "error": error,
+                    },
                 )
-            elif sources[key].source != value:
+            elif sources[key].source != source or sources[key].transpiled != transpiled or sources[key].error != error:
                 performed_changes = True
-                if value is None:
+                if source is None:
                     sources[key].delete()
                     del sources[key]
                 else:
-                    sources[key].source = value
+                    sources[key].source = source
-                    sources[key].status = None
+                    sources[key].transpiled = transpiled
-                    sources[key].transpiled = None
+                    sources[key].status = status
-                    sources[key].error = None
+                    sources[key].error = error
                     sources[key].save()
+
         response: Dict[str, str] = {}
         for _, source in sources.items():
             response[source.filename] = source.source

posthog/api/test/test_plugin.py

@@ -37,6 +37,8 @@ def mocked_plugin_reload(*args, **kwargs):
 @mock.patch("posthog.models.plugin.reload_plugins_on_workers", side_effect=mocked_plugin_reload)
 @mock.patch("requests.get", side_effect=mocked_plugin_requests_get)
 class TestPluginAPI(APIBaseTest, QueryMatchingTest):
+    maxDiff = None
+
     @classmethod
     def setUpTestData(cls):
         super().setUpTestData()
@@ -703,88 +705,122 @@ class TestPluginAPI(APIBaseTest, QueryMatchingTest):
         self.assertEqual(response.json(), {"plugin.json": '{"name":"my plugin"}'})
         self.assertEqual(mock_reload.call_count, 3)
 
-    def test_create_plugin_frontend_source(self, mock_get, mock_reload):
+    def test_transpile_plugin_frontend_source(self, mock_get, mock_reload):
-        self.assertEqual(mock_reload.call_count, 0)
+        # Setup
+        assert mock_reload.call_count == 0
         response = self.client.post(
             "/api/organizations/@current/plugins/",
             {"plugin_type": "source", "name": "myplugin"},
         )
-        self.assertEqual(response.status_code, 201)
+        assert response.status_code == 201
         id = response.json()["id"]
-        self.assertEqual(
+        assert response.json() == {
-            response.json(),
+            "id": id,
-            {
+            "plugin_type": "source",
-                "id": id,
+            "name": "myplugin",
-                "plugin_type": "source",
+            "description": None,
-                "name": "myplugin",
+            "url": None,
-                "description": None,
+            "config_schema": {},
-                "url": None,
+            "tag": None,
-                "config_schema": {},
+            "icon": None,
-                "tag": None,
+            "latest_tag": None,
-                "icon": None,
+            "is_global": False,
-                "latest_tag": None,
+            "organization_id": response.json()["organization_id"],
-                "is_global": False,
+            "organization_name": self.CONFIG_ORGANIZATION_NAME,
-                "organization_id": response.json()["organization_id"],
+            "capabilities": {},
-                "organization_name": self.CONFIG_ORGANIZATION_NAME,
+            "metrics": {},
-                "capabilities": {},
+            "public_jobs": {},
-                "metrics": {},
+        }
-                "public_jobs": {},
-            },
-        )
-        self.assertEqual(Plugin.objects.count(), 1)
-        self.assertEqual(mock_reload.call_count, 0)
 
-        response = self.client.patch(
+        assert Plugin.objects.count() == 1
+        assert mock_reload.call_count == 0
+
+        # Add first source file, frontend.tsx
+        self.client.patch(
             f"/api/organizations/@current/plugins/{id}/update_source",
             {"frontend.tsx": "export const scene = {}"},
         )
+        assert Plugin.objects.count() == 1
+        assert PluginSourceFile.objects.count() == 1
+        assert mock_reload.call_count == 1
 
-        self.assertEqual(Plugin.objects.count(), 1)
+        # Fetch transpiled source via API call
-        self.assertEqual(PluginSourceFile.objects.count(), 1)
-        self.assertEqual(mock_reload.call_count, 1)
-
         plugin = Plugin.objects.get(pk=id)
         plugin_config = PluginConfig.objects.create(plugin=plugin, team=self.team, enabled=True, order=1)
-
-        # no frontend, since no pluginserver transpiles the code
         response = self.client.get(f"/api/plugin_config/{plugin_config.id}/frontend")
         self.assertEqual(response.status_code, 200)
         self.assertEqual(
-            response.content,
+            response.content.decode("utf-8"),
-            b'export function getFrontendApp () { return {"transpiling": true} }',
+            '"use strict";\nexport function getFrontendApp (require) { let exports = {}; '
+            '"use strict";\n\nObject.defineProperty(exports, "__esModule", {\n  value: true\n});\nexports.scene = void 0;\n'
+            "var scene = exports.scene = {};"  # this is it
+            "; return exports; }",
         )
 
-        # mock the plugin server's transpilation
+        # Check in the database
         plugin_source = PluginSourceFile.objects.get(plugin_id=id)
-        self.assertEqual(plugin_source.status, None)
+        assert plugin_source.source == "export const scene = {}"
-        self.assertEqual(plugin_source.transpiled, None)
+        assert plugin_source.error is None
-        plugin_source.status = PluginSourceFile.Status.TRANSPILED
+        assert plugin_source.transpiled == response.content.decode("utf-8")
-        plugin_source.transpiled = "'random transpiled frontend'"
+        assert plugin_source.status == PluginSourceFile.Status.TRANSPILED
-        plugin_source.save()
 
-        # Can get the transpiled frontend
+        # Updates work
-        response = self.client.get(f"/api/plugin_config/{plugin_config.id}/frontend")
-        self.assertEqual(response.status_code, 200)
-        self.assertEqual(response.content, b"'random transpiled frontend'")
-
-        # Update the source frontend
         self.client.patch(
             f"/api/organizations/@current/plugins/{id}/update_source",
             {"frontend.tsx": "export const scene = { name: 'new' }"},
         )
-
-        # It will clear the transpiled frontend
         plugin_source = PluginSourceFile.objects.get(plugin_id=id)
-        self.assertEqual(plugin_source.source, "export const scene = { name: 'new' }")
+        assert plugin_source.source == "export const scene = { name: 'new' }"
-        self.assertEqual(plugin_source.transpiled, None)
+        assert plugin_source.error is None
-
+        assert (
-        # And reply that it's transpiling
+            plugin_source.transpiled
-        response = self.client.get(f"/api/plugin_config/{plugin_config.id}/frontend")
+            == (
-        self.assertEqual(response.status_code, 200)
+                '"use strict";\nexport function getFrontendApp (require) { let exports = {}; "use strict";\n\n'
-        self.assertEqual(
+                'Object.defineProperty(exports, "__esModule", {\n  value: true\n});\nexports.scene = void 0;\n'
-            response.content,
+                "var scene = exports.scene = {\n  name: 'new'\n};"  # this is it
-            b'export function getFrontendApp () { return {"transpiling": true} }',
+                "; return exports; }"
+            )
         )
+        assert plugin_source.status == PluginSourceFile.Status.TRANSPILED
+
+        # Errors as well
+        self.client.patch(
+            f"/api/organizations/@current/plugins/{id}/update_source",
+            {"frontend.tsx": "export const scene = { nam broken code foobar"},
+        )
+        plugin_source = PluginSourceFile.objects.get(plugin_id=id)
+        assert plugin_source.source == "export const scene = { nam broken code foobar"
+        assert plugin_source.transpiled is None
+        assert plugin_source.status == PluginSourceFile.Status.ERROR
+        assert (
+            plugin_source.error
+            == '/frontend.tsx: Unexpected token, expected "," (1:27)\n\n> 1 | export const scene = { nam broken code foobar\n    |                            ^\n'
+        )
+
+        # Deletes work
+        self.client.patch(
+            f"/api/organizations/@current/plugins/{id}/update_source",
+            {"frontend.tsx": None},
+        )
+        try:
+            PluginSourceFile.objects.get(plugin_id=id)
+            assert False, "Should have thrown DoesNotExist"
+        except PluginSourceFile.DoesNotExist:
+            assert True
+
+        # Check that the syntax for "site.ts" is slightly different
+        self.client.patch(
+            f"/api/organizations/@current/plugins/{id}/update_source",
+            {"site.ts": "console.log('hello')"},
+        )
+        plugin_source = PluginSourceFile.objects.get(plugin_id=id)
+        assert plugin_source.source == "console.log('hello')"
+        assert plugin_source.error is None
+        assert (
+            plugin_source.transpiled
+            == "(function () {let exports={};\"use strict\";\n\nconsole.log('hello');;return exports;})"
+        )
+        assert plugin_source.status == PluginSourceFile.Status.TRANSPILED
 
     def test_plugin_repository(self, mock_get, mock_reload):
         response = self.client.get("/api/organizations/@current/plugins/repository/")

production.Dockerfile

@@ -26,7 +26,7 @@ WORKDIR /code
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
 COPY package.json pnpm-lock.yaml ./
-RUN corepack enable && pnpm --version && \
+RUN corepack enable && \
     mkdir /tmp/pnpm-store && \
     pnpm install --frozen-lockfile --store-dir /tmp/pnpm-store --prod && \
     rm -rf /tmp/pnpm-store
@@ -37,6 +37,22 @@ COPY babel.config.js tsconfig.json webpack.config.js ./
 RUN pnpm build
 
 
+#
+# ---------------------------------------------------------
+#
+FROM node:18.12.1-bullseye-slim AS plugin-transpiler-build
+WORKDIR /code
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
+COPY plugin-transpiler/ plugin-transpiler/
+WORKDIR /code/plugin-transpiler
+RUN corepack enable && \
+    mkdir /tmp/pnpm-store && \
+    pnpm install --frozen-lockfile --store-dir /tmp/pnpm-store && \
+    pnpm build && \
+    rm -rf /tmp/pnpm-store
+
+
 #
 # ---------------------------------------------------------
 #

unit.json

@@ -14,27 +14,23 @@
         "metrics": [
             {
                 "match": {
-                    "uri": [
+                    "uri": ["/metrics"]
-                        "/metrics"
-                    ]
                 },
                 "action": {
                     "pass": "applications/metrics"
                 }
-            },
+            }
         ],
         "status": [
             {
                 "match": {
-                    "uri": [
+                    "uri": ["/status"]
-                        "/status"
-                    ]
                 },
                 "action": {
                     "proxy": "http://unix:/var/run/control.unit.sock"
                 }
-            },
+            }
-        ],
+        ]
     },
     "applications": {
         "posthog": {
@@ -52,6 +48,6 @@
             "path": ".",
             "module": "unit_metrics",
             "user": "nobody"
-        },
+        }
     }
 }