mirror of
https://github.com/PostHog/posthog.git
synced 2024-12-01 12:21:02 +01:00
371 lines
17 KiB
TypeScript
371 lines
17 KiB
TypeScript
import { randomBytes } from 'crypto'
|
|
import { DateTime } from 'luxon'
|
|
|
|
import { ClickHouseTimestamp, LogLevel } from '../src/types'
|
|
import { safeClickhouseString } from '../src/utils/db/utils'
|
|
import {
|
|
bufferToStream,
|
|
clickHouseTimestampToDateTime,
|
|
cloneObject,
|
|
escapeClickHouseString,
|
|
groupBy,
|
|
sanitizeSqlIdentifier,
|
|
setLogLevel,
|
|
stringify,
|
|
UUID,
|
|
UUIDT,
|
|
} from '../src/utils/utils'
|
|
|
|
// .zip in Base64: github repo posthog/helloworldplugin
|
|
const zip =
|
|
'UEsDBAoAAAAAAA8LbVEAAAAAAAAAAAAAAAAjAAkAaGVsbG93b3JsZHBsdWdpbi1pbWFnZWxlc3MtdmVyc2lvbi9VVAUAAc9Qrl9QSwMECgAAAAgADwttUQT7a+JUAAAAdQAAAC4ACQBoZWxsb3dvcmxkcGx1Z2luLWltYWdlbGVzcy12ZXJzaW9uLy5wcmV0dGllcnJjVVQFAAHPUK5fq+ZSAAKlkqLEzJzMvHTn/NzcRCUrBaXUYlMlHahcYlJ4ZkpJBlDYBCpUnJqbCeSmJeYUp8KEgLpzUgNL80tSgTIlRaUwiYKizLwSmAGGRgZctVwAUEsDBAoAAAAIAA8LbVG/Zg9y6wAAAPMBAAArAAkAaGVsbG93b3JsZHBsdWdpbi1pbWFnZWxlc3MtdmVyc2lvbi9pbmRleC5qc1VUBQABz1CuX31RTUvEMBC951eMi5AUS1E8e9Sz4FFE0jjNBrKTkkxcZOl/N9u0hyo0hyHDe/PefOj0QwaGTIZdIEjIeXz12TpSFzCBBmdhauAioLySp+Cx88GqwxsyO7KQR+AjAqM+3Ryaf7yq0YhJCL31GmMwmNLzNxIrvMYWVs8WjDZFdWPNJWZijPAE+qwdV1JnkZVcINnC/dLEjKUttgrcwUMjZpoboJp3pZ8RIztMq+n1/cXe5RG9D/KjNCHPIfovucPtdZyZdaqupDvk27XPWjH/d+je9Z+UT/1SUNKXZbXqsa5gqiPGctRIVaHc4RdQSwMECgAAAAgADwttUVpiCFkvAAAAOQAAACkACQBoZWxsb3dvcmxkcGx1Z2luLWltYWdlbGVzcy12ZXJzaW9uL2xpYi5qc1VUBQABz1CuX0srzUsuyczPU8jJTHKDsTXySnOTUos0Faq5FICgKLWktChPASKooKVgZM1VywUAUEsDBAoAAAAIAA8LbVE7Twti+wAAAO0BAAAvAAkAaGVsbG93b3JsZHBsdWdpbi1pbWFnZWxlc3MtdmVyc2lvbi9wYWNrYWdlLmpzb25VVAUAAc9Qrl+VUbtOAzEQ7O8rjJHSQO6S9noEFEgUlGkud4vtyOe1dm1IFOXf8eNE0qazZ2ZnvONzI4R0wwyyF9IjB41qrcFa/EWy09rbqIyTz1n2A8QGXVZu2k27regEPJLxYWFeCSCIoEEUg4cqmgdTWOMmOLYHrmgd5ESc0zUBAThkGYwaxU6+ECH1wqHIhGAPo/k2MO2kWK0EHE0QW5kmL8WNIL3fBKTTjeHJl82UCSUyQZHsgjzpEDz3XZfOOu7bEefuM1Xwhqq7VlAbaLPDf9QQU0+Ubeoi1ozguCR9vH9VbB/VzWZL6h2JnWGOwNdQjTP4QcGdPo8Ew5T+t7k0f1BLAwQKAAAACAAPC21Ru8C8oc0AAABTAQAALgAJAGhlbGxvd29ybGRwbHVnaW4taW1hZ2VsZXNzLXZlcnNpb24vcGx1Z2luLmpzb25VVAUAAc9Qrl9tjz1rwzAQhnf/iquXLCHesxQytKVToEPms3WWr8g6VzqRL/LfKymQdOggxPu8zwvStQFoPc7UbqGdyDk5SnBmccmyb9elTcHVUnWJ266zrFPqN4PM3V6ifojt/t8ZikPgRVl82b8HIgWdCA7FBPQG3kQAYYdhDZ9fQIaL/HKfz8h1x97QafMd79RxX2C+HmgQP7LN9JpTzj2GR/jzucOEuorAvr4hS691Xh09L9WJGtjbJzc0YnJaqh4vTx7oJ3Egk4sRXaTKb005t+YXUEsBAgAACgAAAAAADwttUQAAAAAAAAAAAAAAACMACQAAAAAAAAAQAAAAAAAAAGhlbGxvd29ybGRwbHVnaW4taW1hZ2VsZXNzLXZlcnNpb24vVVQFAAHPUK5fUEsBAgAACgAAAAgADwttUQT7a+JUAAAAdQAAAC4ACQAAAAAAAQAAAAAASgAAAGhlbGxvd29ybGRwbHVnaW4taW1hZ2VsZXNzLXZlcnNpb24vLnByZXR0aWVycmNVVAUAAc9Qrl9QSwECAAAKAAAACAAPC21Rv2YPcusAAADzAQAAKwAJAAAAAAABAAAAAADzAAAAaGVsbG93b3JsZHBsdWdpbi1pbWFnZWxlc3MtdmVyc2lvbi9pbmRleC5qc1VUBQABz1CuX1BLAQIAAAoAAAAIAA8LbVFaYghZLwAAADkAAAApAAkAAAAAAAEAAAAAADACAABoZWxsb3dvcmxkcGx1Z2luLWltYWdlbGVzcy12ZXJzaW9uL2xpYi5qc1VUBQABz1CuX1BLAQIAAAoAAAAIAA8LbVE7Twti+wAAAO0BAAAvAAkAAAAAAAEAAAAAAK8CAABoZWxsb3dvcmxkcGx1Z2luLWltYWdlbGVzcy12ZXJzaW9uL3BhY2thZ2UuanNvblVUBQABz1CuX1BLAQIAAAoAAAAIAA8LbVG7wLyhzQAAAFMBAAAuAAkAAAAAAAEAAAAAAAAEAABoZWxsb3dvcmxkcGx1Z2luLWltYWdlbGVzcy12ZXJzaW9uL3BsdWdpbi5qc29uVVQFAAHPUK5fUEsFBgAAAAAGAAYATAIAACIFAAAoAGQ1YWExZDJiOGE1MzRmMzdjZDkzYmU0OGIyMTRmNDkwZWY5ZWU5MDQ='
|
|
|
|
describe('utils', () => {
|
|
test('bufferToStream', () => {
|
|
const buffer = Buffer.from(zip, 'base64')
|
|
const stream = bufferToStream(buffer)
|
|
expect(stream.read()).toEqual(buffer)
|
|
})
|
|
|
|
test('setLogLevel', () => {
|
|
function resetMocks() {
|
|
console.debug = jest.fn()
|
|
console.info = jest.fn()
|
|
console.log = jest.fn()
|
|
console.warn = jest.fn()
|
|
console.error = jest.fn()
|
|
}
|
|
|
|
resetMocks()
|
|
setLogLevel(LogLevel.Debug)
|
|
console.debug('debug')
|
|
console.info('debug')
|
|
console.log('debug')
|
|
console.warn('debug')
|
|
console.error('debug')
|
|
expect(console.debug).toHaveBeenCalledWith('debug')
|
|
expect(console.info).toHaveBeenCalledWith('debug')
|
|
expect(console.log).toHaveBeenCalledWith('debug')
|
|
expect(console.warn).toHaveBeenCalledWith('debug')
|
|
expect(console.error).toHaveBeenCalledWith('debug')
|
|
|
|
resetMocks()
|
|
setLogLevel(LogLevel.Info)
|
|
console.debug('info')
|
|
console.info('info')
|
|
console.log('info')
|
|
console.warn('info')
|
|
console.error('info')
|
|
expect((console.debug as any)._original).toBeDefined()
|
|
expect(console.info).toHaveBeenCalledWith('info')
|
|
expect(console.log).toHaveBeenCalledWith('info')
|
|
expect(console.warn).toHaveBeenCalledWith('info')
|
|
expect(console.error).toHaveBeenCalledWith('info')
|
|
|
|
resetMocks()
|
|
setLogLevel(LogLevel.Log)
|
|
console.debug('log')
|
|
console.info('log')
|
|
console.log('log')
|
|
console.warn('log')
|
|
console.error('log')
|
|
expect((console.debug as any)._original).toBeDefined()
|
|
expect((console.info as any)._original).toBeDefined()
|
|
expect(console.log).toHaveBeenCalledWith('log')
|
|
expect(console.warn).toHaveBeenCalledWith('log')
|
|
expect(console.error).toHaveBeenCalledWith('log')
|
|
|
|
resetMocks()
|
|
setLogLevel(LogLevel.Warn)
|
|
console.debug('warn')
|
|
console.info('warn')
|
|
console.log('warn')
|
|
console.warn('warn')
|
|
console.error('warn')
|
|
expect((console.debug as any)._original).toBeDefined()
|
|
expect((console.info as any)._original).toBeDefined()
|
|
expect((console.log as any)._original).toBeDefined()
|
|
expect(console.warn).toHaveBeenCalledWith('warn')
|
|
expect(console.error).toHaveBeenCalledWith('warn')
|
|
|
|
resetMocks()
|
|
setLogLevel(LogLevel.Error)
|
|
console.debug('error')
|
|
console.info('error')
|
|
console.log('error')
|
|
console.warn('error')
|
|
console.error('error')
|
|
expect((console.debug as any)._original).toBeDefined()
|
|
expect((console.info as any)._original).toBeDefined()
|
|
expect((console.log as any)._original).toBeDefined()
|
|
expect((console.warn as any)._original).toBeDefined()
|
|
expect(console.error).toHaveBeenCalledWith('error')
|
|
|
|
resetMocks()
|
|
setLogLevel(LogLevel.None)
|
|
console.debug('none')
|
|
console.info('none')
|
|
console.log('none')
|
|
console.warn('none')
|
|
console.error('none')
|
|
expect((console.debug as any)._original).toBeDefined()
|
|
expect((console.info as any)._original).toBeDefined()
|
|
expect((console.log as any)._original).toBeDefined()
|
|
expect((console.warn as any)._original).toBeDefined()
|
|
expect((console.error as any)._original).toBeDefined()
|
|
})
|
|
|
|
test('cloneObject', () => {
|
|
const o1 = ['string', 'value']
|
|
expect(cloneObject(o1)).toEqual(o1)
|
|
expect(cloneObject(o1) === o1).toBe(false)
|
|
|
|
const o2 = { key: 'value' }
|
|
expect(cloneObject(o2)).toEqual(o2)
|
|
expect(cloneObject(o2) === o2).toBe(false)
|
|
|
|
const o3 = { key: 'value', nested: ['a1', 'a2'], nestedObj: { key: 'other' } }
|
|
expect(cloneObject(o3)).toEqual(o3)
|
|
expect(cloneObject(o3) === o3).toBe(false)
|
|
expect((cloneObject(o3) as typeof o3).nested === o3.nested).toBe(false)
|
|
expect((cloneObject(o3) as typeof o3).nestedObj === o3.nestedObj).toBe(false)
|
|
|
|
const o4 = null
|
|
expect(cloneObject(o4)).toEqual(o4)
|
|
expect(cloneObject(o4) === o4).toBe(true)
|
|
|
|
const o5 = 'string'
|
|
expect(cloneObject(o5)).toEqual(o5)
|
|
expect(cloneObject(o5) === o5).toBe(true)
|
|
})
|
|
|
|
describe('UUID', () => {
|
|
describe('#constructor', () => {
|
|
it('works with a valid string', () => {
|
|
const uuid = new UUID('99aBcDeF-1234-4321-0000-dcba87654321')
|
|
expect(uuid.array).toStrictEqual(
|
|
new Uint8Array([
|
|
0x99, 0xab, 0xcd, 0xef, 0x12, 0x34, 0x43, 0x21, 0, 0, 0xdc, 0xba, 0x87, 0x65, 0x43, 0x21,
|
|
])
|
|
)
|
|
})
|
|
|
|
it('throws on an invalid string', () => {
|
|
expect(() => new UUID('99aBcDeF-1234-4321-WXyz-dcba87654321')).toThrow() // "WXyz" are not hexadecimal
|
|
expect(() => new UUID('99aBcDeF123443210000dcba87654321')).toThrow() // lack of hyphens
|
|
expect(() => new UUID('99aBcDeF-1234-4321-0000-dcba87654321A')).toThrow() // one character too many
|
|
expect(() => new UUID('99aBcDeF-1234-4321-0000-dcba8765432')).toThrow() // one character too few
|
|
expect(() => new UUID('')).toThrow() // empty string
|
|
})
|
|
|
|
it('works with a Uint8Array', () => {
|
|
for (let i = 0; i < 10; i++) {
|
|
const uuid = new UUID(
|
|
new Uint8Array([
|
|
0x99, 0xab, 0xcd, 0xef, 0x12, 0x34, 0x43, 0x21, 0, 0, 0xdc, 0xba, 0x87, 0x65, 0x43, 0x21,
|
|
])
|
|
)
|
|
expect(uuid.array).toStrictEqual(
|
|
new Uint8Array([
|
|
0x99, 0xab, 0xcd, 0xef, 0x12, 0x34, 0x43, 0x21, 0, 0, 0xdc, 0xba, 0x87, 0x65, 0x43, 0x21,
|
|
])
|
|
)
|
|
}
|
|
})
|
|
|
|
it('works with a random buffer', () => {
|
|
for (let i = 0; i < 10; i++) {
|
|
const uuid = new UUID(randomBytes(16))
|
|
expect(uuid.array).toHaveLength(16)
|
|
}
|
|
})
|
|
})
|
|
|
|
describe('#valueOf', () => {
|
|
it('returns the right big integer', () => {
|
|
const uuid = new UUID('99aBcDeF-1234-4321-0000-dcba87654321')
|
|
|
|
expect(uuid.valueOf()).toStrictEqual(0x99abcdef123443210000dcba87654321n)
|
|
})
|
|
})
|
|
|
|
describe('#toString', () => {
|
|
it('returns the right string', () => {
|
|
const original = '99aBcDeF-1234-4321-0000-dcba87654321'
|
|
const uuid = new UUID(original)
|
|
const uuidString = uuid.toString()
|
|
// 32 hexadecimal digits + 4 hyphens
|
|
expect(uuidString).toHaveLength(36)
|
|
expect(uuidString).toStrictEqual(original.toLowerCase())
|
|
})
|
|
})
|
|
})
|
|
|
|
describe('UUIDT', () => {
|
|
it('is well-formed', () => {
|
|
const uuidt = new UUIDT()
|
|
const uuidtString = uuidt.toString()
|
|
// UTC timestamp matching (roughly, only comparing the beginning as the timestamp's end inevitably drifts away)
|
|
expect(uuidtString.slice(0, 8)).toEqual(Date.now().toString(16).padStart(12, '0').slice(0, 8))
|
|
// series matching
|
|
expect(uuidtString.slice(14, 18)).toEqual('0000')
|
|
})
|
|
})
|
|
|
|
describe('sanitizeSqlIdentifier', () => {
|
|
it('removes all characters that are neither letter, digit or underscore and adds quotes around identifier', () => {
|
|
const rawIdentifier = 'some_field"; DROP TABLE actually_an_injection-9;'
|
|
|
|
const sanitizedIdentifier = sanitizeSqlIdentifier(rawIdentifier)
|
|
|
|
expect(sanitizedIdentifier).toStrictEqual('some_fieldDROPTABLEactually_an_injection9')
|
|
})
|
|
})
|
|
|
|
describe('escapeClickHouseString', () => {
|
|
it('escapes single quotes and slashes', () => {
|
|
const rawString = "insert'escape \\"
|
|
|
|
const sanitizedString = escapeClickHouseString(rawString)
|
|
|
|
expect(sanitizedString).toStrictEqual("insert\\'escape \\\\")
|
|
})
|
|
})
|
|
|
|
describe('groupBy', () => {
|
|
it('groups simple objects', () => {
|
|
const objects = [
|
|
{ i: 2, foo: 'x' },
|
|
{ i: 2, foo: 'y' },
|
|
{ i: 4, foo: 'x' },
|
|
{ i: 7, foo: 'z' },
|
|
]
|
|
|
|
const groupingByI = groupBy(objects, 'i')
|
|
expect(groupingByI).toEqual({
|
|
2: [
|
|
{ i: 2, foo: 'x' },
|
|
{ i: 2, foo: 'y' },
|
|
],
|
|
4: [{ i: 4, foo: 'x' }],
|
|
7: [{ i: 7, foo: 'z' }],
|
|
})
|
|
|
|
const groupingByFoo = groupBy(objects, 'foo')
|
|
expect(groupingByFoo).toEqual({
|
|
x: [
|
|
{ i: 2, foo: 'x' },
|
|
{ i: 4, foo: 'x' },
|
|
],
|
|
y: [{ i: 2, foo: 'y' }],
|
|
z: [{ i: 7, foo: 'z' }],
|
|
})
|
|
})
|
|
|
|
it('handles undefineds', () => {
|
|
const objects = [{ i: 2, foo: 'x' }, { i: 2, foo: 'y' }, { i: 4, foo: 'x' }, { foo: 'z' }]
|
|
|
|
const groupingByI = groupBy(objects, 'i')
|
|
expect(groupingByI).toEqual({
|
|
2: [
|
|
{ i: 2, foo: 'x' },
|
|
{ i: 2, foo: 'y' },
|
|
],
|
|
4: [{ i: 4, foo: 'x' }],
|
|
undefined: [{ foo: 'z' }],
|
|
})
|
|
})
|
|
|
|
it('works in flat mode', () => {
|
|
const objects = [
|
|
{ i: 2, foo: 'x' },
|
|
{ i: 4, foo: 'x' },
|
|
{ i: 7, foo: 'z' },
|
|
]
|
|
|
|
const groupingByI = groupBy(objects, 'i', true)
|
|
expect(groupingByI).toEqual({
|
|
2: { i: 2, foo: 'x' },
|
|
4: { i: 4, foo: 'x' },
|
|
7: { i: 7, foo: 'z' },
|
|
})
|
|
})
|
|
|
|
it("doesn't work in flat mode if multiple values match a single key", () => {
|
|
const objects = [
|
|
{ i: 2, foo: 'x' },
|
|
{ i: 2, foo: 'y' },
|
|
{ i: 4, foo: 'x' },
|
|
{ i: 7, foo: 'z' },
|
|
]
|
|
|
|
expect(() => groupBy(objects, 'i', true)).toThrowError(
|
|
'Key "i" has more than one matching value, which is not allowed in flat groupBy!'
|
|
)
|
|
})
|
|
})
|
|
|
|
describe('stringify', () => {
|
|
it('leaves strings unaffected', () => {
|
|
expect(stringify('test!')).toStrictEqual('test!')
|
|
})
|
|
|
|
it('transforms numbers into strings', () => {
|
|
expect(stringify(3)).toStrictEqual('3')
|
|
expect(stringify(21.37)).toStrictEqual('21.37')
|
|
})
|
|
|
|
it('transforms nullish values into strings', () => {
|
|
expect(stringify(null)).toStrictEqual('null')
|
|
expect(stringify(undefined)).toStrictEqual('undefined')
|
|
})
|
|
|
|
it('transforms object values into strings', () => {
|
|
expect(stringify({})).toStrictEqual('{}')
|
|
expect(stringify([])).toStrictEqual('[]')
|
|
})
|
|
})
|
|
|
|
describe('safeClickhouseString', () => {
|
|
// includes real data
|
|
const validStrings = [
|
|
`$autocapture`,
|
|
`correlation analyzed`,
|
|
`docs_search_used`,
|
|
`$$plugin_metrics`,
|
|
`996f3e2f-830b-42f0-b2b8-df42bb7f7144`,
|
|
`some?819)389**^371=2++211!!@==-''''..,,weird___id`,
|
|
`form.form-signin:attr__action="/signup"attr__class="form-signin"attr__method="post"nth-child="1"nth-of-type="1";body:nth-child="2"nth-of-type="1"`,
|
|
`a:attr__href="/signup"href="/signup"nth-child="1"nth-of-type="1"text="Create one here.";p:nth-child="8"nth-of-type="1";form.form-signin:attr__action="/login"attr__class="form-signin"attr__method="post"nth-child="1"nth-of-type="1";body:nth-child="2"nth-of-type="1"`,
|
|
`input:nth-child="7"nth-of-type="3";form.form-signin:attr__action="/signup"attr__class="form-signin"attr__method="post"nth-child="1"nth-of-type="1";body:nth-child="2"nth-of-type="1"`,
|
|
`a.nav-link:attr__class="nav-link"attr__href="/actions"href="/actions"nth-child="1"nth-of-type="1"text="Actions";li:nth-child="2"nth-of-type="2";ul.flex-sm-column.nav:attr__class="nav flex-sm-column"nth-child="1"nth-of-type="1";div.bg-light.col-md-2.col-sm-3.flex-shrink-1.pt-3.sidebar:attr__class="col-sm-3 col-md-2 sidebar flex-shrink-1 bg-light pt-3"attr__style="min-height: 100vh;"nth-child="1"nth-of-type="1";div.flex-column.flex-fill.flex-sm-row.row:attr__class="row flex-fill flex-column flex-sm-row"nth-child="1"nth-of-type="1";div.container-fluid.d-flex.flex-grow-1:attr__class="container-fluid flex-grow-1 d-flex"nth-child="1"nth-of-type="1";div:attr__id="root"attr_id="root"nth-child="1"nth-of-type="1";body:nth-child="2"nth-of-type="1"`,
|
|
]
|
|
|
|
test('does not modify valid strings', () => {
|
|
for (const str of validStrings) {
|
|
expect(safeClickhouseString(str)).toEqual(str)
|
|
}
|
|
})
|
|
|
|
test('handles surrogate unicode characters correctly', () => {
|
|
expect(safeClickhouseString(`foo \ud83d\ bar`)).toEqual(`foo \\ud83d\\ bar`)
|
|
expect(safeClickhouseString(`\ud83d\ bar`)).toEqual(`\\ud83d\\ bar`)
|
|
expect(safeClickhouseString(`\ud800\ \ud803\ `)).toEqual(`\\ud800\\ \\ud803\\ `)
|
|
})
|
|
|
|
test('does not modify non-surrogate unicode characters', () => {
|
|
expect(safeClickhouseString(`✨`)).toEqual(`✨`)
|
|
expect(safeClickhouseString(`foo \u2728\ bar`)).toEqual(`foo \u2728\ bar`)
|
|
expect(safeClickhouseString(`💜 \u1f49c\ 💜`)).toEqual(`💜 \u1f49c\ 💜`)
|
|
})
|
|
})
|
|
|
|
describe('clickHouseTimestampToDateTime()', () => {
|
|
it('casts to a datetime', () => {
|
|
expect(clickHouseTimestampToDateTime('2020-02-23 02:15:00.00' as ClickHouseTimestamp)).toEqual(
|
|
DateTime.fromISO('2020-02-23T02:15:00.000Z').toUTC()
|
|
)
|
|
})
|
|
})
|
|
})
|