import { randomBytes } from 'crypto' import { DateTime } from 'luxon' import { ClickHouseTimestamp } from '../src/types' import { safeClickhouseString } from '../src/utils/db/utils' import { bufferToStream, clickHouseTimestampToDateTime, cloneObject, escapeClickHouseString, groupBy, sanitizeSqlIdentifier, stringify, UUID, UUIDT, } from '../src/utils/utils' // .zip in Base64: github repo posthog/helloworldplugin const zip = 'UEsDBAoAAAAAAA8LbVEAAAAAAAAAAAAAAAAjAAkAaGVsbG93b3JsZHBsdWdpbi1pbWFnZWxlc3MtdmVyc2lvbi9VVAUAAc9Qrl9QSwMECgAAAAgADwttUQT7a+JUAAAAdQAAAC4ACQBoZWxsb3dvcmxkcGx1Z2luLWltYWdlbGVzcy12ZXJzaW9uLy5wcmV0dGllcnJjVVQFAAHPUK5fq+ZSAAKlkqLEzJzMvHTn/NzcRCUrBaXUYlMlHahcYlJ4ZkpJBlDYBCpUnJqbCeSmJeYUp8KEgLpzUgNL80tSgTIlRaUwiYKizLwSmAGGRgZctVwAUEsDBAoAAAAIAA8LbVG/Zg9y6wAAAPMBAAArAAkAaGVsbG93b3JsZHBsdWdpbi1pbWFnZWxlc3MtdmVyc2lvbi9pbmRleC5qc1VUBQABz1CuX31RTUvEMBC951eMi5AUS1E8e9Sz4FFE0jjNBrKTkkxcZOl/N9u0hyo0hyHDe/PefOj0QwaGTIZdIEjIeXz12TpSFzCBBmdhauAioLySp+Cx88GqwxsyO7KQR+AjAqM+3Ryaf7yq0YhJCL31GmMwmNLzNxIrvMYWVs8WjDZFdWPNJWZijPAE+qwdV1JnkZVcINnC/dLEjKUttgrcwUMjZpoboJp3pZ8RIztMq+n1/cXe5RG9D/KjNCHPIfovucPtdZyZdaqupDvk27XPWjH/d+je9Z+UT/1SUNKXZbXqsa5gqiPGctRIVaHc4RdQSwMECgAAAAgADwttUVpiCFkvAAAAOQAAACkACQBoZWxsb3dvcmxkcGx1Z2luLWltYWdlbGVzcy12ZXJzaW9uL2xpYi5qc1VUBQABz1CuX0srzUsuyczPU8jJTHKDsTXySnOTUos0Faq5FICgKLWktChPASKooKVgZM1VywUAUEsDBAoAAAAIAA8LbVE7Twti+wAAAO0BAAAvAAkAaGVsbG93b3JsZHBsdWdpbi1pbWFnZWxlc3MtdmVyc2lvbi9wYWNrYWdlLmpzb25VVAUAAc9Qrl+VUbtOAzEQ7O8rjJHSQO6S9noEFEgUlGkud4vtyOe1dm1IFOXf8eNE0qazZ2ZnvONzI4R0wwyyF9IjB41qrcFa/EWy09rbqIyTz1n2A8QGXVZu2k27regEPJLxYWFeCSCIoEEUg4cqmgdTWOMmOLYHrmgd5ESc0zUBAThkGYwaxU6+ECH1wqHIhGAPo/k2MO2kWK0EHE0QW5kmL8WNIL3fBKTTjeHJl82UCSUyQZHsgjzpEDz3XZfOOu7bEefuM1Xwhqq7VlAbaLPDf9QQU0+Ubeoi1ozguCR9vH9VbB/VzWZL6h2JnWGOwNdQjTP4QcGdPo8Ew5T+t7k0f1BLAwQKAAAACAAPC21Ru8C8oc0AAABTAQAALgAJAGhlbGxvd29ybGRwbHVnaW4taW1hZ2VsZXNzLXZlcnNpb24vcGx1Z2luLmpzb25VVAUAAc9Qrl9tjz1rwzAQhnf/iquXLCHesxQytKVToEPms3WWr8g6VzqRL/LfKymQdOggxPu8zwvStQFoPc7UbqGdyDk5SnBmccmyb9elTcHVUnWJ266zrFPqN4PM3V6ifojt/t8ZikPgRVl82b8HIgWdCA7FBPQG3kQAYYdhDZ9fQIaL/HKfz8h1x97QafMd79RxX2C+HmgQP7LN9JpTzj2GR/jzucOEuorAvr4hS691Xh09L9WJGtjbJzc0YnJaqh4vTx7oJ3Egk4sRXaTKb005t+YXUEsBAgAACgAAAAAADwttUQAAAAAAAAAAAAAAACMACQAAAAAAAAAQAAAAAAAAAGhlbGxvd29ybGRwbHVnaW4taW1hZ2VsZXNzLXZlcnNpb24vVVQFAAHPUK5fUEsBAgAACgAAAAgADwttUQT7a+JUAAAAdQAAAC4ACQAAAAAAAQAAAAAASgAAAGhlbGxvd29ybGRwbHVnaW4taW1hZ2VsZXNzLXZlcnNpb24vLnByZXR0aWVycmNVVAUAAc9Qrl9QSwECAAAKAAAACAAPC21Rv2YPcusAAADzAQAAKwAJAAAAAAABAAAAAADzAAAAaGVsbG93b3JsZHBsdWdpbi1pbWFnZWxlc3MtdmVyc2lvbi9pbmRleC5qc1VUBQABz1CuX1BLAQIAAAoAAAAIAA8LbVFaYghZLwAAADkAAAApAAkAAAAAAAEAAAAAADACAABoZWxsb3dvcmxkcGx1Z2luLWltYWdlbGVzcy12ZXJzaW9uL2xpYi5qc1VUBQABz1CuX1BLAQIAAAoAAAAIAA8LbVE7Twti+wAAAO0BAAAvAAkAAAAAAAEAAAAAAK8CAABoZWxsb3dvcmxkcGx1Z2luLWltYWdlbGVzcy12ZXJzaW9uL3BhY2thZ2UuanNvblVUBQABz1CuX1BLAQIAAAoAAAAIAA8LbVG7wLyhzQAAAFMBAAAuAAkAAAAAAAEAAAAAAAAEAABoZWxsb3dvcmxkcGx1Z2luLWltYWdlbGVzcy12ZXJzaW9uL3BsdWdpbi5qc29uVVQFAAHPUK5fUEsFBgAAAAAGAAYATAIAACIFAAAoAGQ1YWExZDJiOGE1MzRmMzdjZDkzYmU0OGIyMTRmNDkwZWY5ZWU5MDQ=' describe('utils', () => { test('bufferToStream', () => { const buffer = Buffer.from(zip, 'base64') const stream = bufferToStream(buffer) expect(stream.read()).toEqual(buffer) }) test('cloneObject', () => { const o1 = ['string', 'value'] expect(cloneObject(o1)).toEqual(o1) expect(cloneObject(o1) === o1).toBe(false) const o2 = { key: 'value' } expect(cloneObject(o2)).toEqual(o2) expect(cloneObject(o2) === o2).toBe(false) const o3 = { key: 'value', nested: ['a1', 'a2'], nestedObj: { key: 'other' } } expect(cloneObject(o3)).toEqual(o3) expect(cloneObject(o3) === o3).toBe(false) expect((cloneObject(o3) as typeof o3).nested === o3.nested).toBe(false) expect((cloneObject(o3) as typeof o3).nestedObj === o3.nestedObj).toBe(false) const o4 = null expect(cloneObject(o4)).toEqual(o4) expect(cloneObject(o4) === o4).toBe(true) const o5 = 'string' expect(cloneObject(o5)).toEqual(o5) expect(cloneObject(o5) === o5).toBe(true) }) describe('UUID', () => { describe('#constructor', () => { it('works with a valid string', () => { const uuid = new UUID('99aBcDeF-1234-4321-0000-dcba87654321') expect(uuid.array).toStrictEqual( new Uint8Array([ 0x99, 0xab, 0xcd, 0xef, 0x12, 0x34, 0x43, 0x21, 0, 0, 0xdc, 0xba, 0x87, 0x65, 0x43, 0x21, ]) ) }) it('throws on an invalid string', () => { expect(() => new UUID('99aBcDeF-1234-4321-WXyz-dcba87654321')).toThrow() // "WXyz" are not hexadecimal expect(() => new UUID('99aBcDeF123443210000dcba87654321')).toThrow() // lack of hyphens expect(() => new UUID('99aBcDeF-1234-4321-0000-dcba87654321A')).toThrow() // one character too many expect(() => new UUID('99aBcDeF-1234-4321-0000-dcba8765432')).toThrow() // one character too few expect(() => new UUID('')).toThrow() // empty string }) it('works with a Uint8Array', () => { for (let i = 0; i < 10; i++) { const uuid = new UUID( new Uint8Array([ 0x99, 0xab, 0xcd, 0xef, 0x12, 0x34, 0x43, 0x21, 0, 0, 0xdc, 0xba, 0x87, 0x65, 0x43, 0x21, ]) ) expect(uuid.array).toStrictEqual( new Uint8Array([ 0x99, 0xab, 0xcd, 0xef, 0x12, 0x34, 0x43, 0x21, 0, 0, 0xdc, 0xba, 0x87, 0x65, 0x43, 0x21, ]) ) } }) it('works with a random buffer', () => { for (let i = 0; i < 10; i++) { const uuid = new UUID(randomBytes(16)) expect(uuid.array).toHaveLength(16) } }) }) describe('#valueOf', () => { it('returns the right big integer', () => { const uuid = new UUID('99aBcDeF-1234-4321-0000-dcba87654321') expect(uuid.valueOf()).toStrictEqual(0x99abcdef123443210000dcba87654321n) }) }) describe('#toString', () => { it('returns the right string', () => { const original = '99aBcDeF-1234-4321-0000-dcba87654321' const uuid = new UUID(original) const uuidString = uuid.toString() // 32 hexadecimal digits + 4 hyphens expect(uuidString).toHaveLength(36) expect(uuidString).toStrictEqual(original.toLowerCase()) }) }) }) describe('UUIDT', () => { it('is well-formed', () => { const uuidt = new UUIDT() const uuidtString = uuidt.toString() // UTC timestamp matching (roughly, only comparing the beginning as the timestamp's end inevitably drifts away) expect(uuidtString.slice(0, 8)).toEqual(Date.now().toString(16).padStart(12, '0').slice(0, 8)) // series matching expect(uuidtString.slice(14, 18)).toEqual('0000') }) }) describe('sanitizeSqlIdentifier', () => { it('removes all characters that are neither letter, digit or underscore and adds quotes around identifier', () => { const rawIdentifier = 'some_field"; DROP TABLE actually_an_injection-9;' const sanitizedIdentifier = sanitizeSqlIdentifier(rawIdentifier) expect(sanitizedIdentifier).toStrictEqual('some_fieldDROPTABLEactually_an_injection9') }) }) describe('escapeClickHouseString', () => { it('escapes single quotes and slashes', () => { const rawString = "insert'escape \\" const sanitizedString = escapeClickHouseString(rawString) expect(sanitizedString).toStrictEqual("insert\\'escape \\\\") }) }) describe('groupBy', () => { it('groups simple objects', () => { const objects = [ { i: 2, foo: 'x' }, { i: 2, foo: 'y' }, { i: 4, foo: 'x' }, { i: 7, foo: 'z' }, ] const groupingByI = groupBy(objects, 'i') expect(groupingByI).toEqual({ 2: [ { i: 2, foo: 'x' }, { i: 2, foo: 'y' }, ], 4: [{ i: 4, foo: 'x' }], 7: [{ i: 7, foo: 'z' }], }) const groupingByFoo = groupBy(objects, 'foo') expect(groupingByFoo).toEqual({ x: [ { i: 2, foo: 'x' }, { i: 4, foo: 'x' }, ], y: [{ i: 2, foo: 'y' }], z: [{ i: 7, foo: 'z' }], }) }) it('handles undefineds', () => { const objects = [{ i: 2, foo: 'x' }, { i: 2, foo: 'y' }, { i: 4, foo: 'x' }, { foo: 'z' }] const groupingByI = groupBy(objects, 'i') expect(groupingByI).toEqual({ 2: [ { i: 2, foo: 'x' }, { i: 2, foo: 'y' }, ], 4: [{ i: 4, foo: 'x' }], undefined: [{ foo: 'z' }], }) }) it('works in flat mode', () => { const objects = [ { i: 2, foo: 'x' }, { i: 4, foo: 'x' }, { i: 7, foo: 'z' }, ] const groupingByI = groupBy(objects, 'i', true) expect(groupingByI).toEqual({ 2: { i: 2, foo: 'x' }, 4: { i: 4, foo: 'x' }, 7: { i: 7, foo: 'z' }, }) }) it("doesn't work in flat mode if multiple values match a single key", () => { const objects = [ { i: 2, foo: 'x' }, { i: 2, foo: 'y' }, { i: 4, foo: 'x' }, { i: 7, foo: 'z' }, ] expect(() => groupBy(objects, 'i', true)).toThrowError( 'Key "i" has more than one matching value, which is not allowed in flat groupBy!' ) }) }) describe('stringify', () => { it('leaves strings unaffected', () => { expect(stringify('test!')).toStrictEqual('test!') }) it('transforms numbers into strings', () => { expect(stringify(3)).toStrictEqual('3') expect(stringify(21.37)).toStrictEqual('21.37') }) it('transforms nullish values into strings', () => { expect(stringify(null)).toStrictEqual('null') expect(stringify(undefined)).toStrictEqual('undefined') }) it('transforms object values into strings', () => { expect(stringify({})).toStrictEqual('{}') expect(stringify([])).toStrictEqual('[]') }) }) describe('safeClickhouseString', () => { // includes real data const validStrings = [ `$autocapture`, `correlation analyzed`, `docs_search_used`, `$$plugin_metrics`, `996f3e2f-830b-42f0-b2b8-df42bb7f7144`, `some?819)389**^371=2++211!!@==-''''..,,weird___id`, `form.form-signin:attr__action="/signup"attr__class="form-signin"attr__method="post"nth-child="1"nth-of-type="1";body:nth-child="2"nth-of-type="1"`, `a:attr__href="/signup"href="/signup"nth-child="1"nth-of-type="1"text="Create one here.";p:nth-child="8"nth-of-type="1";form.form-signin:attr__action="/login"attr__class="form-signin"attr__method="post"nth-child="1"nth-of-type="1";body:nth-child="2"nth-of-type="1"`, `input:nth-child="7"nth-of-type="3";form.form-signin:attr__action="/signup"attr__class="form-signin"attr__method="post"nth-child="1"nth-of-type="1";body:nth-child="2"nth-of-type="1"`, `a.nav-link:attr__class="nav-link"attr__href="/actions"href="/actions"nth-child="1"nth-of-type="1"text="Actions";li:nth-child="2"nth-of-type="2";ul.flex-sm-column.nav:attr__class="nav flex-sm-column"nth-child="1"nth-of-type="1";div.bg-light.col-md-2.col-sm-3.flex-shrink-1.pt-3.sidebar:attr__class="col-sm-3 col-md-2 sidebar flex-shrink-1 bg-light pt-3"attr__style="min-height: 100vh;"nth-child="1"nth-of-type="1";div.flex-column.flex-fill.flex-sm-row.row:attr__class="row flex-fill flex-column flex-sm-row"nth-child="1"nth-of-type="1";div.container-fluid.d-flex.flex-grow-1:attr__class="container-fluid flex-grow-1 d-flex"nth-child="1"nth-of-type="1";div:attr__id="root"attr_id="root"nth-child="1"nth-of-type="1";body:nth-child="2"nth-of-type="1"`, ] test('does not modify valid strings', () => { for (const str of validStrings) { expect(safeClickhouseString(str)).toEqual(str) } }) test('handles surrogate unicode characters correctly', () => { expect(safeClickhouseString(`foo \ud83d\ bar`)).toEqual(`foo \\ud83d\\ bar`) expect(safeClickhouseString(`\ud83d\ bar`)).toEqual(`\\ud83d\\ bar`) expect(safeClickhouseString(`\ud800\ \ud803\ `)).toEqual(`\\ud800\\ \\ud803\\ `) }) test('does not modify non-surrogate unicode characters', () => { expect(safeClickhouseString(`✨`)).toEqual(`✨`) expect(safeClickhouseString(`foo \u2728\ bar`)).toEqual(`foo \u2728\ bar`) expect(safeClickhouseString(`💜 \u1f49c\ 💜`)).toEqual(`💜 \u1f49c\ 💜`) }) }) describe('clickHouseTimestampToDateTime()', () => { it('casts to a datetime', () => { expect(clickHouseTimestampToDateTime('2020-02-23 02:15:00.00' as ClickHouseTimestamp)).toEqual( DateTime.fromISO('2020-02-23T02:15:00.000Z').toUTC() ) }) }) })