mirrors/posthog
0
0
Fork 0
mirror of https://github.com/PostHog/posthog.git synced 2024-12-01 12:21:02 +01:00
Code Issues Releases Activity

feat: Add option to ignore timestamp adjustments (#19105)

Browse Source
This commit is contained in:
Tiina Turban 2023-12-06 14:48:21 +01:00 committed by GitHub
parent 9065466bd0
commit 0e1da1bfcf
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 64 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
plugin-server/src/worker/ingestion/timestamps.ts
View File

@ -12,7 +12,7 @@ export function parseEventTimestamp(data: PluginEvent, callback?: IngestionWarni
const now = DateTime.fromISO(data['now']).toUTC() // now is set by the capture endpoint and assumed valid
let sentAt: DateTime | null = null
if (data['sent_at']) {
if (!data.properties?.['$ignore_sent_at'] && data['sent_at']) {
sentAt = DateTime.fromISO(data['sent_at']).toUTC()
if (!sentAt.isValid) {
callback?.('ignored_invalid_timestamp', {
@ -25,7 +25,26 @@ export function parseEventTimestamp(data: PluginEvent, callback?: IngestionWarni
}
}
const parsedTs = handleTimestamp(data, now, sentAt, data.team_id, callback)
let parsedTs = handleTimestamp(data, now, sentAt, data.team_id)
// Events in the future would indicate an instrumentation bug, lets' ingest them
// but publish an integration warning to help diagnose such issues.
// We will also 'fix' the date to be now()
const nowDiff = parsedTs.toUTC().diff(now).toMillis()
if (nowDiff > FutureEventHoursCutoffMillis) {
callback?.('event_timestamp_in_future', {
timestamp: data['timestamp'] ?? '',
sentAt: data['sent_at'] ?? '',
offset: data['offset'] ?? '',
now: data['now'],
result: parsedTs.toISO(),
eventUuid: data['uuid'],
eventName: data['event'],
})
parsedTs = now
}
if (!parsedTs.isValid) {
callback?.('ignored_invalid_timestamp', {
eventUuid: data['uuid'] ?? '',
@ -39,13 +58,7 @@ export function parseEventTimestamp(data: PluginEvent, callback?: IngestionWarni
return parsedTs
}
function handleTimestamp(
data: PluginEvent,
now: DateTime,
sentAt: DateTime | null,
teamId: number,
callback?: IngestionWarningCallback
): DateTime {
function handleTimestamp(data: PluginEvent, now: DateTime, sentAt: DateTime | null, teamId: number): DateTime {
let parsedTs: DateTime = now
let timestamp: DateTime = now
@ -88,25 +101,6 @@ function handleTimestamp(
parsedTs = now.minus(Duration.fromMillis(data['offset']))
}
const nowDiff = parsedTs.toUTC().diff(now).toMillis()
// Events in the future would indicate an instrumentation bug, lets' ingest them
// but publish an integration warning to help diagnose such issues.
// We will also 'fix' the date to be now()
if (nowDiff > FutureEventHoursCutoffMillis) {
callback?.('event_timestamp_in_future', {
timestamp: data['timestamp'] ?? '',
sentAt: data['sent_at'] ?? '',
offset: data['offset'] ?? '',
now: data['now'],
result: parsedTs.toISO(),
eventUuid: data['uuid'],
eventName: data['event'],
})
parsedTs = now
}
return parsedTs
}

42
plugin-server/tests/worker/ingestion/timestamps.test.ts
View File

@ -51,6 +51,21 @@ describe('parseEventTimestamp()', () => {
expect(timestamp.toISO()).toEqual('2021-10-29T01:34:00.000Z')
})
it('Ignores sent_at if $ignore_sent_at set', () => {
const event = {
properties: { $ignore_sent_at: true },
timestamp: '2021-10-30T03:02:00.000Z',
sent_at: '2021-10-30T03:12:00.000Z',
now: '2021-11-29T01:44:00.000Z',
} as any as PluginEvent
const callbackMock = jest.fn()
const timestamp = parseEventTimestamp(event, callbackMock)
expect(callbackMock.mock.calls.length).toEqual(0)
expect(timestamp.toISO()).toEqual('2021-10-30T03:02:00.000Z')
})
it('ignores and reports invalid sent_at', () => {
const event = {
timestamp: '2021-10-31T00:44:00.000Z',
@ -184,6 +199,33 @@ describe('parseEventTimestamp()', () => {
expect(timestamp.toISO()).toEqual('2021-10-29T01:00:00.000Z')
})
it('reports event_timestamp_in_future with $ignore_sent_at', () => {
const event = {
timestamp: '2021-10-29T02:30:00.000Z',
now: '2021-09-29T01:00:00.000Z',
event: 'test event name',
uuid: '12345678-1234-1234-1234-123456789abc',
} as any as PluginEvent
const callbackMock = jest.fn()
const timestamp = parseEventTimestamp(event, callbackMock)
expect(callbackMock.mock.calls).toEqual([
[
'event_timestamp_in_future',
{
now: '2021-09-29T01:00:00.000Z',
offset: '',
result: '2021-10-29T02:30:00.000Z',
sentAt: '',
timestamp: '2021-10-29T02:30:00.000Z',
eventUuid: '12345678-1234-1234-1234-123456789abc',
eventName: 'test event name',
},
],
])
expect(timestamp.toISO()).toEqual('2021-09-29T01:00:00.000Z')
})
it('reports event_timestamp_in_future with negative offset', () => {
const event = {
offset: -82860000,