mirrors/posthog
0
0
Fork 0
mirror of https://github.com/PostHog/posthog.git synced 2024-11-24 09:14:46 +01:00
Code Issues Releases Activity

feat(experiments): Calculate secondary metric credible interval (#26138)

Browse Source
This commit is contained in:
Daniel Bachhuber 2024-11-13 16:32:55 -08:00 committed by GitHub
parent 854bdb161c
commit c4df3e0ee2
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 97 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
frontend/src/scenes/experiments/ExperimentView/SecondaryMetricsTable.tsx
View File

@ -131,6 +131,7 @@ export function SecondaryMetricsTable({
countDataForVariant,
exposureCountDataForVariant,
conversionRateForVariant,
credibleIntervalForVariant,
experimentMathAggregationForTrends,
getHighestProbabilityVariant,
} = useValues(experimentLogic({ experimentId }))
@ -223,6 +224,24 @@ export function SecondaryMetricsTable({
)
},
},
{
title: 'Credible interval (95%)',
render: function Key(_, item: TabularSecondaryMetricResults): JSX.Element {
if (item.variant === 'control') {
return <em>Baseline</em>
}
const credibleInterval = credibleIntervalForVariant(targetResults || null, item.variant)
if (!credibleInterval) {
return <>—</>
}
const [lowerBound, upperBound] = credibleInterval
return (
<div className="font-semibold">{`[${lowerBound > 0 ? '+' : ''}${lowerBound.toFixed(
2
)}%, ${upperBound > 0 ? '+' : ''}${upperBound.toFixed(2)}%]`}</div>
)
},
},
{
title: 'Win probability',
render: function Key(_, item: TabularSecondaryMetricResults): JSX.Element {
@ -255,6 +274,25 @@ export function SecondaryMetricsTable({
return <div>{`${conversionRate.toFixed(2)}%`}</div>
},
},
{
title: 'Credible interval (95%)',
render: function Key(_, item: TabularSecondaryMetricResults): JSX.Element {
if (item.variant === 'control') {
return <em>Baseline</em>
}
const credibleInterval = credibleIntervalForVariant(targetResults || null, item.variant)
if (!credibleInterval) {
return <>—</>
}
const [lowerBound, upperBound] = credibleInterval
return (
<div className="font-semibold">{`[${lowerBound > 0 ? '+' : ''}${lowerBound.toFixed(
2
)}%, ${upperBound > 0 ? '+' : ''}${upperBound.toFixed(2)}%]`}</div>
)
},
},
{
title: 'Win probability',
render: function Key(_, item: TabularSecondaryMetricResults): JSX.Element {

59
frontend/src/scenes/experiments/experimentLogic.tsx
View File

@ -105,6 +105,18 @@ export interface ExperimentResultCalculationError {
statusCode: number
}
export interface CachedSecondaryMetricExperimentFunnelsQueryResponse extends CachedExperimentFunnelsQueryResponse {
filters?: {
insight?: InsightType
}
}
export interface CachedSecondaryMetricExperimentTrendsQueryResponse extends CachedExperimentTrendsQueryResponse {
filters?: {
insight?: InsightType
}
}
export const experimentLogic = kea<experimentLogicType>([
props({} as ExperimentLogicProps),
key((props) => props.experimentId || 'new'),
@ -1261,6 +1273,53 @@ export const experimentLogic = kea<experimentLogicType>([
return (variantResults[variantResults.length - 1].count / variantResults[0].count) * 100
},
],
credibleIntervalForVariant: [
() => [],
() =>
(
experimentResults:
| Partial<ExperimentResults['result']>
| CachedSecondaryMetricExperimentFunnelsQueryResponse
| CachedSecondaryMetricExperimentTrendsQueryResponse
| null,
variantKey: string
): [number, number] | null => {
const credibleInterval = experimentResults?.credible_intervals?.[variantKey]
if (!credibleInterval) {
return null
}
if (experimentResults.filters?.insight === InsightType.FUNNELS) {
const controlVariant = (experimentResults.variants as FunnelExperimentVariant[]).find(
({ key }) => key === 'control'
) as FunnelExperimentVariant
const controlConversionRate =
controlVariant.success_count / (controlVariant.success_count + controlVariant.failure_count)
if (!controlConversionRate) {
return null
}
// Calculate the percentage difference between the credible interval bounds of the variant and the control's conversion rate.
// This represents the range in which the true percentage change relative to the control is likely to fall.
const lowerBound = ((credibleInterval[0] - controlConversionRate) / controlConversionRate) * 100
const upperBound = ((credibleInterval[1] - controlConversionRate) / controlConversionRate) * 100
return [lowerBound, upperBound]
}
const controlVariant = (experimentResults.variants as TrendExperimentVariant[]).find(
({ key }) => key === 'control'
) as TrendExperimentVariant
const controlMean = controlVariant.count / controlVariant.absolute_exposure
// Calculate the percentage difference between the credible interval bounds of the variant and the control's mean.
// This represents the range in which the true percentage change relative to the control is likely to fall.
const lowerBound = ((credibleInterval[0] - controlMean) / controlMean) * 100
const upperBound = ((credibleInterval[1] - controlMean) / controlMean) * 100
return [lowerBound, upperBound]
},
],
getIndexForVariant: [
(s) => [s.experimentInsightType],
(experimentInsightType) =>