From c4df3e0ee2a522bc61b28b38d8d8a20e5f73564c Mon Sep 17 00:00:00 2001
From: Daniel Bachhuber <daniel.b@posthog.com>
Date: Wed, 13 Nov 2024 16:32:55 -0800
Subject: [PATCH] feat(experiments): Calculate secondary metric credible
 interval (#26138)

---
 .../ExperimentView/SecondaryMetricsTable.tsx  | 38 ++++++++++++
 .../scenes/experiments/experimentLogic.tsx    | 59 +++++++++++++++++++
 2 files changed, 97 insertions(+)

diff --git a/frontend/src/scenes/experiments/ExperimentView/SecondaryMetricsTable.tsx b/frontend/src/scenes/experiments/ExperimentView/SecondaryMetricsTable.tsx
index 86776c02592..066144566b0 100644
--- a/frontend/src/scenes/experiments/ExperimentView/SecondaryMetricsTable.tsx
+++ b/frontend/src/scenes/experiments/ExperimentView/SecondaryMetricsTable.tsx
@@ -131,6 +131,7 @@ export function SecondaryMetricsTable({
         countDataForVariant,
         exposureCountDataForVariant,
         conversionRateForVariant,
+        credibleIntervalForVariant,
         experimentMathAggregationForTrends,
         getHighestProbabilityVariant,
     } = useValues(experimentLogic({ experimentId }))
@@ -223,6 +224,24 @@ export function SecondaryMetricsTable({
                             )
                         },
                     },
+                    {
+                        title: 'Credible interval (95%)',
+                        render: function Key(_, item: TabularSecondaryMetricResults): JSX.Element {
+                            if (item.variant === 'control') {
+                                return <em>Baseline</em>
+                            }
+                            const credibleInterval = credibleIntervalForVariant(targetResults || null, item.variant)
+                            if (!credibleInterval) {
+                                return <>—</>
+                            }
+                            const [lowerBound, upperBound] = credibleInterval
+                            return (
+                                <div className="font-semibold">{`[${lowerBound > 0 ? '+' : ''}${lowerBound.toFixed(
+                                    2
+                                )}%, ${upperBound > 0 ? '+' : ''}${upperBound.toFixed(2)}%]`}</div>
+                            )
+                        },
+                    },
                     {
                         title: 'Win probability',
                         render: function Key(_, item: TabularSecondaryMetricResults): JSX.Element {
@@ -255,6 +274,25 @@ export function SecondaryMetricsTable({
                             return <div>{`${conversionRate.toFixed(2)}%`}</div>
                         },
                     },
+                    {
+                        title: 'Credible interval (95%)',
+                        render: function Key(_, item: TabularSecondaryMetricResults): JSX.Element {
+                            if (item.variant === 'control') {
+                                return <em>Baseline</em>
+                            }
+
+                            const credibleInterval = credibleIntervalForVariant(targetResults || null, item.variant)
+                            if (!credibleInterval) {
+                                return <>—</>
+                            }
+                            const [lowerBound, upperBound] = credibleInterval
+                            return (
+                                <div className="font-semibold">{`[${lowerBound > 0 ? '+' : ''}${lowerBound.toFixed(
+                                    2
+                                )}%, ${upperBound > 0 ? '+' : ''}${upperBound.toFixed(2)}%]`}</div>
+                            )
+                        },
+                    },
                     {
                         title: 'Win probability',
                         render: function Key(_, item: TabularSecondaryMetricResults): JSX.Element {
diff --git a/frontend/src/scenes/experiments/experimentLogic.tsx b/frontend/src/scenes/experiments/experimentLogic.tsx
index 37e1eba5877..5ec2c1a5fe2 100644
--- a/frontend/src/scenes/experiments/experimentLogic.tsx
+++ b/frontend/src/scenes/experiments/experimentLogic.tsx
@@ -105,6 +105,18 @@ export interface ExperimentResultCalculationError {
     statusCode: number
 }
 
+export interface CachedSecondaryMetricExperimentFunnelsQueryResponse extends CachedExperimentFunnelsQueryResponse {
+    filters?: {
+        insight?: InsightType
+    }
+}
+
+export interface CachedSecondaryMetricExperimentTrendsQueryResponse extends CachedExperimentTrendsQueryResponse {
+    filters?: {
+        insight?: InsightType
+    }
+}
+
 export const experimentLogic = kea<experimentLogicType>([
     props({} as ExperimentLogicProps),
     key((props) => props.experimentId || 'new'),
@@ -1261,6 +1273,53 @@ export const experimentLogic = kea<experimentLogicType>([
                     return (variantResults[variantResults.length - 1].count / variantResults[0].count) * 100
                 },
         ],
+        credibleIntervalForVariant: [
+            () => [],
+            () =>
+                (
+                    experimentResults:
+                        | Partial<ExperimentResults['result']>
+                        | CachedSecondaryMetricExperimentFunnelsQueryResponse
+                        | CachedSecondaryMetricExperimentTrendsQueryResponse
+                        | null,
+                    variantKey: string
+                ): [number, number] | null => {
+                    const credibleInterval = experimentResults?.credible_intervals?.[variantKey]
+                    if (!credibleInterval) {
+                        return null
+                    }
+
+                    if (experimentResults.filters?.insight === InsightType.FUNNELS) {
+                        const controlVariant = (experimentResults.variants as FunnelExperimentVariant[]).find(
+                            ({ key }) => key === 'control'
+                        ) as FunnelExperimentVariant
+                        const controlConversionRate =
+                            controlVariant.success_count / (controlVariant.success_count + controlVariant.failure_count)
+
+                        if (!controlConversionRate) {
+                            return null
+                        }
+
+                        // Calculate the percentage difference between the credible interval bounds of the variant and the control's conversion rate.
+                        // This represents the range in which the true percentage change relative to the control is likely to fall.
+                        const lowerBound = ((credibleInterval[0] - controlConversionRate) / controlConversionRate) * 100
+                        const upperBound = ((credibleInterval[1] - controlConversionRate) / controlConversionRate) * 100
+                        return [lowerBound, upperBound]
+                    }
+
+                    const controlVariant = (experimentResults.variants as TrendExperimentVariant[]).find(
+                        ({ key }) => key === 'control'
+                    ) as TrendExperimentVariant
+
+                    const controlMean = controlVariant.count / controlVariant.absolute_exposure
+
+                    // Calculate the percentage difference between the credible interval bounds of the variant and the control's mean.
+                    // This represents the range in which the true percentage change relative to the control is likely to fall.
+                    const lowerBound = ((credibleInterval[0] - controlMean) / controlMean) * 100
+                    const upperBound = ((credibleInterval[1] - controlMean) / controlMean) * 100
+                    return [lowerBound, upperBound]
+                },
+        ],
         getIndexForVariant: [
             (s) => [s.experimentInsightType],
             (experimentInsightType) =>