mirrors/posthog
0
0
Fork 0
mirror of https://github.com/PostHog/posthog.git synced 2024-11-21 13:39:22 +01:00
Code Issues Releases Activity

feat(experiments): Enable holdouts for everyone (#26301)

Browse Source 
Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com>
This commit is contained in:
Daniel Bachhuber 2024-11-20 11:51:13 -08:00 committed by GitHub
parent 402fbf1852
commit 5aa461c31a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
10 changed files with 72 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
frontend/__snapshots__/scenes-app-insights--funnel-top-to-bottom--dark.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 96 KiB

After

Width:  |  Height:  |  Size: 96 KiB

BIN
frontend/__snapshots__/scenes-app-insights--funnel-top-to-bottom-breakdown-edit--dark.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 160 KiB

After

Width:  |  Height:  |  Size: 160 KiB

1
frontend/src/lib/constants.tsx
View File

@ -222,7 +222,6 @@ export const FEATURE_FLAGS = {
REPLAY_TEMPLATES: 'replay-templates', // owner: @raquelmsmith #team-replay
EXPERIMENTS_HOGQL: 'experiments-hogql', // owner: @jurajmajerik #team-experiments
ROLE_BASED_ACCESS_CONTROL: 'role-based-access-control', // owner: @zach
EXPERIMENTS_HOLDOUTS: 'experiments-holdouts', // owner: @jurajmajerik #team-experiments
MESSAGING: 'messaging', // owner @mariusandra #team-cdp
SESSION_REPLAY_URL_BLOCKLIST: 'session-replay-url-blocklist', // owner: @richard-better #team-replay
BILLING_TRIAL_FLOW: 'billing-trial-flow', // owner: @zach

22
frontend/src/lib/utils/eventUsageLogic.ts
View File

@ -14,6 +14,7 @@ import { TimeToSeeDataPayload } from 'lib/internalMetrics'
import { isCoreFilter, PROPERTY_KEYS } from 'lib/taxonomy'
import { objectClean } from 'lib/utils'
import posthog from 'posthog-js'
import { Holdout } from 'scenes/experiments/holdoutsLogic'
import { isFilterWithDisplay, isFunnelsFilter, isTrendsFilter } from 'scenes/insights/sharedUtils'
import { preflightLogic } from 'scenes/PreflightCheck/preflightLogic'
import { EventIndex } from 'scenes/session-recordings/player/eventIndex'
@ -486,6 +487,14 @@ export const eventUsageLogic = kea<eventUsageLogicType>([
reportExperimentResultsLoadingTimeout: (experimentId: ExperimentIdType) => ({ experimentId }),
reportExperimentReleaseConditionsViewed: (experimentId: ExperimentIdType) => ({ experimentId }),
reportExperimentReleaseConditionsUpdated: (experimentId: ExperimentIdType) => ({ experimentId }),
reportExperimentHoldoutCreated: (holdout: Holdout) => ({ holdout }),
reportExperimentHoldoutAssigned: ({
experimentId,
holdoutId,
}: {
experimentId: ExperimentIdType
holdoutId: Holdout['id']
}) => ({ experimentId, holdoutId }),
// Definition Popover
reportDataManagementDefinitionHovered: (type: TaxonomicFilterGroupType) => ({ type }),
@ -1082,6 +1091,19 @@ export const eventUsageLogic = kea<eventUsageLogicType>([
experiment_id: experimentId,
})
},
reportExperimentHoldoutCreated: ({ holdout }) => {
posthog.capture('experiment holdout created', {
name: holdout.name,
holdout_id: holdout.id,
filters: holdout.filters,
})
},
reportExperimentHoldoutAssigned: ({ experimentId, holdoutId }) => {
posthog.capture('experiment holdout assigned', {
experiment_id: experimentId,
holdout_id: holdoutId,
})
},
reportPropertyGroupFilterAdded: () => {
posthog.capture('property group filter added')
},

14
frontend/src/scenes/experiments/ExperimentForm.tsx
View File

@ -224,14 +224,12 @@ const ExperimentFormFields = (): JSX.Element => {
</div>
</div>
</div>
{featureFlags[FEATURE_FLAGS.EXPERIMENTS_HOLDOUTS] && (
<div>
<h3>Holdout group</h3>
<div className="text-xs text-muted">Exclude a stable group of users from the experiment.</div>
<LemonDivider />
<HoldoutSelector />
</div>
)}
<div>
<h3>Holdout group</h3>
<div className="text-xs text-muted">Exclude a stable group of users from the experiment.</div>
<LemonDivider />
<HoldoutSelector />
</div>
</div>
<LemonButton
className="mt-2"

34
frontend/src/scenes/experiments/ExperimentView/DistributionTable.tsx
View File

@ -20,11 +20,12 @@ import { Experiment, MultivariateFlagVariant } from '~/types'
import { experimentLogic } from '../experimentLogic'
import { VariantTag } from './components'
import { HoldoutSelector } from './HoldoutSelector'
import { VariantScreenshot } from './VariantScreenshot'
export function DistributionModal({ experimentId }: { experimentId: Experiment['id'] }): JSX.Element {
const { experiment, experimentLoading, isDistributionModalOpen } = useValues(experimentLogic({ experimentId }))
const { closeDistributionModal } = useActions(experimentLogic({ experimentId }))
const { closeDistributionModal, updateExperiment } = useActions(experimentLogic({ experimentId }))
const _featureFlagLogic = featureFlagLogic({ id: experiment.feature_flag?.id ?? null } as FeatureFlagLogicProps)
const { featureFlag, areVariantRolloutsValid, variantRolloutSum } = useValues(_featureFlagLogic)
@ -63,6 +64,7 @@ export function DistributionModal({ experimentId }: { experimentId: Experiment['
<LemonButton
onClick={() => {
saveSidebarExperimentFeatureFlag(featureFlag)
updateExperiment({ holdout_id: experiment.holdout_id })
closeDistributionModal()
}}
type="primary"
@ -124,6 +126,7 @@ export function DistributionModal({ experimentId }: { experimentId: Experiment['
</p>
)}
</div>
<HoldoutSelector />
</div>
</LemonModal>
)
@ -179,6 +182,9 @@ export function DistributionTable(): JSX.Element {
key: 'variant_screenshot',
title: 'Screenshot',
render: function Key(_, item): JSX.Element {
if (item.key === `holdout-${experiment.holdout?.id}`) {
return <div className="h-16" />
}
return (
<div className="my-2">
<VariantScreenshot variantKey={item.key} rolloutPercentage={item.rollout_percentage} />
@ -213,6 +219,23 @@ export function DistributionTable(): JSX.Element {
})
}
const holdoutData = experiment.holdout
? [
{
key: `holdout-${experiment.holdout.id}`,
rollout_percentage: experiment.holdout.filters[0].rollout_percentage,
} as MultivariateFlagVariant,
]
: []
const variantData = (experiment.feature_flag?.filters.multivariate?.variants || []).map((variant) => ({
...variant,
rollout_percentage:
variant.rollout_percentage * ((100 - (experiment.holdout?.filters[0].rollout_percentage || 0)) / 100),
}))
const tableData = [...variantData, ...holdoutData]
return (
<div>
<div className="flex">
@ -237,10 +260,17 @@ export function DistributionTable(): JSX.Element {
</div>
</div>
</div>
{experiment.holdout && (
<LemonBanner type="info" className="mb-4">
This experiment has a holdout group of {experiment.holdout.filters[0].rollout_percentage}%. The
variants are modified to show their relative rollout percentage.
</LemonBanner>
)}
<LemonTable
loading={false}
columns={columns}
dataSource={experiment.feature_flag?.filters.multivariate?.variants || []}
dataSource={tableData}
rowClassName={(item) => (item.key === `holdout-${experiment.holdout?.id}` ? 'bg-mid' : '')}
/>
</div>
)

4
frontend/src/scenes/experiments/ExperimentView/HoldoutSelector.tsx
View File

@ -6,7 +6,7 @@ import { experimentLogic } from '../experimentLogic'
export function HoldoutSelector(): JSX.Element {
const { experiment, holdouts, isExperimentRunning } = useValues(experimentLogic)
const { setExperiment, updateExperiment } = useActions(experimentLogic)
const { setExperiment, reportExperimentHoldoutAssigned } = useActions(experimentLogic)
const holdoutOptions = holdouts.map((holdout) => ({
value: holdout.id,
@ -37,7 +37,7 @@ export function HoldoutSelector(): JSX.Element {
...experiment,
holdout_id: value,
})
updateExperiment({ holdout_id: value })
reportExperimentHoldoutAssigned({ experimentId: experiment.id, holdoutId: value })
}}
data-attr="experiment-holdout-selector"
/>

6
frontend/src/scenes/experiments/Experiments.tsx
View File

@ -249,13 +249,11 @@ export function Experiments(): JSX.Element {
{ key: ExperimentsTabs.All, label: 'All experiments' },
{ key: ExperimentsTabs.Yours, label: 'Your experiments' },
{ key: ExperimentsTabs.Archived, label: 'Archived experiments' },
...(featureFlags[FEATURE_FLAGS.EXPERIMENTS_HOLDOUTS]
? [{ key: ExperimentsTabs.Holdouts, label: 'Holdout groups' }]
: []),
{ key: ExperimentsTabs.Holdouts, label: 'Holdout groups' },
]}
/>
{featureFlags[FEATURE_FLAGS.EXPERIMENTS_HOLDOUTS] && tab === ExperimentsTabs.Holdouts ? (
{tab === ExperimentsTabs.Holdouts ? (
<Holdouts />
) : (
<>

1
frontend/src/scenes/experiments/experimentLogic.tsx
View File

@ -162,6 +162,7 @@ export const experimentLogic = kea<experimentLogicType>([
'reportExperimentVariantScreenshotUploaded',
'reportExperimentResultsLoadingTimeout',
'reportExperimentReleaseConditionsViewed',
'reportExperimentHoldoutAssigned',
],
],
})),

9
frontend/src/scenes/experiments/holdoutsLogic.tsx
View File

@ -1,7 +1,8 @@
import { actions, events, kea, listeners, path, reducers } from 'kea'
import { actions, connect, events, kea, listeners, path, reducers } from 'kea'
import { loaders } from 'kea-loaders'
import api from 'lib/api'
import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast'
import { eventUsageLogic } from 'lib/utils/eventUsageLogic'
import { UserBasicType } from '~/types'
@ -42,6 +43,9 @@ export const holdoutsLogic = kea<holdoutsLogicType>([
deleteHoldout: (id: number | null) => ({ id }),
loadHoldout: (id: number | null) => ({ id }),
}),
connect({
actions: [eventUsageLogic, ['reportExperimentHoldoutCreated']],
}),
reducers({
holdout: [
NEW_HOLDOUT,
@ -50,7 +54,7 @@ export const holdoutsLogic = kea<holdoutsLogicType>([
},
],
}),
loaders(({ values }) => ({
loaders(({ actions, values }) => ({
holdouts: [
[] as Holdout[],
{
@ -60,6 +64,7 @@ export const holdoutsLogic = kea<holdoutsLogicType>([
},
createHoldout: async () => {
const response = await api.create(`api/projects/@current/experiment_holdouts/`, values.holdout)
actions.reportExperimentHoldoutCreated(response)
return [...values.holdouts, response] as Holdout[]
},
updateHoldout: async ({ id, holdout }) => {