A/B Testing Setup on Website (Google Optimize/VWO/Optimizely)

A/B Testing Setup

Google Optimize shut down in 2023. Current tools include Statsig, Growthbook, Optimizely, VWO, or custom implementation via Edge Functions.

Tool Selection

Implementation via Vercel Edge Middleware

// middleware.ts
import { NextResponse } from 'next/server';
import type { NextRequest } from 'next/server';

const EXPERIMENT_COOKIE = 'exp_checkout_v2';
const VARIANTS = ['control', 'variant-a', 'variant-b'];

function assignVariant(): string {
  const rand = Math.random();
  if (rand < 0.34) return 'control';
  if (rand < 0.67) return 'variant-a';
  return 'variant-b';
}

export function middleware(request: NextRequest) {
  const response = NextResponse.next();

  // Exclude: bots, already-assigned users
  const existing = request.cookies.get(EXPERIMENT_COOKIE)?.value;
  if (existing && VARIANTS.includes(existing)) {
    return response;
  }

  const variant = assignVariant();
  response.cookies.set(EXPERIMENT_COOKIE, variant, {
    maxAge: 60 * 60 * 24 * 30, // 30 days
    httpOnly: true,
    sameSite: 'lax',
  });

  // Pass variant in header for Server Components
  response.headers.set('x-ab-checkout', variant);
  return response;
}

export const config = {
  matcher: ['/checkout/:path*'],
};

// app/checkout/page.tsx
import { cookies, headers } from 'next/headers';

export default function CheckoutPage() {
  const variant = headers().get('x-ab-checkout') ??
                  cookies().get('exp_checkout_v2')?.value ??
                  'control';

  return (
    <>
      {variant === 'control' && <CheckoutV1 />}
      {variant === 'variant-a' && <CheckoutV2OneStep />}
      {variant === 'variant-b' && <CheckoutV2TwoStep />}
      <ABTracker experiment="checkout_v2" variant={variant} />
    </>
  );
}

Results Tracking

// components/ABTracker.tsx (Client Component)
'use client';

import { useEffect } from 'react';

export function ABTracker({ experiment, variant }: {
  experiment: string;
  variant: string;
}) {
  useEffect(() => {
    // GA4
    gtag('event', 'experiment_impression', {
      experiment_id: experiment,
      variant_id: variant,
    });

    // PostHog
    posthog.capture('$experiment_started', {
      '$experiment_id': experiment,
      '$variant_key': variant,
    });
  }, [experiment, variant]);

  return null;
}

// Conversion tracking — at purchase time
function trackConversion(variant: string) {
  gtag('event', 'purchase', {
    experiment_id: 'checkout_v2',
    variant_id: variant,
    value: orderTotal,
  });
}

Statsig: Quick Integration

// Statsig SDK
import Statsig from 'statsig-node';

await Statsig.initialize(process.env.STATSIG_SERVER_KEY!);

// In API route / Server Action
const experiment = Statsig.getExperiment(
  { userID: userId, email: userEmail },
  'checkout_redesign'
);

const checkoutLayout = experiment.get('layout', 'single-page');
const ctaColor = experiment.get('cta_color', 'blue');

// Client side (React SDK)
import { useExperiment } from 'statsig-react';

function PricingCTA() {
  const { config } = useExperiment('pricing_cta');
  const buttonText = config.get('button_text', 'Get Started');
  const buttonVariant = config.get('button_variant', 'primary');

  return (
    <Button
      variant={buttonVariant}
      onClick={() => {
        statsig.logEvent('cta_clicked', buttonText);
      }}
    >
      {buttonText}
    </Button>
  );
}

Statistical Significance

Before running a test, calculate the required sample size:

# Python: sample size calculation
from statsmodels.stats.power import zt_ind_solve_power

# CVR = 3%, expected effect = +15% (to 3.45%)
baseline_rate = 0.03
expected_effect = 0.15  # relative improvement
lift = baseline_rate * expected_effect  # 0.0045 absolute

n = zt_ind_solve_power(
    effect_size=lift / (baseline_rate * (1 - baseline_rate)) ** 0.5,
    alpha=0.05,  # 95% confidence
    power=0.8,   # 80% statistical power
)
print(f"Sample size per variant: {int(n)}")  # ~12,000

Rule: do not stop the test before reaching the planned sample size, even if results look promising.

Result Analysis in GA4

GA4 → Explore → Free Form
Dimension: Experiment Variant (custom event parameter)
Metric: Conversions, Revenue
Segment by variant → compare conversions

A/B testing setup with GA4/PostHog tracking — 2–4 business days depending on the tech stack.