Email Campaign A/B Testing

A/B Testing Email Campaigns

A/B testing email is sending different versions of an email to different audience segments to determine which variant yields better open and click rates. You can test subject lines, preview text, content, CTAs, and send time.

What to Test

• Subject line — highest impact, directly affects open rate
• Prehead/Preview text — displayed next to the subject in inbox
• CTA button — text, color, position
• Hero image — product photo vs illustration vs no image
• Personalization — with user name vs without
• Send time — morning vs evening, weekday vs weekend

Backend A/B Test Implementation

interface ABTestVariant {
  id: 'A' | 'B' | 'C';
  subject: string;
  templateId: string;
  weight: number;  // traffic share, e.g., 0.5 for 50/50
}

interface ABTest {
  id: string;
  campaignId: string;
  variants: ABTestVariant[];
  winnerMetric: 'open_rate' | 'click_rate';
  sampleSize: number;       // how many to send for testing
  winnerSendAt?: Date;      // when to send winner to remainder
}

async function sendABTest(test: ABTest, users: User[]) {
  // Shuffle users randomly
  const shuffled = users.sort(() => Math.random() - 0.5);

  // Split into groups according to weights
  let offset = 0;
  for (const variant of test.variants) {
    const count = Math.floor(test.sampleSize * variant.weight);
    const group = shuffled.slice(offset, offset + count);
    offset += count;

    await Promise.allSettled(
      group.map(user =>
        sendVariantEmail(user, variant, test.id)
      )
    );
  }

  // Save test information
  await db.abTests.create(test);

  // Schedule winner determination
  if (test.winnerSendAt) {
    await scheduleWinnerSelection(test.id, test.winnerSendAt);
  }
}

async function sendVariantEmail(user: User, variant: ABTestVariant, testId: string) {
  const html = await renderTemplate(variant.templateId, { user });
  const emailLogId = await sendEmail({
    to: user.email,
    subject: variant.subject,
    html,
  });

  await db.abTestParticipants.create({
    testId,
    variantId: variant.id,
    userId: user.id,
    emailLogId,
  });
}

Determining the Winner

async function determineWinner(testId: string): Promise<'A' | 'B' | 'C'> {
  const test = await db.abTests.findById(testId);

  const stats = await db.query<{
    variant_id: string;
    sent: number;
    opened: number;
    clicked: number;
  }>(`
    SELECT
      p.variant_id,
      COUNT(DISTINCT p.id) AS sent,
      COUNT(DISTINCT oe.email_log_id) AS opened,
      COUNT(DISTINCT ce.email_log_id) AS clicked
    FROM ab_test_participants p
    LEFT JOIN email_open_events oe ON oe.email_log_id = p.email_log_id
    LEFT JOIN email_click_events ce ON ce.email_log_id = p.email_log_id
    WHERE p.test_id = $1
    GROUP BY p.variant_id
  `, [testId]);

  const withRates = stats.map(s => ({
    ...s,
    open_rate: s.opened / s.sent,
    click_rate: s.clicked / s.sent,
  }));

  // Check statistical significance (z-test for proportions)
  const winner = withRates.reduce((best, current) => {
    const metric = test.winnerMetric === 'open_rate' ? 'open_rate' : 'click_rate';
    return current[metric] > best[metric] ? current : best;
  });

  return winner.variant_id as 'A' | 'B' | 'C';
}

// Send winner to remaining users
async function sendWinnerToRemainder(testId: string) {
  const winnerId = await determineWinner(testId);
  const test = await db.abTests.findById(testId);
  const winnerVariant = test.variants.find(v => v.id === winnerId)!;

  // Users who didn't participate in test
  const participantIds = await db.abTestParticipants.getUserIdsByTest(testId);
  const remainderUsers = await db.users.findExcluding(participantIds, test.campaignId);

  await Promise.allSettled(
    remainderUsers.map(user =>
      sendVariantEmail(user, winnerVariant, testId)
    )
  );
}

Statistical Significance

It's important not to declare a winner too early. Minimum sample size for 95% confidence with expected 25% open rate and 5% delta is about 1,200 recipients per variant. Online calculators (Optimizely, Evan Miller) are used for calculation.

function isStatisticallySignificant(
  controlConverted: number,
  controlTotal: number,
  variantConverted: number,
  variantTotal: number,
  confidenceLevel: number = 0.95
): boolean {
  const p1 = controlConverted / controlTotal;
  const p2 = variantConverted / variantTotal;
  const pPool = (controlConverted + variantConverted) / (controlTotal + variantTotal);
  const se = Math.sqrt(pPool * (1 - pPool) * (1/controlTotal + 1/variantTotal));
  const z = Math.abs(p2 - p1) / se;
  const zThreshold = confidenceLevel === 0.95 ? 1.96 : 2.576;  // 95% or 99%
  return z >= zThreshold;
}

Timeline

An A/B testing system with traffic split, metric collection, winner determination, and remainder send takes 3–5 days.