Varnish Cache setup for web application acceleration

Setting Up Varnish Cache for Web Application Acceleration

Varnish is an HTTP accelerator that caches backend server responses in memory. It serves cached requests significantly faster than the application — hundreds of thousands of requests per second on a single server.

Installation and Basic Configuration

# Debian/Ubuntu
apt install varnish

# /etc/varnish/default.vcl

vcl 4.1;

import std;

backend default {
    .host = "127.0.0.1";
    .port = "8080";
    .first_byte_timeout = 60s;
    .connect_timeout = 5s;
    .between_bytes_timeout = 10s;
}

# Multiple backends with load balancing
import directors;

backend app1 { .host = "10.0.0.1"; .port = "8080"; }
backend app2 { .host = "10.0.0.2"; .port = "8080"; }

sub vcl_init {
    new cluster = directors.round_robin();
    cluster.add_backend(app1);
    cluster.add_backend(app2);
}

sub vcl_recv {
    set req.backend_hint = cluster.backend();

    # URL normalization (remove trailing slash except root)
    if (req.url ~ "^(/[^?]+)/$") {
        set req.url = regsub(req.url, "/$", "");
    }

    # Don't cache admin panel
    if (req.url ~ "^/admin") {
        return (pass);
    }

    # Don't cache authorized users (if content is personalized)
    if (req.http.Authorization || req.http.Cookie ~ "session_id|auth_token") {
        return (pass);
    }

    # Remove marketing parameters from cache key
    set req.url = regsuball(req.url, "(\?|&)(utm_source|utm_medium|utm_campaign|utm_content|fbclid|gclid)=[^&]+", "");
    set req.url = regsub(req.url, "\?&", "?");
    set req.url = regsub(req.url, "\?$", "");

    # Normalize Accept-Encoding
    if (req.http.Accept-Encoding) {
        if (req.url ~ "\.(gif|jpg|jpeg|png|webp|avif|ico|zip|gz|mp4)$") {
            unset req.http.Accept-Encoding;
        } elsif (req.http.Accept-Encoding ~ "br") {
            set req.http.Accept-Encoding = "br";
        } elsif (req.http.Accept-Encoding ~ "gzip") {
            set req.http.Accept-Encoding = "gzip";
        } else {
            unset req.http.Accept-Encoding;
        }
    }
}

sub vcl_backend_response {
    # Cache 404 briefly
    if (beresp.status == 404) {
        set beresp.ttl = 30s;
        return (deliver);
    }

    # Cache only successful responses
    if (beresp.status != 200 && beresp.status != 301 && beresp.status != 302) {
        set beresp.uncacheable = true;
        return (deliver);
    }

    # If backend didn't send Cache-Control — set default
    if (!beresp.http.Cache-Control) {
        if (bereq.url ~ "\.(css|js|woff2|png|jpg|svg)$") {
            set beresp.ttl = 30d;
        } else {
            set beresp.ttl = 5m;
        }
    }

    # Grace period: serve stale cache if backend fails
    set beresp.grace = 6h;

    # Compression
    if (beresp.http.content-type ~ "text/(html|css|javascript|xml|plain)" ||
        beresp.http.content-type ~ "application/(json|javascript)") {
        set beresp.do_gzip = true;
    }

    # Don't store Set-Cookie in cache
    unset beresp.http.Set-Cookie;
}

sub vcl_deliver {
    # Debug header (disable in production)
    if (obj.hits > 0) {
        set resp.http.X-Cache = "HIT " + obj.hits;
    } else {
        set resp.http.X-Cache = "MISS";
    }

    # Hide internal headers from client
    unset resp.http.X-Varnish;
    unset resp.http.Via;
    unset resp.http.X-Powered-By;
}

sub vcl_hit {
    if (obj.ttl >= 0s) { return (deliver); }
    # Grace: deliver stale object
    if (obj.ttl + obj.grace > 0s) { return (deliver); }
    return (restart);
}

Setting Up Systemd for Varnish

# /etc/systemd/system/varnish.service.d/override.conf
[Service]
ExecStart=
ExecStart=/usr/sbin/varnishd \
    -a :80 \
    -a :8443,PROXY \
    -f /etc/varnish/default.vcl \
    -s malloc,4G \
    -t 120 \
    -p thread_pools=2 \
    -p thread_pool_min=100 \
    -p thread_pool_max=1000 \
    -p thread_pool_timeout=300 \
    -p http_max_hdr=128 \
    -p workspace_backend=256k

-s malloc,4G — 4GB for cache in RAM. For SSD servers: -s file,/var/lib/varnish/cache.bin,20G

PURGE Invalidation

acl purge_acl {
    "127.0.0.1";
    "10.0.0.0"/8;
}

sub vcl_recv {
    if (req.method == "PURGE") {
        if (!client.ip ~ purge_acl) {
            return (synth(405, "Not allowed"));
        }
        return (purge);
    }

    if (req.method == "BAN") {
        if (!client.ip ~ purge_acl) {
            return (synth(405, "Not allowed"));
        }
        ban("req.http.host == " + req.http.host + " && req.url ~ " + req.url);
        return (synth(200, "Ban added"));
    }
}

Usage from application:

# Invalidate specific URL
curl -X PURGE http://localhost/page/about

# Invalidate by pattern (BAN)
curl -X BAN -H "Host: company.com" http://localhost/category/

ESI (Edge Side Includes) for Fragmented Cache

<!-- Page cached entirely, except personalized blocks -->
<esi:include src="/fragments/user-menu" />
<div class="product-list">...</div>
<esi:include src="/fragments/recently-viewed?user_id={{ user_id }}" />

sub vcl_backend_response {
    if (beresp.http.Surrogate-Control ~ "ESI/1.0") {
        set beresp.do_esi = true;
    }
}

Monitoring

# Hit rate statistics
varnishstat -1 -f MAIN.cache_hit,MAIN.cache_miss

# Live logs
varnishlog -g request -q "ReqURL eq '/api/products'"

# Top URLs by miss
varnishtop -i ReqURL

Prometheus exporter via prometheus-varnish-exporter.

Timeline

Varnish installation and basic configuration with VCL for web application — 1–2 business days. With ESI and custom invalidation rules — 2–3 days.