Elasticsearch cluster setup for web application

Elasticsearch Cluster Setup

Elasticsearch cluster is multiple nodes coordinating indexing and search. Single node is risky: data loss on server failure, no shard distribution. Minimum production cluster: 3 nodes (master quorum = 2).

Docker Compose Cluster

version: '3.8'
services:
  es-master-1:
    image: docker.elastic.co/elasticsearch/elasticsearch:8.11.0
    environment:
      - cluster.name=prod-cluster
      - node.name=master-1
      - node.roles=master,data
      - discovery.seed_hosts=es-master-1,es-master-2,es-master-3
      - cluster.initial_master_nodes=master-1,master-2,master-3
      - xpack.security.enabled=false
      - ES_JAVA_OPTS=-Xms2g -Xmx2g
    volumes:
      - esdata1:/usr/share/elasticsearch/data
    ports:
      - "9201:9200"
    healthcheck:
      test: curl -s http://localhost:9200/_cluster/health | grep status
      interval: 10s
      timeout: 5s
      retries: 5

  es-master-2:
    image: docker.elastic.co/elasticsearch/elasticsearch:8.11.0
    environment:
      - cluster.name=prod-cluster
      - node.name=master-2
      - node.roles=master,data
      - discovery.seed_hosts=es-master-1,es-master-2,es-master-3
      - cluster.initial_master_nodes=master-1,master-2,master-3
      - xpack.security.enabled=false
      - ES_JAVA_OPTS=-Xms2g -Xmx2g
    volumes:
      - esdata2:/usr/share/elasticsearch/data
    ports:
      - "9202:9200"

  es-master-3:
    image: docker.elastic.co/elasticsearch/elasticsearch:8.11.0
    environment:
      - cluster.name=prod-cluster
      - node.name=master-3
      - node.roles=master,data
      - discovery.seed_hosts=es-master-1,es-master-2,es-master-3
      - cluster.initial_master_nodes=master-1,master-2,master-3
      - xpack.security.enabled=false
      - ES_JAVA_OPTS=-Xms2g -Xmx2g
    volumes:
      - esdata3:/usr/share/elasticsearch/data
    ports:
      - "9203:9200"

  kibana:
    image: docker.elastic.co/kibana/kibana:8.11.0
    ports:
      - "5601:5601"
    environment:
      - ELASTICSEARCH_HOSTS=http://es-master-1:9200

volumes:
  esdata1:
  esdata2:
  esdata3:

Cluster Health Verification

curl -s http://localhost:9200/_cluster/health | jq

# Output:
# {
#   "cluster_name": "prod-cluster",
#   "status": "green",           # green = all shards allocated
#   "timed_out": false,
#   "number_of_nodes": 3,
#   "number_of_data_nodes": 3,
#   "active_primary_shards": 5,
#   "active_shards": 10
# }

Shard Strategy

# For an index with 100 GB data, 3 nodes:
# - 3 primary shards (one per node)
# - 2 replicas (total 9 shards)

curl -X PUT "localhost:9200/my_index" \
  -H 'Content-Type: application/json' \
  -d '{
    "settings": {
      "number_of_shards": 3,
      "number_of_replicas": 2,
      "index.refresh_interval": "30s"
    }
  }'

Node Roles

# Master node (elect leaders)
node.roles: [ master ]

# Data node (store shards)
node.roles: [ data ]

# Ingest node (preprocess documents)
node.roles: [ ingest ]

# Combined (all roles)
node.roles: [ master, data, ingest ]