Different Types of ELK Architecture on K8s

  1. Single-node ELK on single container

Use Packed centralized ELK image : sebp/elk

docker run -p 5601:5601 -p 9200:9200 -p 5044:5044 -v elk-data:/var/lib/elasticsearch -it --name elkstack sebp/elk

http://localhost:9200 — Elasticsearch

http://localhost:5601 — Kibana web

http://localhost:5044 — Logstash

Enter container : docker exec -it <container-name> /bin/bash

Edit Logstash config : /opt/logstash/bin/logstash -e ‘input { stdin { } } output { elasticsearch { hosts => [“localhost”] } }’

2. Elasticsearch clusters on a single host

3. ELK cluster on K8S

Service, ConfigMap, StatefulSet :

---
kind: Service
apiVersion: v1
metadata:
name: elasticsearch
labels:
app: elasticsearch
spec:
selector:
app: elasticsearch
clusterIP: None
ports:
- port: 9200
name: rest
- port: 9300
name: inter-node
---kind: ConfigMap
apiVersion: v1
metadata:
name: elasticsearch-config
labels:
app: elasticsearch
data:
elasticsearch.yml: |
network.host: 0.0.0.0
http.port: 9200
transport.port: 9300
# xpack.security.enabled: true
# xpack.security.transport.ssl.enabled: true
---apiVersion: apps/v1
kind: StatefulSet
metadata:
name: es-cluster
spec:
serviceName: elasticsearch
replicas: 3
selector:
matchLabels:
app: elasticsearch
template:
metadata:
labels:
app: elasticsearch
spec:
containers:
- name: elasticsearch
image: elasticsearch:7.13.1
resources:
limits:
cpu: 1000m
requests:
cpu: 100m
ports:
- containerPort: 9200
name: rest
protocol: TCP
- containerPort: 9300
name: inter-node
protocol: TCP
volumeMounts:
- name: elk-data
mountPath: /usr/share/elasticsearch/data
- name: elasticsearch-config
mountPath: /usr/share/elasticsearch/config/elasticsearch.yml
subPath: elasticsearch.yml
env:
- name: cluster.name
value: k8s-logs
- name: node.name
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: discovery.seed_hosts
value: "es-cluster-0.elasticsearch,es-cluster-1.elasticsearch,es-cluster-2.elasticsearch"
- name: cluster.initial_master_nodes
value: "es-cluster-0,es-cluster-1,es-cluster-2"
- name: ES_JAVA_OPTS
value: "-Xms512m -Xmx512m"
- name: ELASTIC_PASSWORD
value: "elastic"
securityContext:
capabilities:
add: ["SYS_CHROOT"]
volumes:
- name: elasticsearch-config
configMap:
name: elasticsearch-config
items:
- key: elasticsearch.yml
path: elasticsearch.yml
initContainers:
- name: fix-permissions
image: /base/alpine:3.13.4
command: ["sh", "-c", "chown -R 1000:1000 /usr/share/elasticsearch/data"]
securityContext:
privileged: true
volumeMounts:
- name: elk-data
mountPath: /usr/share/elasticsearch/data
- name: increase-vm-max-map
image: alpine:3.13.4
command: ["sysctl", "-w", "vm.max_map_count=262144"]
securityContext:
privileged: true
- name: increase-fd-ulimit
image: /alpine:3.13.4
command: ["sh", "-c", "ulimit -n 65536"]
securityContext:
privileged: true
volumeClaimTemplates:
- apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: elk-data
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 20Gi
storageClassName: csi-rbd-sc
volumeMode: Filesystem

Kibana

apiVersion: v1
kind: ConfigMap
metadata:
name: kibana-config
labels:
app: kibana
data:
kibana.yml: |
server.port: 5601
elasticsearch.hosts: ["http://elasticsearch:9200"]
kibana.index: ".kibana"
kibana.defaultAppId: "discover"
elasticsearch.ssl.verificationMode: none
elasticsearch.pingTimeout: 1500
elasticsearch.requestTimeout: 30000
logging.dest: stdout
logging.verbose: false
ops.interval: 5000
---apiVersion: v1
kind: Service
metadata:
name: kibana
labels:
app: kibana
spec:
selector:
app: kibana
clusterIP: None
ports:
- name: http
port: 5601
targetPort: 5601
---apiVersion: apps/v1
kind: Deployment
metadata:
name: kibana
labels:
app: kibana
spec:
replicas: 1
selector:
matchLabels:
app: kibana
template:
metadata:
labels:
app: kibana
spec:
containers:
- name: kibana
image: kibana:7.13.1
resources:
limits:
cpu: 100m
requests:
cpu: 100m
env:
- name: ELASTICSEARCH_URL
value: http://elasticsearch:9200
ports:
- containerPort: 5601
volumeMounts:
- name: config
mountPath: /usr/share/kibana/config
volumes:
- name: config
configMap:
name: kibana-config
items:
- key: kibana.yml
path: kibana.yml