ELK - DevOps Buzz

Deploy Elasticsearch
Setup the host node
The vm.max_map_count kernel setting needs to be set to at least 262144 for production use. Make sure the node(s) that will host Elasticsearch have the following config:
1
sysctl -w vm.max_map_count=262144
Copied!
Create the data dir:
1
mkdir /storage/storage-001/mnt-elasticsearch
2
chown nobody:nogroup /storage/storage-001/mnt-elasticsearch/
Copied!
Create the namespace
Connect to your kubectl workstation and create the namespace:
1
kubectl create namespace elk
Copied!
Create the ConfigMap
Create Elasticsearch config file:
1
cat <<EOF >>elasticsearch.yml
2
​
3
cluster.name: "docker-cluster"
4
network.host: 0.0.0.0
5
discovery.zen.minimum_master_nodes: 1
6
discovery.type: single-node
7
​
8
EOF
Copied!
Create its ConfigMap:
1
kubectl -n elk \
2
  create configmap cm-elasticsearch \
3
  --from-file=elasticsearch.yml \
4
  -o yaml --dry-run | kubectl apply -f -
Copied!
If you need to update the ConfigMap, run:
1
kubectl -n elk \
2
  create configmap cm-elasticsearch \
3
  --from-file=elasticsearch.yml \
4
  -o yaml --dry-run | kubectl apply -f -
Copied!
Then run:
1
kubectl -n elk scale deployment/elasticsearch --replicas=0
2
kubectl -n elk scale deployment/elasticsearch --replicas=1
Copied!
Deploy Elasticsearch
Run:
1
kubectl create -f - <<EOF
2
​
3
apiVersion: apps/v1
4
kind: Deployment
5
metadata:
6
  name: elasticsearch
7
  namespace: elk
8
  labels:
9
    app: elasticsearch
10
spec:
11
  replicas: 1
12
  selector:
13
    matchLabels:
14
      app: elasticsearch
15
  template:
16
    metadata:
17
      labels:
18
        app: elasticsearch
19
    spec:
20
      securityContext:
21
        runAsUser: 65534
22
        fsGroup: 65534
23
      hostNetwork: true
24
      containers:
25
      - name: elasticsearch
26
        image: docker.elastic.co/elasticsearch/elasticsearch:6.7.0
27
                   
28
        ports:
29
        - containerPort: 9200
30
        - containerPort: 9300
31
        
32
        env:
33
        - name: discovery.type
34
          value: "single-node"
35
​
36
        volumeMounts:              
37
          - name: config-volume
38
            mountPath: /usr/share/elasticsearch/config/elasticsearch.yml
39
            subPath: elasticsearch.yml
40
            
41
          - name: mnt-elasticsearch
42
            mountPath: /usr/share/elasticsearch/data
43
​
44
      volumes:
45
        - name: config-volume
46
          configMap:
47
           name: cm-elasticsearch
48
           
49
        - name: mnt-elasticsearch
50
          hostPath:
51
            path: /storage/storage-001/mnt-elasticsearch
52
            
53
      nodeSelector:
54
        kubernetes.io/hostname: k8snode
55
​
56
EOF
Copied!
Check the latest docker image here https://www.elastic.co/downloads/past-releases and here https://www.docker.elastic.co/​
Create Elasticsearch service
Run:
1
kubectl create -f - <<EOF
2
        
3
---
4
apiVersion: v1
5
kind: Service
6
metadata:
7
  labels:
8
    app: elasticsearch
9
  name: srv-elasticsearch
10
  namespace: elk
11
spec:
12
  externalTrafficPolicy: Cluster
13
  ports:
14
  - name: "port-9200"
15
    nodePort: 30920
16
    port: 9200
17
    protocol: TCP
18
    targetPort: 9200
19
  - name: "port-9300"
20
    nodePort: 30930
21
    port: 9300
22
    protocol: TCP
23
    targetPort: 9300
24
  selector:
25
    app: elasticsearch
26
  sessionAffinity: None
27
  type: NodePort
28
​
29
EOF
Copied!
Test
Get indices:
1
curl PUT-YOUR-NODE-IP-HERE:30920
2
curl PUT-YOUR-NODE-IP-HERE:30920/_cat/indices?v
Copied!
Post content:
1
curl -H "Content-Type: application/json" -XPOST "http://<HOST>:<PORT>/YOUR-INDEX/YOUR-TYPE/optionalUniqueId" -d "{ \"field\" : \"value\"}"
Copied!
Another example of post:
1
curl -X POST -H "Content-Type: application/json" -H "Cache-Control: no-cache" -d '{
2
"user" : "Arun Thundyill Saseendran",
3
"post_date" : "2009-03-23T12:30:00",
4
"message" : "trying out Elasticsearch"
5
}' "http://<HOST>:<IP>/sampleindex/sampletype/"
Copied!
Deploy Logstash
Create the ConfigMap (config file)
Create the config file:
1
cat <<EOF >>logstash.yml
2
​
3
http.host: "0.0.0.0"
4
path.config: /usr/share/logstash/pipeline
5
​
6
EOF
Copied!
Create its ConfigMap:
1
kubectl -n elk \
2
  create configmap cm-config-logstash \
3
  --from-file=logstash.yml \
4
  -o yaml --dry-run | kubectl apply -f -
Copied!
If you need to update the ConfigMap, run:
1
kubectl -n elk \
2
  create configmap cm-config-logstash \
3
  --from-file=logstash.yml \
4
  -o yaml --dry-run | kubectl apply -f -
Copied!
Then run:
1
kubectl -n elk scale deployment/logstash --replicas=0
2
kubectl -n elk scale deployment/logstash --replicas=1
Copied!
Create the ConfigMap (pipeline)
Create the config file:
1
cat <<EOF >>logstash.conf
2
​
3
#input {
4
#	tcp {
5
#		port => 5959
6
#	}
7
#}
8
​
9
input {
10
	http {
11
		port => 5959
12
		response_headers => {
13
			"Access-Control-Allow-Origin" => "*"
14
			"Content-Type" => "application/json"
15
			"Access-Control-Allow-Headers" => "Origin, X-Requested-With, Content-Type, Accept"
16
		}
17
	}
18
}
19
​
20
## Add your filters / logstash plugins configuration here
21
​
22
output {
23
	elasticsearch {
24
		hosts => "PUT-YOUR-HOST-HERE:PUT-YOUR-PORT-HERE"
25
	}
26
}
27
​
28
EOF
Copied!
Replace output.elasticsearch.hosts with your Elasticsearch host and port.
Create its ConfigMap:
1
kubectl -n elk \
2
  create configmap cm-pipeline-logstash \
3
  --from-file=logstash.conf \
4
  -o yaml --dry-run | kubectl apply -f -
Copied!
If you need to update the ConfigMap, run:
1
kubectl -n elk \
2
  create configmap cm-pipeline-logstash \
3
  --from-file=logstash.conf \
4
  -o yaml --dry-run | kubectl apply -f -
Copied!
Then run:
1
kubectl -n elk scale deployment/logstash --replicas=0
2
kubectl -n elk scale deployment/logstash --replicas=1
Copied!
Deploy
Connect to your node and create the data dir:
1
mkdir -p /storage/storage-001/mnt-logstash
2
chown nobody:nogroup /storage/storage-001/mnt-logstash
Copied!
Connect to your kubectl workstation and run:
1
kubectl create -f - <<EOF
2
​
3
apiVersion: apps/v1
4
kind: Deployment
5
metadata:
6
  name: logstash
7
  namespace: elk
8
  labels:
9
    app: logstash
10
spec:
11
  replicas: 1
12
  selector:
13
    matchLabels:
14
      app: logstash
15
  template:
16
    metadata:
17
      labels:
18
        app: logstash
19
    spec:
20
      securityContext:
21
        runAsUser: 65534
22
        fsGroup: 65534
23
      hostNetwork: true
24
      containers:
25
      - name: logstash
26
        image: docker.elastic.co/logstash/logstash:6.7.0
27
                   
28
        ports:
29
        - containerPort: 5959
30
        
31
        env:
32
        - name: discovery.type
33
          value: "single-node"
34
​
35
        volumeMounts:              
36
          - name: config-volume
37
            mountPath: /usr/share/logstash/config/logstash.yml
38
            subPath: logstash.yml
39
​
40
          - name: pipeline-volume
41
            mountPath: /usr/share/logstash/pipeline/logstash.conf
42
            subPath: logstash.conf
43
            
44
          - name: mnt-logstash
45
            mountPath: /usr/share/logstash/data
46
​
47
      volumes:
48
        - name: config-volume
49
          configMap:
50
           name: cm-config-logstash
51
​
52
        - name: pipeline-volume
53
          configMap:
54
           name: cm-pipeline-logstash
55
           
56
        - name: mnt-logstash
57
          hostPath:
58
            path: /storage/storage-001/mnt-logstash
59
            
60
      nodeSelector:
61
        kubernetes.io/hostname: k8snode
62
​
63
EOF
Copied!
Check the latest docker image here https://www.elastic.co/downloads/past-releases and here https://www.docker.elastic.co/​
Create service
Run:
1
kubectl create -f - <<EOF
2
        
3
---
4
apiVersion: v1
5
kind: Service
6
metadata:
7
  labels:
8
    app: logstash
9
  name: srv-logstash
10
  namespace: elk
11
spec:
12
  externalTrafficPolicy: Cluster
13
  ports:
14
  - name: "port-5959"
15
    nodePort: 30595
16
    port: 5959
17
    protocol: TCP
18
    targetPort: 5959
19
  selector:
20
    app: logstash
21
  sessionAffinity: None
22
  type: NodePort
23
​
24
EOF
Copied!
Test
Get service info:
1
curl -XGET 'PUT-YOUR-HOST-HERE:9600/_node/logging?pretty'
Copied!
Telnet test:
1
telnet PUT-NODE-HOST-HERE 30595
Copied!
​
Deploy Kibana
Create the ConfigMap
Create the config file:
1
cat <<EOF >>kibana.yml
2
​
3
server.name: kibana
4
server.host: "0"
5
elasticsearch.hosts: [ "http://PUT-YOUR-HOST-HERE:30920" ]
6
xpack.monitoring.ui.container.elasticsearch.enabled: true
7
​
8
EOF
Copied!
Replace elasticsearch.hosts with your Elasticsearch host and port.
Create its ConfigMap:
1
kubectl -n elk \
2
  create configmap cm-kibana \
3
  --from-file=kibana.yml \
4
  -o yaml --dry-run | kubectl apply -f -
Copied!
If you need to update the ConfigMap, run:
1
kubectl -n elk \
2
  create configmap cm-kibana \
3
  --from-file=kibana.yml \
4
  -o yaml --dry-run | kubectl apply -f -
Copied!
Then run:
1
kubectl -n elk scale deployment/kibana --replicas=0
2
kubectl -n elk scale deployment/kibana --replicas=1
Copied!
Deploy
Connect to your node and create the data dir:
1
mkdir /storage/storage-001/mnt-kibana
2
chown nobody:nogroup /storage/storage-001/mnt-kibana/
Copied!
Connect to your kubectl workstation and run:
1
kubectl create -f - <<EOF
2
​
3
apiVersion: apps/v1
4
kind: Deployment
5
metadata:
6
  name: kibana
7
  namespace: elk
8
  labels:
9
    app: kibana
10
spec:
11
  replicas: 1
12
  selector:
13
    matchLabels:
14
      app: kibana
15
  template:
16
    metadata:
17
      labels:
18
        app: kibana
19
    spec:
20
      securityContext:
21
        runAsUser: 65534
22
        fsGroup: 65534
23
      hostNetwork: true
24
      containers:
25
      - name: kibana
26
        image: docker.elastic.co/kibana/kibana:6.7.0
27
                   
28
        ports:
29
        - containerPort: 5601
30
        
31
        env:
32
        - name: ELASTICSEARCH_HOSTS
33
          value: "http://PUT-YOUR-HOST-HERE:30920"
34
​
35
        volumeMounts:
36
          - name: config-volume
37
            mountPath: /usr/share/kibana/config/kibana.yml
38
            subPath: kibana.yml
39
                          
40
          - name: mnt-kibana
41
            mountPath: /usr/share/kibana/data
42
​
43
      volumes:
44
        - name: config-volume
45
          configMap:
46
           name: cm-kibana
47
           
48
        - name: mnt-kibana
49
          hostPath:
50
            path: /storage/storage-001/mnt-kibana
51
            
52
      nodeSelector:
53
        kubernetes.io/hostname: k8snode
54
​
55
EOF
Copied!
Replace spec.template.spec.containers.env with your Elasticsearch host and port.
Check the latest docker image here https://www.elastic.co/downloads/past-releases and here https://www.docker.elastic.co/​
Create service
Run:
1
kubectl create -f - <<EOF
2
        
3
---
4
apiVersion: v1
5
kind: Service
6
metadata:
7
  labels:
8
    app: kibana
9
  name: srv-kibana
10
  namespace: elk
11
spec:
12
  externalTrafficPolicy: Cluster
13
  ports:
14
  - name: "port-5601"
15
    nodePort: 30560
16
    port: 5601
17
    protocol: TCP
18
    targetPort: 5601
19
  selector:
20
    app: kibana
21
  sessionAffinity: None
22
  type: NodePort
23
​
24
EOF
Copied!
Test
Curl test:
1
curl http://PUT-YOUR-HOST-HERE:30560
Copied!
Deploy filebeat
Example to stream log files from /elk/*.log to elasticsearch.
1
cat <<EOF >>filebeat.docker.yml
2
​
3
filebeat:
4
  config:
5
    modules:
6
      path: ${path.config}/modules.d/*.yml
7
      reload.enabled: false
8
    
9
  autodiscover:
10
    providers:
11
      - type: docker
12
        hints.enabled: true
13
​
14
  prospectors:  
15
  - input_type: log
16
    paths:
17
      - /elk/*.log
18
​
19
#output.logstash:
20
#  hosts: ["PUT-YOUR-IP-HERE:30595"]
21
​
22
output.elasticsearch:
23
  hosts: ["http://PUT-YOUR-IP-HERE:30920"]
24
  #index: "filebeat-%{[beat.version]}-%{+yyyy.MM.dd}"
25
  #index: "test-filebeat"
26
​
27
logging:
28
  files:
29
    rotateeverybytes: 10485760 # = 10MB
30
​
31
​
32
EOF
Copied!
Text:
1
docker run -tid \
2
  --name=filebeat \
3
  --user=root \
4
  --volume="$(pwd)/filebeat.docker.yml:/usr/share/filebeat/filebeat.yml:ro" \
5
  --volume="/elk:/elk:ro" \
6
  --volume="/var/lib/docker/containers:/var/lib/docker/containers:ro" \
7
  --volume="/var/run/docker.sock:/var/run/docker.sock:ro" \
8
  docker.elastic.co/beats/filebeat:6.7.0 filebeat
Copied!
Text:
1
command
Copied!
Text:
1
command
Copied!
Text:
1
command
Copied!
Text:
1
command
Copied!
Text:
1
command
Copied!