reorganise directory

fluentd/Docker-Compose/README.md

@@ -1,10 +1,6 @@
 # Configuration
 
-Edit `setenv.sh`, set `IMAGE_NAME` to the repository, image and tag where you want to save the image. Pushing the image is not required when just running locally.
-
-Build the image with the `build.sh` script.
-
-Edit `humio.conf` setting the parameters to match either your Humio account or Elasticsearch instance. See the [fluentd documentation](https://docs.fluentd.org/output/elasticsearch) and/or [Humio documentation](https://docs.humio.com/docs/ingesting-data/data-shippers/fluentd/) for details.
+Edit `fluent.conf` setting the parameters to match either your Humio account or Elasticsearch instance. See the [fluentd documentation](https://docs.fluentd.org/output/elasticsearch) and/or [Humio documentation](https://docs.humio.com/docs/ingesting-data/data-shippers/fluentd/) for details.
 
 Start `fluentd` in a Docker container using the `run.sh` script.
 
@@ -30,5 +26,7 @@ services:
 
 If Robot Shop is already running, shut it down `docker-compose down`
 
-Start Robot Shop with `docker-compose up -d`. It takes a few minutes to start, after that check eith Humio or ELK for log entries.
+Start Robot Shop with `docker-compose up -d`. It takes a few minutes to start, after that check with Humio or ELK for log entries.
+
+Set up [logging integration](https://www.instana.com/docs/logging/) in Instana.
 

fluentd/Docker-Compose/run.sh

@@ -1,13 +1,12 @@
 #!/bin/sh
 
-. ./setenv.sh
+IMAGE_NAME="robotshop/fluentd:elastic"
 
 docker run \
     -d \
     --rm \
     --name fluentd \
     -p 24224:24224 \
-    -v $(pwd)/humio.conf:/fluentd/etc/humio.conf \
-    -e FLUENTD_CONF=humio.conf \
+    -v $(pwd)/fluent.conf:/fluentd/etc/fluent.conf \
     $IMAGE_NAME
 

fluentd/Docker/Dockerfile

@@ -1,5 +0,0 @@
-FROM fluentd
-USER root
-RUN fluent-gem install fluent-plugin-elasticsearch
-USER fluent
-

fluentd/Docker/build.sh

@@ -1,6 +0,0 @@
-#!/bin/sh
-
-. ./setenv.sh
-
-docker build -t $IMAGE_NAME .
-

fluentd/Docker/setenv.sh

@@ -1,4 +0,0 @@
-#!/bin/sh
-
-IMAGE_NAME="repo/image:tag"
-

fluentd/Dockerfile

@@ -0,0 +1,10 @@
+FROM fluentd
+USER root
+RUN apk update && \
+    apk add --virtual .build-dependencies build-base ruby-dev
+
+RUN fluent-gem install fluent-plugin-elasticsearch && \
+    fluent-gem install fluent-plugin-kubernetes_metadata_filter  && \
+    fluent-gem install fluent-plugin-multi-format-parser
+USER fluent
+

fluentd/Kubernetes/README.md

@@ -0,0 +1,10 @@
+# Kubernetes
+
+Edit the `fluentd.yaml` file inserting your Humio or Elasticsearch instance details.
+
+Apply the configuration:
+
+```shell
+$ kubectl apply -f fluentd.yaml
+```
+

fluentd/Kubernetes/fluentd.yaml

@@ -0,0 +1,147 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: logging
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: fluentd
+  namespace: logging
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: fluentd
+  namespace: logging
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  - namespaces
+  verbs:
+  - get
+  - list
+  - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: fluentd
+  namespace: logging
+roleRef:
+  kind: ClusterRole
+  name: fluentd
+  apiGroup: rbac.authorization.k8s.io
+subjects:
+- kind: ServiceAccount
+  name: fluentd
+  namespace: logging
+#
+# CONFIGURATION
+#
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: fluentd-config
+  namespace: logging
+data:
+  fluent.conf: |
+    <source>
+      @type tail
+      path /var/log/containers/*.log
+      pos_file /var/log/fluentd-containers.log.pos
+      tag kubernetes.*
+      read_from_head false
+      <parse>
+        @type json
+      </parse>
+    </source>
+
+    <filter kubernetes.**>
+      @type kubernetes_metadata
+      @id filter_kube_metadata
+    </filter>
+
+    # Throw away what is not needed first
+
+    #<match fluent.**>
+    #@type null
+    #</match>
+
+    <match kubernetes.var.log.containers.**kube-system**.log>
+      @type null
+    </match>
+
+    # Capture what is left
+    <match **>
+      @type elasticsearch
+      host cloud.humio.com
+      port 9200
+      scheme https
+      ssl_version TLSv1_2
+      logstash_format true
+      user <Humio index or Elasticsearch user>
+      password <Humio API key or Elasticsearch password>
+    </match>
+
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: fluentd
+  namespace: logging
+  labels:
+    k8s-app: fluentd
+    #https://github.com/kubernetes/kubernetes/issues/51376
+    #kubernetes.io/cluster-service: "true"
+spec:
+  selector:
+    matchLabels:
+      name: fluentd
+  template:
+    metadata:
+      labels:
+        name: fluentd
+        #kubernetes.io/cluster-service: "true"
+    spec:
+      serviceAccount: fluentd
+      serviceAccountName: fluentd
+      terminationGracePeriodSeconds: 30
+      tolerations:
+        - key: node-role.kubernetes.io/master
+          effect: NoSchedule
+      containers:
+        - name: fluentd
+          image: robotshop/fluentd:elastic
+          #args:
+          #  - "-v"
+          resources:
+            limits:
+              cpu: 500m
+              memory: 500Mi
+            requests:
+              cpu: 100m
+              memory: 200Mi
+          volumeMounts:
+            - name: fluentd-config
+              mountPath: /fluentd/etc
+            - name: varlog
+              mountPath: /var/log
+            - name: varlibdockercontainers
+              mountPath: /var/lib/docker/containers
+              readOnly: true
+      volumes:
+        - name: fluentd-config
+          configMap:
+            name: fluentd-config
+        - name: varlog
+          hostPath:
+            path: /var/log
+        - name: varlibdockercontainers
+          hostPath:
+            path: /var/lib/docker/containers
+

fluentd/build.sh

@@ -0,0 +1,12 @@
+#!/bin/sh
+
+IMAGE_NAME="robotshop/fluentd:elastic"
+
+
+docker build -t "$IMAGE_NAME" .
+
+if [ "$1" = "push" ]
+then
+    docker push "$IMAGE_NAME"
+fi
+