From 5a22b100b25dec9b457d036be2af9e3d69da99b9 Mon Sep 17 00:00:00 2001 From: Steve Waterworth Date: Fri, 19 Mar 2021 10:17:14 +0000 Subject: [PATCH] reorganise directory --- fluentd/{Docker => Docker-Compose}/README.md | 10 +- .../humio.conf => Docker-Compose/fluent.conf} | 0 fluentd/{Docker => Docker-Compose}/run.sh | 5 +- fluentd/Docker/Dockerfile | 5 - fluentd/Docker/build.sh | 6 - fluentd/Docker/setenv.sh | 4 - fluentd/Dockerfile | 10 ++ fluentd/Kubernetes/README.md | 10 ++ fluentd/Kubernetes/fluentd.yaml | 147 ++++++++++++++++++ fluentd/build.sh | 12 ++ 10 files changed, 185 insertions(+), 24 deletions(-) rename fluentd/{Docker => Docker-Compose}/README.md (54%) rename fluentd/{Docker/humio.conf => Docker-Compose/fluent.conf} (100%) rename fluentd/{Docker => Docker-Compose}/run.sh (51%) delete mode 100644 fluentd/Docker/Dockerfile delete mode 100644 fluentd/Docker/build.sh delete mode 100644 fluentd/Docker/setenv.sh create mode 100644 fluentd/Dockerfile create mode 100644 fluentd/Kubernetes/README.md create mode 100644 fluentd/Kubernetes/fluentd.yaml create mode 100755 fluentd/build.sh diff --git a/fluentd/Docker/README.md b/fluentd/Docker-Compose/README.md similarity index 54% rename from fluentd/Docker/README.md rename to fluentd/Docker-Compose/README.md index e7e244b..84ce949 100644 --- a/fluentd/Docker/README.md +++ b/fluentd/Docker-Compose/README.md @@ -1,10 +1,6 @@ # Configuration -Edit `setenv.sh`, set `IMAGE_NAME` to the repository, image and tag where you want to save the image. Pushing the image is not required when just running locally. - -Build the image with the `build.sh` script. - -Edit `humio.conf` setting the parameters to match either your Humio account or Elasticsearch instance. See the [fluentd documentation](https://docs.fluentd.org/output/elasticsearch) and/or [Humio documentation](https://docs.humio.com/docs/ingesting-data/data-shippers/fluentd/) for details. +Edit `fluent.conf` setting the parameters to match either your Humio account or Elasticsearch instance. See the [fluentd documentation](https://docs.fluentd.org/output/elasticsearch) and/or [Humio documentation](https://docs.humio.com/docs/ingesting-data/data-shippers/fluentd/) for details. Start `fluentd` in a Docker container using the `run.sh` script. @@ -30,5 +26,7 @@ services: If Robot Shop is already running, shut it down `docker-compose down` -Start Robot Shop with `docker-compose up -d`. It takes a few minutes to start, after that check eith Humio or ELK for log entries. +Start Robot Shop with `docker-compose up -d`. It takes a few minutes to start, after that check with Humio or ELK for log entries. + +Set up [logging integration](https://www.instana.com/docs/logging/) in Instana. diff --git a/fluentd/Docker/humio.conf b/fluentd/Docker-Compose/fluent.conf similarity index 100% rename from fluentd/Docker/humio.conf rename to fluentd/Docker-Compose/fluent.conf diff --git a/fluentd/Docker/run.sh b/fluentd/Docker-Compose/run.sh similarity index 51% rename from fluentd/Docker/run.sh rename to fluentd/Docker-Compose/run.sh index 48a16c1..4ec3a4c 100644 --- a/fluentd/Docker/run.sh +++ b/fluentd/Docker-Compose/run.sh @@ -1,13 +1,12 @@ #!/bin/sh -. ./setenv.sh +IMAGE_NAME="robotshop/fluentd:elastic" docker run \ -d \ --rm \ --name fluentd \ -p 24224:24224 \ - -v $(pwd)/humio.conf:/fluentd/etc/humio.conf \ - -e FLUENTD_CONF=humio.conf \ + -v $(pwd)/fluent.conf:/fluentd/etc/fluent.conf \ $IMAGE_NAME diff --git a/fluentd/Docker/Dockerfile b/fluentd/Docker/Dockerfile deleted file mode 100644 index 2a48f1d..0000000 --- a/fluentd/Docker/Dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -FROM fluentd -USER root -RUN fluent-gem install fluent-plugin-elasticsearch -USER fluent - diff --git a/fluentd/Docker/build.sh b/fluentd/Docker/build.sh deleted file mode 100644 index 55bae44..0000000 --- a/fluentd/Docker/build.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/sh - -. ./setenv.sh - -docker build -t $IMAGE_NAME . - diff --git a/fluentd/Docker/setenv.sh b/fluentd/Docker/setenv.sh deleted file mode 100644 index afebf19..0000000 --- a/fluentd/Docker/setenv.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh - -IMAGE_NAME="repo/image:tag" - diff --git a/fluentd/Dockerfile b/fluentd/Dockerfile new file mode 100644 index 0000000..7fd24dd --- /dev/null +++ b/fluentd/Dockerfile @@ -0,0 +1,10 @@ +FROM fluentd +USER root +RUN apk update && \ + apk add --virtual .build-dependencies build-base ruby-dev + +RUN fluent-gem install fluent-plugin-elasticsearch && \ + fluent-gem install fluent-plugin-kubernetes_metadata_filter && \ + fluent-gem install fluent-plugin-multi-format-parser +USER fluent + diff --git a/fluentd/Kubernetes/README.md b/fluentd/Kubernetes/README.md new file mode 100644 index 0000000..30d9f05 --- /dev/null +++ b/fluentd/Kubernetes/README.md @@ -0,0 +1,10 @@ +# Kubernetes + +Edit the `fluentd.yaml` file inserting your Humio or Elasticsearch instance details. + +Apply the configuration: + +```shell +$ kubectl apply -f fluentd.yaml +``` + diff --git a/fluentd/Kubernetes/fluentd.yaml b/fluentd/Kubernetes/fluentd.yaml new file mode 100644 index 0000000..3ab5036 --- /dev/null +++ b/fluentd/Kubernetes/fluentd.yaml @@ -0,0 +1,147 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: logging +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: fluentd + namespace: logging +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: fluentd + namespace: logging +rules: +- apiGroups: + - "" + resources: + - pods + - namespaces + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: fluentd + namespace: logging +roleRef: + kind: ClusterRole + name: fluentd + apiGroup: rbac.authorization.k8s.io +subjects: +- kind: ServiceAccount + name: fluentd + namespace: logging +# +# CONFIGURATION +# +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: fluentd-config + namespace: logging +data: + fluent.conf: | + + @type tail + path /var/log/containers/*.log + pos_file /var/log/fluentd-containers.log.pos + tag kubernetes.* + read_from_head false + + @type json + + + + + @type kubernetes_metadata + @id filter_kube_metadata + + + # Throw away what is not needed first + + # + #@type null + # + + + @type null + + + # Capture what is left + + @type elasticsearch + host cloud.humio.com + port 9200 + scheme https + ssl_version TLSv1_2 + logstash_format true + user + password + + +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: fluentd + namespace: logging + labels: + k8s-app: fluentd + #https://github.com/kubernetes/kubernetes/issues/51376 + #kubernetes.io/cluster-service: "true" +spec: + selector: + matchLabels: + name: fluentd + template: + metadata: + labels: + name: fluentd + #kubernetes.io/cluster-service: "true" + spec: + serviceAccount: fluentd + serviceAccountName: fluentd + terminationGracePeriodSeconds: 30 + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + containers: + - name: fluentd + image: robotshop/fluentd:elastic + #args: + # - "-v" + resources: + limits: + cpu: 500m + memory: 500Mi + requests: + cpu: 100m + memory: 200Mi + volumeMounts: + - name: fluentd-config + mountPath: /fluentd/etc + - name: varlog + mountPath: /var/log + - name: varlibdockercontainers + mountPath: /var/lib/docker/containers + readOnly: true + volumes: + - name: fluentd-config + configMap: + name: fluentd-config + - name: varlog + hostPath: + path: /var/log + - name: varlibdockercontainers + hostPath: + path: /var/lib/docker/containers + diff --git a/fluentd/build.sh b/fluentd/build.sh new file mode 100755 index 0000000..ac054d3 --- /dev/null +++ b/fluentd/build.sh @@ -0,0 +1,12 @@ +#!/bin/sh + +IMAGE_NAME="robotshop/fluentd:elastic" + + +docker build -t "$IMAGE_NAME" . + +if [ "$1" = "push" ] +then + docker push "$IMAGE_NAME" +fi +