imfreedom/k8s-cluster
Clone
Summary
Browse
Changes
Graph
update the keep manifests to include hgweb
2019-12-18, Gary Kramlich
e9ac2e1d1613
update the keep manifests to include hgweb
# This manifest sets up an the FluentBit portion of the EFK stack to capture
# logging from all of the pods in the cluster.
---
apiVersion
:
v1
kind
:
ServiceAccount
metadata
:
name
:
fluent-bit
namespace
:
kube-logging
---
apiVersion
:
rbac.authorization.k8s.io/v1beta1
kind
:
ClusterRole
metadata
:
name
:
fluent-bit-read
rules
:
-
apiGroups
:
[
""
]
resources
:
-
namespaces
-
pods
verbs
:
[
"get"
,
"list"
,
"watch"
]
---
apiVersion
:
rbac.authorization.k8s.io/v1beta1
kind
:
ClusterRoleBinding
metadata
:
name
:
fluent-bit-read
roleRef
:
apiGroup
:
rbac.authorization.k8s.io
kind
:
ClusterRole
name
:
fluent-bit-read
subjects
:
-
kind
:
ServiceAccount
name
:
fluent-bit
namespace
:
kube-logging
---
apiVersion
:
v1
kind
:
ConfigMap
metadata
:
name
:
fluent-bit-config
namespace
:
kube-logging
labels
:
k8s-app
:
fluent-bit
data
:
# Configuration files: server, input, filters and output
# ======================================================
fluent-bit.conf
:
|
[SERVICE]
Flush 1
Log_Level info
Daemon off
Parsers_File parsers.conf
HTTP_Server On
HTTP_Listen 0.0.0.0
HTTP_Port 2020
@INCLUDE input-kubernetes.conf
@INCLUDE filter-kubernetes.conf
@INCLUDE output-elasticsearch.conf
input-kubernetes.conf
:
|
[INPUT]
Name tail
Tag kube.*
Path /var/log/containers/*.log
Parser docker
DB /var/log/flb_kube.db
Mem_Buf_Limit 5MB
Skip_Long_Lines On
Refresh_Interval 10
filter-kubernetes.conf
:
|
[FILTER]
Name kubernetes
Match kube.*
Kube_URL https://kubernetes.default.svc:443
Kube_CA_File /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
Kube_Token_File /var/run/secrets/kubernetes.io/serviceaccount/token
Kube_Tag_Prefix kube.var.log.containers.
Merge_Log On
Merge_Log_Key log_processed
K8S-Logging.Parser On
K8S-Logging.Exclude Off
output-elasticsearch.conf
:
|
[OUTPUT]
Name es
Match *
Host ${FLUENT_ELASTICSEARCH_HOST}
Port ${FLUENT_ELASTICSEARCH_PORT}
Logstash_Format On
Replace_Dots On
Retry_Limit False
parsers.conf
:
|
[PARSER]
Name apache
Format regex
Regex ^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$
Time_Key time
Time_Format %d/%b/%Y:%H:%M:%S %z
[PARSER]
Name apache2
Format regex
Regex ^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$
Time_Key time
Time_Format %d/%b/%Y:%H:%M:%S %z
[PARSER]
Name apache_error
Format regex
Regex ^\[[^ ]* (?<time>[^\]]*)\] \[(?<level>[^\]]*)\](?: \[pid (?<pid>[^\]]*)\])?( \[client (?<client>[^\]]*)\])? (?<message>.*)$
[PARSER]
Name nginx
Format regex
Regex ^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$
Time_Key time
Time_Format %d/%b/%Y:%H:%M:%S %z
[PARSER]
Name json
Format json
Time_Key time
Time_Format %d/%b/%Y:%H:%M:%S %z
[PARSER]
Name docker
Format json
Time_Key time
Time_Format %Y-%m-%dT%H:%M:%S.%L
Time_Keep On
[PARSER]
Name syslog
Format regex
Regex ^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$
Time_Key time
Time_Format %b %d %H:%M:%S
[PARSER]
Name bamboo
Format regex
Regex ^(?<time>[^ ]+ [^ ]+) (?<level>[^ ]+) \[(?<process>[^ ]+)\] \[(?<logger>[^ ]+)\] (?<user>[^ ]+) (?<method>[^ ]+) (?<url>[^ ]+) (?<response_size>[^ ]+)$
Time_Key time
Time_Format %Y-%m-%d %H:%M:%S,%L
[PARSER]
Name ingress-nginx
Format regex
Regex ^(?<host>[^ ]*) - \[(?<real_ip>[^ ]*)\] - (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<status_code>[^ ]*) (?<size>[^ ]*) "(?<referer>[^\"]*)" "(?<agent>[^\"]*)" (?<request_length>[^ ]*) (?<request_time>[^ ]*) \[(?<proxy_upstream_name>[^ ]*)\] (?<upstream_addr>[^ ]*) (?<upstream_response_length>[^ ]*) (?<upstream_response_time>[^ ]*) (?<upstream_status>[^ ]*) (?<last>[^$]*)
Time_Key time
Time_Format %d/%b/%Y:%H:%M:%S %z
---
apiVersion
:
apps/v1
kind
:
DaemonSet
metadata
:
name
:
fluent-bit
namespace
:
kube-logging
labels
:
k8s-app
:
"fluent-bit-logging"
kubernetes.io/cluster-service
:
"true"
spec
:
updateStrategy
:
rollingUpdate
:
maxUnavailable
:
1
minReadySeconds
:
10
selector
:
matchLabels
:
k8s-app
:
"fluent-bit-logging"
kubernetes.io/cluster-service
:
"true"
template
:
metadata
:
labels
:
k8s-app
:
"fluent-bit-logging"
kubernetes.io/cluster-service
:
"true"
annotations
:
prometheus.io/scrape
:
"true"
prometheus.io/port
:
"2020"
prometheus.io/path
:
/api/v1/metrics/prometheus
spec
:
containers
:
-
name
:
fluent-bit
image
:
fluent/fluent-bit:1.2.2
imagePullPolicy
:
Always
ports
:
-
containerPort
:
2020
env
:
-
name
:
FLUENT_ELASTICSEARCH_HOST
value
:
"elasticsearch"
-
name
:
FLUENT_ELASTICSEARCH_PORT
value
:
"9200"
volumeMounts
:
-
name
:
varlog
mountPath
:
/var/log
-
name
:
varlibdockercontainers
mountPath
:
/var/lib/docker/containers
readOnly
:
true
-
name
:
fluent-bit-config
mountPath
:
/fluent-bit/etc/
terminationGracePeriodSeconds
:
10
volumes
:
-
name
:
varlog
hostPath
:
path
:
/var/log
-
name
:
varlibdockercontainers
hostPath
:
path
:
/var/lib/docker/containers
-
name
:
fluent-bit-config
configMap
:
name
:
fluent-bit-config
serviceAccountName
:
fluent-bit
tolerations
:
-
key
:
node-role.kubernetes.io/master
operator
:
Exists
effect
:
NoSchedule
-
operator
:
"Exists"
effect
:
"NoExecute"
-
operator
:
"Exists"
effect
:
"NoSchedule"
---