From c0fb818677cead02ff46001e320974dc957f3330 Mon Sep 17 00:00:00 2001 From: Carlos de Paula Date: Mon, 19 Aug 2019 20:20:17 -0300 Subject: [PATCH] Add config parameter and overrides to remove kube-rbac-proxy from exporters --- k3s-overrides.jsonnet | 140 ++++++++++++++++++ main.jsonnet | 6 +- manifests/kube-state-metrics-deployment.yaml | 40 +---- manifests/kube-state-metrics-service.yaml | 12 +- .../kube-state-metrics-serviceMonitor.yaml | 12 +- manifests/node-exporter-daemonset.yaml | 25 +--- manifests/node-exporter-service.yaml | 4 +- manifests/node-exporter-serviceMonitor.yaml | 9 +- vars.jsonnet | 2 + 9 files changed, 166 insertions(+), 84 deletions(-) create mode 100644 k3s-overrides.jsonnet diff --git a/k3s-overrides.jsonnet b/k3s-overrides.jsonnet new file mode 100644 index 0000000..3b917d3 --- /dev/null +++ b/k3s-overrides.jsonnet @@ -0,0 +1,140 @@ +local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet'; +local vars = import 'vars.jsonnet'; + +{ + nodeExporter+:: { + daemonset+: { + spec+: { + template+: { + spec+: { + containers: + std.filterMap( + function(c) std.startsWith(c.name, 'kube-rbac') != true, + function(c) + if std.startsWith(c.name, 'node-exporter') then + c { + args: [ + '--web.listen-address=:' + $._config.nodeExporter.port, + '--path.procfs=/host/proc', + '--path.sysfs=/host/sys', + '--path.rootfs=/host/root', + // The following settings have been taken from + // https://github.com/prometheus/node_exporter/blob/0662673/collector/filesystem_linux.go#L30-L31 + // Once node exporter is being released with those settings, this can be removed. + '--collector.filesystem.ignored-mount-points=^/(dev|proc|sys|var/lib/docker/.+)($|/)', + '--collector.filesystem.ignored-fs-types=^(autofs|binfmt_misc|cgroup|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|mqueue|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|sysfs|tracefs)$', + ], + } + else + c, + super.containers, + ), + }, + }, + }, + }, + + service+: + { + spec+: { + ports: [{ + name: 'http', + port: 9100, + targetPort: 'http' + }] + } + }, + + serviceMonitor+: + { + spec+: { + endpoints: [ + { + port: 'http', + scheme: 'http', + interval: '30s', + relabelings: [ + { + action: 'replace', + regex: '(.*)', + replacment: '$1', + sourceLabels: ['__meta_kubernetes_pod_node_name'], + targetLabel: 'instance', + }, + ], + }, + ], + }, + }, + }, + + + kubeStateMetrics+:: { + deployment+: { + spec+: { + template+: { + spec+: { + containers: + std.filterMap( + function(c) std.startsWith(c.name, 'kube-rbac') != true, + function(c) + if std.startsWith(c.name, 'kube-state-metrics') then + c { + args: [ + '--port=8080', + '--telemetry-port=8081', + ], + } + else + c, + super.containers, + ), + }, + }, + }, + }, + + service+: + { + spec+: { + ports: [{ + name: 'http-main', + port: 8080, + targetPort: 'http' + }, + { + name: 'http-self', + port: 8081, + targetPort: 'http' + }] + } + }, + + serviceMonitor+: + { + spec+: { + endpoints: [ + { + port: 'http-main', + scheme: 'http', + interval: $._config.kubeStateMetrics.scrapeInterval, + scrapeTimeout: $._config.kubeStateMetrics.scrapeTimeout, + honorLabels: true, + tlsConfig: { + insecureSkipVerify: true, + }, + }, + { + port: 'http-self', + scheme: 'https', + interval: '30s', + tlsConfig: { + insecureSkipVerify: true, + }, + }, + ], + }, + }, + }, + +} \ No newline at end of file diff --git a/main.jsonnet b/main.jsonnet index 5d0b23f..25bb626 100644 --- a/main.jsonnet +++ b/main.jsonnet @@ -15,13 +15,17 @@ local kp = (import 'kube-prometheus/kube-prometheus.libsonnet') + (import 'kube-prometheus/kube-prometheus-kubeadm.libsonnet') // Use http Kubelet targets. Comment to revert to https + (import 'kube-prometheus/kube-prometheus-insecure-kubelet.libsonnet') - + (import 'base_operator_stack.jsonnet') + (import 'smtp_server.jsonnet') // Additional modules are loaded dynamically from vars.jsonnet + join_objects([module.file for module in vars.modules if module.enabled]) + // Load K3s customized modules + + join_objects([m for m in [import 'k3s-overrides.jsonnet'] if vars.k3s]) + // Base stack is loaded at the end to override previous definitions + + (import 'base_operator_stack.jsonnet') // Load image versions last to override default from modules + (import 'image_sources_versions.jsonnet'); + // Generate core modules { ['00namespace-' + name]: kp.kubePrometheus[name] for name in std.objectFields(kp.kubePrometheus) } { ['0prometheus-operator-' + name]: kp.prometheusOperator[name] for name in std.objectFields(kp.prometheusOperator) } diff --git a/manifests/kube-state-metrics-deployment.yaml b/manifests/kube-state-metrics-deployment.yaml index 27a77bb..3d3dbdb 100644 --- a/manifests/kube-state-metrics-deployment.yaml +++ b/manifests/kube-state-metrics-deployment.yaml @@ -17,44 +17,8 @@ spec: spec: containers: - args: - - --logtostderr - - --secure-listen-address=:8443 - - --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 - - --upstream=http://127.0.0.1:8081/ - image: carlosedp/kube-rbac-proxy:v0.4.1 - name: kube-rbac-proxy-main - ports: - - containerPort: 8443 - name: https-main - resources: - limits: - cpu: 20m - memory: 40Mi - requests: - cpu: 10m - memory: 20Mi - - args: - - --logtostderr - - --secure-listen-address=:9443 - - --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 - - --upstream=http://127.0.0.1:8082/ - image: carlosedp/kube-rbac-proxy:v0.4.1 - name: kube-rbac-proxy-self - ports: - - containerPort: 9443 - name: https-self - resources: - limits: - cpu: 20m - memory: 40Mi - requests: - cpu: 10m - memory: 20Mi - - args: - - --host=127.0.0.1 - - --port=8081 - - --telemetry-host=127.0.0.1 - - --telemetry-port=8082 + - --port=8080 + - --telemetry-port=8081 image: carlosedp/kube-state-metrics:v1.7.2 name: kube-state-metrics resources: diff --git a/manifests/kube-state-metrics-service.yaml b/manifests/kube-state-metrics-service.yaml index 84927af..13b158b 100644 --- a/manifests/kube-state-metrics-service.yaml +++ b/manifests/kube-state-metrics-service.yaml @@ -8,11 +8,11 @@ metadata: spec: clusterIP: None ports: - - name: https-main - port: 8443 - targetPort: https-main - - name: https-self - port: 9443 - targetPort: https-self + - name: http-main + port: 8080 + targetPort: http + - name: http-self + port: 8081 + targetPort: http selector: app: kube-state-metrics diff --git a/manifests/kube-state-metrics-serviceMonitor.yaml b/manifests/kube-state-metrics-serviceMonitor.yaml index 2100449..cef7d5b 100644 --- a/manifests/kube-state-metrics-serviceMonitor.yaml +++ b/manifests/kube-state-metrics-serviceMonitor.yaml @@ -7,17 +7,15 @@ metadata: namespace: monitoring spec: endpoints: - - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token - honorLabels: true + - honorLabels: true interval: 30s - port: https-main - scheme: https + port: http-main + scheme: http scrapeTimeout: 30s tlsConfig: insecureSkipVerify: true - - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token - interval: 30s - port: https-self + - interval: 30s + port: http-self scheme: https tlsConfig: insecureSkipVerify: true diff --git a/manifests/node-exporter-daemonset.yaml b/manifests/node-exporter-daemonset.yaml index b6a62ff..672969b 100644 --- a/manifests/node-exporter-daemonset.yaml +++ b/manifests/node-exporter-daemonset.yaml @@ -16,7 +16,7 @@ spec: spec: containers: - args: - - --web.listen-address=127.0.0.1:9100 + - --web.listen-address=:9100 - --path.procfs=/host/proc - --path.sysfs=/host/sys - --path.rootfs=/host/root @@ -42,29 +42,6 @@ spec: mountPropagation: HostToContainer name: root readOnly: true - - args: - - --logtostderr - - --secure-listen-address=$(IP):9100 - - --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 - - --upstream=http://127.0.0.1:9100/ - env: - - name: IP - valueFrom: - fieldRef: - fieldPath: status.podIP - image: carlosedp/kube-rbac-proxy:v0.4.1 - name: kube-rbac-proxy - ports: - - containerPort: 9100 - hostPort: 9100 - name: https - resources: - limits: - cpu: 20m - memory: 60Mi - requests: - cpu: 10m - memory: 20Mi hostNetwork: true hostPID: true nodeSelector: diff --git a/manifests/node-exporter-service.yaml b/manifests/node-exporter-service.yaml index 1d728d7..6f32c7d 100644 --- a/manifests/node-exporter-service.yaml +++ b/manifests/node-exporter-service.yaml @@ -8,8 +8,8 @@ metadata: spec: clusterIP: None ports: - - name: https + - name: http port: 9100 - targetPort: https + targetPort: http selector: app: node-exporter diff --git a/manifests/node-exporter-serviceMonitor.yaml b/manifests/node-exporter-serviceMonitor.yaml index 89d65be..63b4ad2 100644 --- a/manifests/node-exporter-serviceMonitor.yaml +++ b/manifests/node-exporter-serviceMonitor.yaml @@ -7,9 +7,8 @@ metadata: namespace: monitoring spec: endpoints: - - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token - interval: 30s - port: https + - interval: 30s + port: http relabelings: - action: replace regex: (.*) @@ -17,9 +16,7 @@ spec: sourceLabels: - __meta_kubernetes_pod_node_name targetLabel: instance - scheme: https - tlsConfig: - insecureSkipVerify: true + scheme: http jobLabel: k8s-app selector: matchLabels: diff --git a/vars.jsonnet b/vars.jsonnet index 2e490fd..8bc80eb 100644 --- a/vars.jsonnet +++ b/vars.jsonnet @@ -28,6 +28,8 @@ }, ], + k3s: true, + // Setting these to false, defaults to emptyDirs enablePersistence: { prometheus: false,