From fb08cda75e7338c9548d10249d6ccd1286662951 Mon Sep 17 00:00:00 2001 From: Carlos de Paula Date: Thu, 26 Sep 2019 19:02:33 -0300 Subject: [PATCH] Refactor into classes --- arm_exporter.jsonnet | 40 ++----- base_operator_stack.jsonnet | 56 ++------- elasticsearch_exporter.jsonnet | 84 +++---------- k3s-overrides.jsonnet | 56 +-------- main.jsonnet | 14 +-- ...er-main.yaml => ingress-alertmanager.yaml} | 0 ...theus-k8s.yaml => ingress-prometheus.yaml} | 0 metallb.jsonnet | 34 +----- traefik.jsonnet | 33 +----- ups_exporter.jsonnet | 50 +------- utils.libsonnet | 110 ++++++++++++++++++ vars.jsonnet | 5 +- 12 files changed, 166 insertions(+), 316 deletions(-) rename manifests/{ingress-alertmanager-main.yaml => ingress-alertmanager.yaml} (100%) rename manifests/{ingress-prometheus-k8s.yaml => ingress-prometheus.yaml} (100%) create mode 100644 utils.libsonnet diff --git a/arm_exporter.jsonnet b/arm_exporter.jsonnet index de067de..af6a1fc 100644 --- a/arm_exporter.jsonnet +++ b/arm_exporter.jsonnet @@ -1,4 +1,5 @@ local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet'; +local utils = import 'utils.libsonnet'; { _config+:: { @@ -94,37 +95,16 @@ local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet'; daemonset.mixin.spec.template.spec.withNodeSelector({ 'beta.kubernetes.io/arch': 'arm64' }) + daemonset.mixin.spec.template.spec.withServiceAccountName('arm-exporter') + daemonset.mixin.spec.template.spec.withContainers(c), + serviceMonitor: - { - apiVersion: 'monitoring.coreos.com/v1', - kind: 'ServiceMonitor', - metadata: { - name: 'arm-exporter', - namespace: $._config.namespace, - labels: { - 'k8s-app': 'arm-exporter', - }, - }, - spec: { - jobLabel: 'k8s-app', - selector: { - matchLabels: { - 'k8s-app': 'arm-exporter', - }, - }, - endpoints: [ - { - port: 'https', - scheme: 'https', - interval: '30s', - bearerTokenFile: '/var/run/secrets/kubernetes.io/serviceaccount/token', - tlsConfig: { - insecureSkipVerify: true, - }, - }, - ], - }, - }, + utils.newServiceMonitorHTTPS('arm-exporter', + $._config.namespace, + {'k8s-app': 'arm-exporter'}, + $._config.namespace, + 'https', + 'https', + '/var/run/secrets/kubernetes.io/serviceaccount/token', + ), service: local service = k.core.v1.service; diff --git a/base_operator_stack.jsonnet b/base_operator_stack.jsonnet index 858f94e..a593f8d 100644 --- a/base_operator_stack.jsonnet +++ b/base_operator_stack.jsonnet @@ -1,5 +1,6 @@ local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet'; local vars = import 'vars.jsonnet'; +local utils = import 'utils.libsonnet'; { _config+:: { @@ -119,55 +120,16 @@ local vars = import 'vars.jsonnet'; grafanaDashboards+:: $._config.grafanaDashboards, // Create ingress objects per application - ingress+: { - local secret = k.core.v1.secret, - local ingress = k.extensions.v1beta1.ingress, - local ingressTls = ingress.mixin.spec.tlsType, - local ingressRule = ingress.mixin.spec.rulesType, - local httpIngressPath = ingressRule.mixin.http.pathsType, + ingress+:: { + alertmanager: + utils.newIngress('alertmanager-main', $._config.namespace, $._config.urls.alert_ingress, '/', 'alertmanager-main', 'web'), - 'alertmanager-main': - ingress.new() + - ingress.mixin.metadata.withName('alertmanager-main') + - ingress.mixin.metadata.withNamespace($._config.namespace) + - ingress.mixin.spec.withRules( - ingressRule.new() + - ingressRule.withHost($._config.urls.alert_ingress) + - ingressRule.mixin.http.withPaths( - httpIngressPath.new() + - httpIngressPath.withPath('/') + - httpIngressPath.mixin.backend.withServiceName('alertmanager-main') + - httpIngressPath.mixin.backend.withServicePort('web') - ), - ), grafana: - ingress.new() + - ingress.mixin.metadata.withName('grafana') + - ingress.mixin.metadata.withNamespace($._config.namespace) + - ingress.mixin.spec.withRules( - ingressRule.new() + - ingressRule.withHost($._config.urls.grafana_ingress) + - ingressRule.mixin.http.withPaths( - httpIngressPath.new() + - httpIngressPath.withPath('/') + - httpIngressPath.mixin.backend.withServiceName('grafana') + - httpIngressPath.mixin.backend.withServicePort('http') - ), - ), - 'prometheus-k8s': - ingress.new() + - ingress.mixin.metadata.withName('prometheus-k8s') + - ingress.mixin.metadata.withNamespace($._config.namespace) + - ingress.mixin.spec.withRules( - ingressRule.new() + - ingressRule.withHost($._config.urls.prom_ingress) + - ingressRule.mixin.http.withPaths( - httpIngressPath.new() + - httpIngressPath.withPath('/') + - httpIngressPath.mixin.backend.withServiceName('prometheus-k8s') + - httpIngressPath.mixin.backend.withServicePort('web') - ), - ), + utils.newIngress('grafana', $._config.namespace, $._config.urls.grafana_ingress, '/', 'grafana', 'http'), + + prometheus: + utils.newIngress('prometheus-k8s', $._config.namespace, $._config.urls.prom_ingress, '/', 'prometheus-k8s', 'web'), + // // Example external ingress with authentication // 'grafana-external': // ingress.new() + diff --git a/elasticsearch_exporter.jsonnet b/elasticsearch_exporter.jsonnet index 681bd8b..b29f21a 100644 --- a/elasticsearch_exporter.jsonnet +++ b/elasticsearch_exporter.jsonnet @@ -1,4 +1,5 @@ local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet'; +local utils = import 'utils.libsonnet'; { _config+:: { @@ -26,8 +27,8 @@ local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet'; elasticExporter+:: { deployment: - local deployment = k.apps.v1beta2.deployment; - local container = k.apps.v1beta2.deployment.mixin.spec.template.spec.containersType; + local deployment = k.apps.v1.deployment; + local container = k.apps.v1.deployment.mixin.spec.template.spec.containersType; local containerPort = container.portsType; local podLabels = { 'k8s-app': 'elasticsearch-exporter' }; @@ -44,7 +45,7 @@ local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet'; container.mixin.securityContext.capabilities.withDrop(['SETPCAP', 'MKNOD', 'AUDIT_WRITE', 'CHOWN', 'NET_RAW', 'DAC_OVERRIDE', 'FOWNER', 'FSETID', 'KILL', 'SETGID', 'SETUID', 'NET_BIND_SERVICE', 'SYS_CHROOT', 'SETFCAP']) + container.mixin.securityContext.withRunAsNonRoot(true) + container.mixin.securityContext.withRunAsUser(1000) + - container.mixin.securityContext.withReadOnlyRootFilesystem(true) + + // container.mixin.securityContext.withReadOnlyRootFilesystem(true) + container.mixin.resources.withRequests({ memory: '64Mi', cpu: '25m' }) + container.mixin.resources.withLimits({ memory: '128Mi', cpu: '100m' }) + container.mixin.livenessProbe.httpGet.withPath('/health') + @@ -78,69 +79,22 @@ local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet'; service.mixin.metadata.withLabels({ 'k8s-app': 'elasticsearch-exporter' }), serviceMonitorElastic: - { - apiVersion: 'monitoring.coreos.com/v1', - kind: 'ServiceMonitor', - metadata: { - name: 'elasticsearch-exporter', - namespace: $._config.namespace, - labels: { - 'k8s-app': 'elasticsearch-exporter', - }, - }, - spec: { - jobLabel: 'k8s-app', - selector: { - matchLabels: { - 'k8s-app': 'elasticsearch-exporter', - }, - }, - endpoints: [ - { - port: 'es-metrics', - scheme: 'http', - interval: '30s', - }, - ], - namespaceSelector: { - matchNames: [ - 'monitoring', - ], - }, - }, - }, + utils.newServiceMonitor( + 'elasticsearch', + $._config.namespace, + {'k8s-app': 'elasticsearch-exporter'}, + 'monitoring', + 'es-metrics', + 'http'), + serviceMonitorFluentd: - { - apiVersion: 'monitoring.coreos.com/v1', - kind: 'ServiceMonitor', - metadata: { - name: 'fluentd-es', - namespace: $._config.namespace, - labels: { - 'k8s-app': 'fluentd-es', - }, - }, - spec: { - jobLabel: 'k8s-app', - selector: { - matchLabels: { - 'k8s-app': 'fluentd-es', - }, - }, - endpoints: [ - { - port: 'metrics', - scheme: 'http', - interval: '30s', - }, - ], - namespaceSelector: { - matchNames: [ - 'logging', - ], - }, - }, - }, + utils.newServiceMonitor( + 'fluentd-es', + $._config.namespace, + {'k8s-app': 'fluentd-es'}, + 'logging', + 'metrics', + 'http'), }, // Add Prometheus monitoring rules for ElasticSearch prometheusRules+:: { diff --git a/k3s-overrides.jsonnet b/k3s-overrides.jsonnet index 8d8672e..d8ad728 100644 --- a/k3s-overrides.jsonnet +++ b/k3s-overrides.jsonnet @@ -1,62 +1,14 @@ local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet'; local vars = import 'vars.jsonnet'; -local service = k.core.v1.service; -local servicePort = k.core.v1.service.mixin.spec.portsType; +local utils = import 'utils.libsonnet'; { - prometheus+:: { - kubeControllerManagerPrometheusDiscoveryService: - service.new('kube-controller-manager-prometheus-discovery', { 'k8s-app': 'kube-controller-manager' }, servicePort.newNamed('http-metrics', 10252, 10252)) + - service.mixin.metadata.withNamespace('kube-system') + - service.mixin.metadata.withLabels({ 'k8s-app': 'kube-controller-manager' }) + - service.mixin.spec.withClusterIp('None'), + prometheus+:: { kubeControllerManagerPrometheusDiscoveryEndpoints: - local endpoints = k.core.v1.endpoints; - local endpointSubset = endpoints.subsetsType; - local endpointPort = endpointSubset.portsType; - - local Port = endpointPort.new() + - endpointPort.withName('http-metrics') + - endpointPort.withPort(10252) + - endpointPort.withProtocol('TCP'); - - local subset = endpointSubset.new() + - endpointSubset.withAddresses([ - { ip: vars.k3s.master_ip }]) + - endpointSubset.withPorts(Port); - - endpoints.new() + - endpoints.mixin.metadata.withName('kube-controller-manager-prometheus-discovery') + - endpoints.mixin.metadata.withNamespace('kube-system') + - endpoints.mixin.metadata.withLabels({ 'k8s-app': 'kube-controller-manager' }) + - endpoints.withSubsets(subset), - - kubeSchedulerPrometheusDiscoveryService: - service.new('kube-scheduler-prometheus-discovery', { 'k8s-app': 'kube-scheduler' }, servicePort.newNamed('http-metrics', 10251, 10251)) + - service.mixin.metadata.withNamespace('kube-system') + - service.mixin.metadata.withLabels({ 'k8s-app': 'kube-scheduler' }) + - service.mixin.spec.withClusterIp('None'), + utils.newEndpoint('kube-controller-manager', 'kube-system', vars.k3s.master_ip, 'http-metrics', 10252), kubeSchedulerPrometheusDiscoveryEndpoints: - local endpoints = k.core.v1.endpoints; - local endpointSubset = endpoints.subsetsType; - local endpointPort = endpointSubset.portsType; - - local Port = endpointPort.new() + - endpointPort.withName('http-metrics') + - endpointPort.withPort(10251) + - endpointPort.withProtocol('TCP'); - - local subset = endpointSubset.new() + - endpointSubset.withAddresses([ - { ip: vars.k3s.master_ip }]) + - endpointSubset.withPorts(Port); - - endpoints.new() + - endpoints.mixin.metadata.withName('kube-scheduler-prometheus-discovery') + - endpoints.mixin.metadata.withNamespace('kube-system') + - endpoints.mixin.metadata.withLabels({ 'k8s-app': 'kube-scheduler' }) + - endpoints.withSubsets(subset), + utils.newEndpoint('kube-scheduler', 'kube-system', vars.k3s.master_ip, 'http-metrics', 10251), serviceMonitorKubelet+: { diff --git a/main.jsonnet b/main.jsonnet index 094a403..330077f 100644 --- a/main.jsonnet +++ b/main.jsonnet @@ -1,13 +1,5 @@ -local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet'; local vars = import 'vars.jsonnet'; - -local join_objects(objs) = - local aux(arr, i, running) = - if i >= std.length(arr) then - running - else - aux(arr, i + 1, running + arr[i]) tailstrict; - aux(objs, 0, {}); +local utils = import 'utils.libsonnet'; local kp = (import 'kube-prometheus/kube-prometheus.libsonnet') + (import 'kube-prometheus/kube-prometheus-anti-affinity.libsonnet') @@ -17,9 +9,9 @@ local kp = (import 'kube-prometheus/kube-prometheus.libsonnet') + (import 'kube-prometheus/kube-prometheus-insecure-kubelet.libsonnet') + (import 'smtp_server.jsonnet') // Additional modules are loaded dynamically from vars.jsonnet - + join_objects([module.file for module in vars.modules if module.enabled]) + + utils.join_objects([module.file for module in vars.modules if module.enabled]) // Load K3s customized modules - + join_objects([m for m in [import 'k3s-overrides.jsonnet'] if vars.k3s.enabled]) + + utils.join_objects([m for m in [import 'k3s-overrides.jsonnet'] if vars.k3s.enabled]) // Base stack is loaded at the end to override previous definitions + (import 'base_operator_stack.jsonnet') // Load image versions last to override default from modules diff --git a/manifests/ingress-alertmanager-main.yaml b/manifests/ingress-alertmanager.yaml similarity index 100% rename from manifests/ingress-alertmanager-main.yaml rename to manifests/ingress-alertmanager.yaml diff --git a/manifests/ingress-prometheus-k8s.yaml b/manifests/ingress-prometheus.yaml similarity index 100% rename from manifests/ingress-prometheus-k8s.yaml rename to manifests/ingress-prometheus.yaml diff --git a/metallb.jsonnet b/metallb.jsonnet index 51b4b74..8aa7c4f 100644 --- a/metallb.jsonnet +++ b/metallb.jsonnet @@ -1,4 +1,5 @@ local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet'; +local utils = import 'utils.libsonnet'; { _config+:: { @@ -12,38 +13,7 @@ local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet'; metallbExporter+:: { serviceMonitor: - { - apiVersion: 'monitoring.coreos.com/v1', - kind: 'ServiceMonitor', - metadata: { - name: 'metallb', - namespace: $._config.namespace, - labels: { - 'k8s-app': 'metallb-controller', - }, - }, - spec: { - jobLabel: 'k8s-app', - selector: { - matchLabels: { - 'k8s-app': 'metallb-controller', - }, - }, - endpoints: [ - { - port: 'http', - scheme: 'http', - interval: '30s', - }, - ], - namespaceSelector: { - matchNames: [ - 'metallb-system', - ], - }, - - }, - }, + utils.newServiceMonitor('metallb', $._config.namespace, {'k8s-app': 'metallb-controller'}, 'metallb-system', 'http', 'http'), service: local service = k.core.v1.service; diff --git a/traefik.jsonnet b/traefik.jsonnet index 243c802..afbc444 100644 --- a/traefik.jsonnet +++ b/traefik.jsonnet @@ -1,4 +1,5 @@ local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet'; +local utils = import 'utils.libsonnet'; { _config+:: { @@ -11,36 +12,6 @@ local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet'; traefikExporter+:: { serviceMonitor: - { - apiVersion: 'monitoring.coreos.com/v1', - kind: 'ServiceMonitor', - metadata: { - name: 'traefik', - namespace: $._config.namespace, - labels: { - 'app': 'traefik', - }, - }, - spec: { - jobLabel: 'traefik-exporter', - selector: { - matchLabels: { - 'app': 'traefik', - }, - }, - endpoints: [ - { - port: 'metrics', - scheme: 'http', - interval: '30s', - }, - ], - namespaceSelector: { - matchNames: [ - 'kube-system', - ], - }, - }, - }, + utils.newServiceMonitor('traefik', $._config.namespace, {'app': 'traefik'}, 'kube-system', 'metrics', 'http'), }, } diff --git a/ups_exporter.jsonnet b/ups_exporter.jsonnet index c6b6832..3cb261a 100644 --- a/ups_exporter.jsonnet +++ b/ups_exporter.jsonnet @@ -1,4 +1,5 @@ local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet'; +local utils = import 'utils.libsonnet'; { _config+:: { @@ -16,32 +17,7 @@ local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet'; upsExporter+:: { serviceMonitor: - { - apiVersion: 'monitoring.coreos.com/v1', - kind: 'ServiceMonitor', - metadata: { - name: 'ups-exporter', - namespace: $._config.namespace, - labels: { - 'k8s-app': 'ups-exporter', - }, - }, - spec: { - jobLabel: 'k8s-app', - selector: { - matchLabels: { - 'k8s-app': 'ups-exporter', - }, - }, - endpoints: [ - { - port: 'metrics', - scheme: 'http', - interval: '30s', - }, - ], - }, - }, + utils.newServiceMonitor('ups-exporter', $._config.namespace, {'k8s-app': 'ups-exporter'}, $._config.namespace, 'metrics', 'http'), service: local service = k.core.v1.service; @@ -55,26 +31,6 @@ local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet'; service.mixin.spec.withClusterIp('None'), endpoints: - local endpoints = k.core.v1.endpoints; - local endpointSubset = endpoints.subsetsType; - local endpointPort = endpointSubset.portsType; - - local upsPort = endpointPort.new() + - endpointPort.withName('metrics') + - endpointPort.withPort(9099) + - endpointPort.withProtocol('TCP'); - - local subset = endpointSubset.new() + - endpointSubset.withAddresses([ - { ip: IP } - for IP in $._config.ups.ips - ]) + - endpointSubset.withPorts(upsPort); - - endpoints.new() + - endpoints.mixin.metadata.withName('ups-exporter') + - endpoints.mixin.metadata.withNamespace($._config.namespace) + - endpoints.mixin.metadata.withLabels({ 'k8s-app': 'ups-exporter' }) + - endpoints.withSubsets(subset), + utils.newEndpoint('ups-exporter', $._config.namespace, $._config.ups.ips, 'metrics', 9099), }, } diff --git a/utils.libsonnet b/utils.libsonnet new file mode 100644 index 0000000..fc32ae3 --- /dev/null +++ b/utils.libsonnet @@ -0,0 +1,110 @@ +local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet'; +local vars = import 'vars.jsonnet'; + +{ + // Join multiple objects into one + join_objects(objs):: + local aux(arr, i, running) = + if i >= std.length(arr) then + running + else + aux(arr, i + 1, running + arr[i]) tailstrict; + aux(objs, 0, {}), + + // Creates endpoint objects + newEndpoint(name, namespace, ips, portName, portNumber):: ( + local endpoints = k.core.v1.endpoints; + local endpointSubset = endpoints.subsetsType; + local endpointPort = endpointSubset.portsType; + local Port = endpointPort.new() + + endpointPort.withName(portName) + + endpointPort.withPort(portNumber) + + endpointPort.withProtocol('TCP'); + + local subset = endpointSubset.new() + + endpointSubset.withAddresses([ + { ip: IP } + for IP in ips + ]) + + endpointSubset.withPorts(Port); + endpoints.new() + + endpoints.mixin.metadata.withName(name) + + endpoints.mixin.metadata.withNamespace(namespace) + + endpoints.mixin.metadata.withLabels({ 'k8s-app': name }) + + endpoints.withSubsets(subset) + ), + + // Creates ingress objects + newIngress(name, namespace, host, path, serviceName, servicePort):: ( + local secret = k.core.v1.secret; + local ingress = k.extensions.v1beta1.ingress; + local ingressTls = ingress.mixin.spec.tlsType; + local ingressRule = ingress.mixin.spec.rulesType; + local httpIngressPath = ingressRule.mixin.http.pathsType; + + ingress.new() + + ingress.mixin.metadata.withName(name) + + ingress.mixin.metadata.withNamespace(namespace) + + ingress.mixin.spec.withRules( + ingressRule.new() + + ingressRule.withHost(host) + + ingressRule.mixin.http.withPaths( + httpIngressPath.new() + + httpIngressPath.withPath(path) + + httpIngressPath.mixin.backend.withServiceName(serviceName) + + httpIngressPath.mixin.backend.withServicePort(servicePort) + ), + ) + ), + + // Creates http ServiceMonitor objects + newServiceMonitor(name, namespace, matchLabel, matchNamespace, portName, portScheme):: ( + { + apiVersion: 'monitoring.coreos.com/v1', + kind: 'ServiceMonitor', + metadata: { + name: name, + namespace: namespace, + labels: { + 'app': name, + }, + }, + spec: { + jobLabel: name+'-exporter', + selector: { + matchLabels: matchLabel, + }, + endpoints: [ + { + port: portName, + scheme: portScheme, + interval: '30s', + }, + ], + namespaceSelector: { + matchNames: [matchNamespace], + }, + }, + } + ), + + // Creates https ServiceMonitor objects + newServiceMonitorHTTPS(name, namespace, matchLabel, matchNamespace, portName, portScheme, token):: ( + local s = $.newServiceMonitor(name, namespace, matchLabel, matchNamespace, portName, portScheme); + // Replace endpoint with https and token + local t = { + spec: { + endpoints: [{ + port: portName, + scheme: portScheme, + interval: '30s', + bearerTokenFile: token, + tlsConfig: { + insecureSkipVerify: true, + } + }], + } + }; + s + t + ), +} \ No newline at end of file diff --git a/vars.jsonnet b/vars.jsonnet index a3f2118..c294aea 100644 --- a/vars.jsonnet +++ b/vars.jsonnet @@ -1,4 +1,7 @@ { + _config+:: { + namespace: 'monitoring' + }, // Enable or disable additional modules modules: [ { @@ -30,7 +33,7 @@ k3s: { enabled: false, - master_ip: '192.168.99.100' + master_ip: ['192.168.164.120'] }, // Domain suffix for the ingresses