From 056dd8a8b0bcbe95a115cd1fbc104290f7d14dc1 Mon Sep 17 00:00:00 2001 From: CarlosEDP Date: Fri, 3 May 2019 14:00:05 -0300 Subject: [PATCH 1/3] Change domain to generic nip.io. Use http Kubelet port --- main.jsonnet | 2 ++ manifests/grafana-config.yaml | 2 +- manifests/ingress-alertmanager-main.yaml | 2 +- manifests/ingress-grafana.yaml | 2 +- manifests/ingress-prometheus-k8s.yaml | 2 +- manifests/prometheus-prometheus.yaml | 2 +- .../prometheus-serviceMonitorKubelet.yaml | 18 ++++-------------- vars.jsonnet | 6 +++--- 8 files changed, 14 insertions(+), 22 deletions(-) diff --git a/main.jsonnet b/main.jsonnet index 58b851b..735e674 100644 --- a/main.jsonnet +++ b/main.jsonnet @@ -5,6 +5,8 @@ local kp = (import 'kube-prometheus/kube-prometheus.libsonnet') + (import 'kube-prometheus/kube-prometheus-anti-affinity.libsonnet') + (import 'kube-prometheus/kube-prometheus-kops-coredns.libsonnet') + (import 'kube-prometheus/kube-prometheus-kubeadm.libsonnet') + // Use http Kubelet targets. Comment to revert to https + + (import 'kube-prometheus/kube-prometheus-insecure-kubelet.libsonnet') + (import 'base_operator_stack.jsonnet') + (import 'smtp_server.jsonnet') // Additional Modules diff --git a/manifests/grafana-config.yaml b/manifests/grafana-config.yaml index 14966e2..750e3c8 100644 --- a/manifests/grafana-config.yaml +++ b/manifests/grafana-config.yaml @@ -1,6 +1,6 @@ apiVersion: v1 data: - grafana.ini: W2F1dGguYW5vbnltb3VzXQplbmFibGVkID0gZmFsc2UKW2F1dGguYmFzaWNdCmVuYWJsZWQgPSBmYWxzZQpbc2Vzc2lvbl0KcHJvdmlkZXIgPSBtZW1vcnkKW3NtdHBdCmVuYWJsZWQgPSB0cnVlCmZyb21fYWRkcmVzcyA9IGNhcmxvc2VkcEBnbWFpbC5jb20KZnJvbV9uYW1lID0gR3JhZmFuYSBBbGVydApob3N0ID0gc210cC1zZXJ2ZXIubW9uaXRvcmluZy5zdmM6MjUKcGFzc3dvcmQgPSAKc2tpcF92ZXJpZnkgPSB0cnVlCnVzZXIgPSAK + grafana.ini: W2F1dGguYW5vbnltb3VzXQplbmFibGVkID0gZmFsc2UKW2F1dGguYmFzaWNdCmVuYWJsZWQgPSBmYWxzZQpbc2Vzc2lvbl0KcHJvdmlkZXIgPSBtZW1vcnkKW3NtdHBdCmVuYWJsZWQgPSB0cnVlCmZyb21fYWRkcmVzcyA9IG15ZW1haWxAZ21haWwuY29tCmZyb21fbmFtZSA9IEdyYWZhbmEgQWxlcnQKaG9zdCA9IHNtdHAtc2VydmVyLm1vbml0b3Jpbmcuc3ZjOjI1CnBhc3N3b3JkID0gCnNraXBfdmVyaWZ5ID0gdHJ1ZQp1c2VyID0gCg== kind: Secret metadata: name: grafana-config diff --git a/manifests/ingress-alertmanager-main.yaml b/manifests/ingress-alertmanager-main.yaml index 540a0b2..343d8e4 100644 --- a/manifests/ingress-alertmanager-main.yaml +++ b/manifests/ingress-alertmanager-main.yaml @@ -5,7 +5,7 @@ metadata: namespace: monitoring spec: rules: - - host: alertmanager.internal.carlosedp.com + - host: alertmanager.192.168.99.102.nip.io http: paths: - backend: diff --git a/manifests/ingress-grafana.yaml b/manifests/ingress-grafana.yaml index cd56db0..defc89a 100644 --- a/manifests/ingress-grafana.yaml +++ b/manifests/ingress-grafana.yaml @@ -5,7 +5,7 @@ metadata: namespace: monitoring spec: rules: - - host: grafana.internal.carlosedp.com + - host: grafana.192.168.99.102.nip.io http: paths: - backend: diff --git a/manifests/ingress-prometheus-k8s.yaml b/manifests/ingress-prometheus-k8s.yaml index aad643d..b8542b3 100644 --- a/manifests/ingress-prometheus-k8s.yaml +++ b/manifests/ingress-prometheus-k8s.yaml @@ -5,7 +5,7 @@ metadata: namespace: monitoring spec: rules: - - host: prometheus.internal.carlosedp.com + - host: prometheus.192.168.99.102.nip.io http: paths: - backend: diff --git a/manifests/prometheus-prometheus.yaml b/manifests/prometheus-prometheus.yaml index d9280c8..542828a 100644 --- a/manifests/prometheus-prometheus.yaml +++ b/manifests/prometheus-prometheus.yaml @@ -26,7 +26,7 @@ spec: namespace: monitoring port: web baseImage: carlosedp/prometheus - externalUrl: http://prometheus.internal.carlosedp.com + externalUrl: http://prometheus.192.168.99.102.nip.io nodeSelector: beta.kubernetes.io/os: linux replicas: 1 diff --git a/manifests/prometheus-serviceMonitorKubelet.yaml b/manifests/prometheus-serviceMonitorKubelet.yaml index 91da377..71b95a5 100644 --- a/manifests/prometheus-serviceMonitorKubelet.yaml +++ b/manifests/prometheus-serviceMonitorKubelet.yaml @@ -8,25 +8,15 @@ metadata: spec: endpoints: - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token - honorLabels: true interval: 30s - port: https-metrics - scheme: https - tlsConfig: - insecureSkipVerify: true + port: http-metrics + scheme: http - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token honorLabels: true interval: 30s - metricRelabelings: - - action: drop - regex: container_(network_tcp_usage_total|network_udp_usage_total|tasks_state|cpu_load_average_10s) - sourceLabels: - - __name__ path: /metrics/cadvisor - port: https-metrics - scheme: https - tlsConfig: - insecureSkipVerify: true + port: http-metrics + scheme: http jobLabel: k8s-app namespaceSelector: matchNames: diff --git a/vars.jsonnet b/vars.jsonnet index 145b502..f130685 100644 --- a/vars.jsonnet +++ b/vars.jsonnet @@ -15,10 +15,10 @@ }, // Domain suffix for the ingresses - suffixDomain: "internal.carlosedp.com", + suffixDomain: "192.168.99.102.nip.io", - // Grafana from email + // Grafana "from" email grafana: { - from_address: 'carlosedp@gmail.com', + from_address: 'myemail@gmail.com', }, } \ No newline at end of file From ad1677dad5a34e0f3f0ac831207763d05dc75a07 Mon Sep 17 00:00:00 2001 From: CarlosEDP Date: Tue, 7 May 2019 12:16:16 -0300 Subject: [PATCH 2/3] Start working on dynamic jsonnet loading for modules --- base_operator_stack.jsonnet | 28 +++++++++--------- main.jsonnet | 57 ++++++++++++++++++++++++++----------- metallb.jsonnet | 2 +- traefik.jsonnet | 2 +- vars.jsonnet | 38 +++++++++++++++++++------ 5 files changed, 86 insertions(+), 41 deletions(-) diff --git a/base_operator_stack.jsonnet b/base_operator_stack.jsonnet index 68cb364..c30fcfe 100644 --- a/base_operator_stack.jsonnet +++ b/base_operator_stack.jsonnet @@ -67,24 +67,24 @@ local vars = import 'vars.jsonnet'; local pvc = k.core.v1.persistentVolumeClaim, prometheus+: { spec+: { - retention: '15d', - externalUrl: 'http://' + $._config.urls.prom_ingress, - } - + ( if vars.enablePersistence['prometheus'] then { - storage: { - volumeClaimTemplate: - pvc.new() + - pvc.mixin.spec.withAccessModes('ReadWriteOnce') + - pvc.mixin.spec.resources.withRequests({ storage: '20Gi' }), - // Uncomment below to define a StorageClass name - //+ pvc.mixin.spec.withStorageClassName('nfs-master-ssd'), - }, - } else {}), + retention: '15d', + externalUrl: 'http://' + $._config.urls.prom_ingress, + } + + (if vars.enablePersistence.prometheus then { + storage: { + volumeClaimTemplate: + pvc.new() + + pvc.mixin.spec.withAccessModes('ReadWriteOnce') + + pvc.mixin.spec.resources.withRequests({ storage: '20Gi' }), + // Uncomment below to define a StorageClass name + //+ pvc.mixin.spec.withStorageClassName('nfs-master-ssd'), + }, + } else {}), }, }, // Override deployment for Grafana data persistence - grafana+:: if vars.enablePersistence['grafana'] then { + grafana+:: if vars.enablePersistence.grafana then { deployment+: { spec+: { template+: { diff --git a/main.jsonnet b/main.jsonnet index 735e674..a346dc4 100644 --- a/main.jsonnet +++ b/main.jsonnet @@ -1,5 +1,14 @@ local k = import 'ksonnet/ksonnet.beta.3/k.libsonnet'; local vars = import 'vars.jsonnet'; +local enabledModules = [module.name for module in vars.modules if module.enabled]; + +local join_objects(objs) = + local aux(arr, i, running) = + if i >= std.length(arr) then + running + else + aux(arr, i + 1, running + arr[i]) tailstrict; + aux(objs, 0, {}); local kp = (import 'kube-prometheus/kube-prometheus.libsonnet') + (import 'kube-prometheus/kube-prometheus-anti-affinity.libsonnet') @@ -10,11 +19,7 @@ local kp = (import 'kube-prometheus/kube-prometheus.libsonnet') + (import 'base_operator_stack.jsonnet') + (import 'smtp_server.jsonnet') // Additional Modules - + (if vars.installModules['arm-exporter'] then (import 'arm_exporter.jsonnet') else {}) - + (if vars.installModules.metallb then (import 'metallb.jsonnet') else {}) - + (if vars.installModules.traefik then (import 'traefik.jsonnet') else {}) - + (if vars.installModules['ups-exporter'] then (import 'ups_exporter.jsonnet') else {}) - + (if vars.installModules['elastic-exporter'] then (import 'elasticsearch_exporter.jsonnet') else {}) + + join_objects([module.file for module in vars.modules if module.enabled]) // Load image versions last to override default from modules + (import 'image_sources_versions.jsonnet'); @@ -27,15 +32,35 @@ local kp = (import 'kube-prometheus/kube-prometheus.libsonnet') { ['node-exporter-' + name]: kp.nodeExporter[name] for name in std.objectFields(kp.nodeExporter) } + { ['grafana-' + name]: kp.grafana[name] for name in std.objectFields(kp.grafana) } + { ['ingress-' + name]: kp.ingress[name] for name in std.objectFields(kp.ingress) } + -{ ['smtp-server-' + name]: kp.smtpServer[name] for name in std.objectFields(kp.smtpServer) } + +{ ['smtp-server-' + name]: kp.smtpServer[name] for name in std.objectFields(kp.smtpServer) } // Additional Modules -(if vars.installModules['arm-exporter'] then - { ['arm-exporter-' + name]: kp.armExporter[name] for name in std.objectFields(kp.armExporter) } else {}) + -(if vars.installModules.metallb then - { ['metallb-' + name]: kp.metallb[name] for name in std.objectFields(kp.metallb) } else {}) + -(if vars.installModules.traefik then - { ['traefik-' + name]: kp.traefik[name] for name in std.objectFields(kp.traefik) } else {}) + -(if vars.installModules['ups-exporter'] then - { ['ups-exporter-' + name]: kp.upsExporter[name] for name in std.objectFields(kp.upsExporter) } else {}) + -(if vars.installModules['elastic-exporter'] then - { ['elasticexporter-' + name]: kp.elasticExporter[name] for name in std.objectFields(kp.elasticExporter) } else {}) + +{ + [std.asciiLower(moduleName) + '-' + objName]: {kp.moduleName[objName]} + for objName in [std.objectFields(kp[moduleName]] for moduleName in enabledModules +} + + +// { +// local items = ["a", "b", "c"], +// joined: { +// [x + i]: { +// data: "x = %s and i = %s" % [x, i], +// } for i in ["1", "2"] for x in items +// } +// } + +// { ['arm-exporter-' + name]: kp.armExporter[name] for name in std.objectFields(kp.armExporter) } + +// {[std.asciiLower(moduleName) + '-' + objName]: kp.moduleName[objName] for objName in [std.objectFields(kp[moduleName]) for moduleName in enabledModules]} + +// (if vars.installModules['arm-exporter'] then +// { ['arm-exporter-' + name]: kp.armExporter[name] for name in std.objectFields(kp.armExporter) } else {}) + +// (if vars.installModules.metallb then +// { ['metallb-' + name]: kp.metallb[name] for name in std.objectFields(kp.metallb) } else {}) + +// (if vars.installModules.traefik then +// { ['traefik-' + name]: kp.traefik[name] for name in std.objectFields(kp.traefik) } else {}) + +// (if vars.installModules['ups-exporter'] then +// { ['ups-exporter-' + name]: kp.upsExporter[name] for name in std.objectFields(kp.upsExporter) } else {}) + +// (if vars.installModules['elastic-exporter'] then +// { ['elasticexporter-' + name]: kp.elasticExporter[name] for name in std.objectFields(kp.elasticExporter) } else {}) diff --git a/metallb.jsonnet b/metallb.jsonnet index 8500a9d..a9874de 100644 --- a/metallb.jsonnet +++ b/metallb.jsonnet @@ -10,7 +10,7 @@ local k = import 'ksonnet/ksonnet.beta.3/k.libsonnet'; }, - metallb+:: { + metallbExporter+:: { serviceMonitor: { apiVersion: 'monitoring.coreos.com/v1', diff --git a/traefik.jsonnet b/traefik.jsonnet index 6cafe98..2e6223e 100644 --- a/traefik.jsonnet +++ b/traefik.jsonnet @@ -9,7 +9,7 @@ local k = import 'ksonnet/ksonnet.beta.3/k.libsonnet'; }, }, - traefik+:: { + traefikExporter+:: { serviceMonitor: { apiVersion: 'monitoring.coreos.com/v1', diff --git a/vars.jsonnet b/vars.jsonnet index f130685..1f01c51 100644 --- a/vars.jsonnet +++ b/vars.jsonnet @@ -1,12 +1,32 @@ { // Enable or disable additional modules - installModules: { - 'arm-exporter': false, - metallb: false, - traefik: false, - 'ups-exporter': false, - 'elastic-exporter': false, - }, + modules: [ + { + name: 'armExporter', + enabled: true, + file: import 'arm_exporter.jsonnet', + }, + { + name: 'upsExporter', + enabled: true, + file: import 'ups_exporter.jsonnet', + }, + { + name: 'metallbExporter', + enabled: true, + file: import 'metallb.jsonnet', + }, + { + name: 'traefikExporter', + enabled: true, + file: import 'traefik.jsonnet', + }, + { + name: 'elasticExporter', + enabled: true, + file: import 'elasticsearch_exporter.jsonnet', + }, + ], // Setting these to false, defaults to emptyDirs enablePersistence: { @@ -15,10 +35,10 @@ }, // Domain suffix for the ingresses - suffixDomain: "192.168.99.102.nip.io", + suffixDomain: '192.168.99.100.nip.io', // Grafana "from" email grafana: { from_address: 'myemail@gmail.com', }, -} \ No newline at end of file +} From 87ed1bf6653a869dda0308fc68b6fbb45415aafa Mon Sep 17 00:00:00 2001 From: CarlosEDP Date: Wed, 15 May 2019 17:28:47 -0300 Subject: [PATCH 3/3] Load additional modules dynamically from vars module definition --- main.jsonnet | 66 ++++++++---------------- manifests/ingress-alertmanager-main.yaml | 2 +- manifests/ingress-grafana.yaml | 2 +- manifests/ingress-prometheus-k8s.yaml | 2 +- manifests/prometheus-prometheus.yaml | 2 +- vars.jsonnet | 10 ++-- 6 files changed, 30 insertions(+), 54 deletions(-) diff --git a/main.jsonnet b/main.jsonnet index a346dc4..7126b12 100644 --- a/main.jsonnet +++ b/main.jsonnet @@ -1,14 +1,13 @@ local k = import 'ksonnet/ksonnet.beta.3/k.libsonnet'; local vars = import 'vars.jsonnet'; -local enabledModules = [module.name for module in vars.modules if module.enabled]; local join_objects(objs) = - local aux(arr, i, running) = - if i >= std.length(arr) then - running - else - aux(arr, i + 1, running + arr[i]) tailstrict; - aux(objs, 0, {}); + local aux(arr, i, running) = + if i >= std.length(arr) then + running + else + aux(arr, i + 1, running + arr[i]) tailstrict; + aux(objs, 0, {}); local kp = (import 'kube-prometheus/kube-prometheus.libsonnet') + (import 'kube-prometheus/kube-prometheus-anti-affinity.libsonnet') @@ -18,49 +17,26 @@ local kp = (import 'kube-prometheus/kube-prometheus.libsonnet') + (import 'kube-prometheus/kube-prometheus-insecure-kubelet.libsonnet') + (import 'base_operator_stack.jsonnet') + (import 'smtp_server.jsonnet') - // Additional Modules + // Additional modules are loaded dynamically from vars.jsonnet + join_objects([module.file for module in vars.modules if module.enabled]) // Load image versions last to override default from modules + (import 'image_sources_versions.jsonnet'); +// Generate core modules { ['00namespace-' + name]: kp.kubePrometheus[name] for name in std.objectFields(kp.kubePrometheus) } -{ ['0prometheus-operator-' + name]: kp.prometheusOperator[name] for name in std.objectFields(kp.prometheusOperator) } + -{ ['prometheus-adapter-' + name]: kp.prometheusAdapter[name] for name in std.objectFields(kp.prometheusAdapter) } + -{ ['prometheus-' + name]: kp.prometheus[name] for name in std.objectFields(kp.prometheus) } + -{ ['alertmanager-' + name]: kp.alertmanager[name] for name in std.objectFields(kp.alertmanager) } + -{ ['kube-state-metrics-' + name]: kp.kubeStateMetrics[name] for name in std.objectFields(kp.kubeStateMetrics) } + -{ ['node-exporter-' + name]: kp.nodeExporter[name] for name in std.objectFields(kp.nodeExporter) } + -{ ['grafana-' + name]: kp.grafana[name] for name in std.objectFields(kp.grafana) } + -{ ['ingress-' + name]: kp.ingress[name] for name in std.objectFields(kp.ingress) } + +{ ['0prometheus-operator-' + name]: kp.prometheusOperator[name] for name in std.objectFields(kp.prometheusOperator) } +{ ['prometheus-adapter-' + name]: kp.prometheusAdapter[name] for name in std.objectFields(kp.prometheusAdapter) } +{ ['prometheus-' + name]: kp.prometheus[name] for name in std.objectFields(kp.prometheus) } +{ ['alertmanager-' + name]: kp.alertmanager[name] for name in std.objectFields(kp.alertmanager) } +{ ['kube-state-metrics-' + name]: kp.kubeStateMetrics[name] for name in std.objectFields(kp.kubeStateMetrics) } +{ ['node-exporter-' + name]: kp.nodeExporter[name] for name in std.objectFields(kp.nodeExporter) } +{ ['grafana-' + name]: kp.grafana[name] for name in std.objectFields(kp.grafana) } +{ ['ingress-' + name]: kp.ingress[name] for name in std.objectFields(kp.ingress) } { ['smtp-server-' + name]: kp.smtpServer[name] for name in std.objectFields(kp.smtpServer) } -// Additional Modules -{ - [std.asciiLower(moduleName) + '-' + objName]: {kp.moduleName[objName]} - for objName in [std.objectFields(kp[moduleName]] for moduleName in enabledModules +{ // Dynamically generate additional modules from vars.jsonnet + [std.asciiLower(module.name) + '-' + name]: kp[module.name][name] + for module in vars.modules + if module.enabled + for name in std.objectFields(kp[module.name]) } - - -// { -// local items = ["a", "b", "c"], -// joined: { -// [x + i]: { -// data: "x = %s and i = %s" % [x, i], -// } for i in ["1", "2"] for x in items -// } -// } - -// { ['arm-exporter-' + name]: kp.armExporter[name] for name in std.objectFields(kp.armExporter) } - -// {[std.asciiLower(moduleName) + '-' + objName]: kp.moduleName[objName] for objName in [std.objectFields(kp[moduleName]) for moduleName in enabledModules]} - -// (if vars.installModules['arm-exporter'] then -// { ['arm-exporter-' + name]: kp.armExporter[name] for name in std.objectFields(kp.armExporter) } else {}) + -// (if vars.installModules.metallb then -// { ['metallb-' + name]: kp.metallb[name] for name in std.objectFields(kp.metallb) } else {}) + -// (if vars.installModules.traefik then -// { ['traefik-' + name]: kp.traefik[name] for name in std.objectFields(kp.traefik) } else {}) + -// (if vars.installModules['ups-exporter'] then -// { ['ups-exporter-' + name]: kp.upsExporter[name] for name in std.objectFields(kp.upsExporter) } else {}) + -// (if vars.installModules['elastic-exporter'] then -// { ['elasticexporter-' + name]: kp.elasticExporter[name] for name in std.objectFields(kp.elasticExporter) } else {}) diff --git a/manifests/ingress-alertmanager-main.yaml b/manifests/ingress-alertmanager-main.yaml index 343d8e4..3b0723a 100644 --- a/manifests/ingress-alertmanager-main.yaml +++ b/manifests/ingress-alertmanager-main.yaml @@ -5,7 +5,7 @@ metadata: namespace: monitoring spec: rules: - - host: alertmanager.192.168.99.102.nip.io + - host: alertmanager.192.168.99.100.nip.io http: paths: - backend: diff --git a/manifests/ingress-grafana.yaml b/manifests/ingress-grafana.yaml index defc89a..7b058b0 100644 --- a/manifests/ingress-grafana.yaml +++ b/manifests/ingress-grafana.yaml @@ -5,7 +5,7 @@ metadata: namespace: monitoring spec: rules: - - host: grafana.192.168.99.102.nip.io + - host: grafana.192.168.99.100.nip.io http: paths: - backend: diff --git a/manifests/ingress-prometheus-k8s.yaml b/manifests/ingress-prometheus-k8s.yaml index b8542b3..fc6b211 100644 --- a/manifests/ingress-prometheus-k8s.yaml +++ b/manifests/ingress-prometheus-k8s.yaml @@ -5,7 +5,7 @@ metadata: namespace: monitoring spec: rules: - - host: prometheus.192.168.99.102.nip.io + - host: prometheus.192.168.99.100.nip.io http: paths: - backend: diff --git a/manifests/prometheus-prometheus.yaml b/manifests/prometheus-prometheus.yaml index 542828a..fe21f9c 100644 --- a/manifests/prometheus-prometheus.yaml +++ b/manifests/prometheus-prometheus.yaml @@ -26,7 +26,7 @@ spec: namespace: monitoring port: web baseImage: carlosedp/prometheus - externalUrl: http://prometheus.192.168.99.102.nip.io + externalUrl: http://prometheus.192.168.99.100.nip.io nodeSelector: beta.kubernetes.io/os: linux replicas: 1 diff --git a/vars.jsonnet b/vars.jsonnet index 1f01c51..2e490fd 100644 --- a/vars.jsonnet +++ b/vars.jsonnet @@ -3,27 +3,27 @@ modules: [ { name: 'armExporter', - enabled: true, + enabled: false, file: import 'arm_exporter.jsonnet', }, { name: 'upsExporter', - enabled: true, + enabled: false, file: import 'ups_exporter.jsonnet', }, { name: 'metallbExporter', - enabled: true, + enabled: false, file: import 'metallb.jsonnet', }, { name: 'traefikExporter', - enabled: true, + enabled: false, file: import 'traefik.jsonnet', }, { name: 'elasticExporter', - enabled: true, + enabled: false, file: import 'elasticsearch_exporter.jsonnet', }, ],