From c408fa7a776c98c6fd81c0bc8f5641c200bef888 Mon Sep 17 00:00:00 2001 From: Michael McCulloch Date: Sun, 14 Jan 2024 16:14:28 -0700 Subject: [PATCH] Hubble client (#2) * Optionally include the hubble cli * Add gateway API Support * Add convergeance test for cilium * and run it. * Idempotent CRD Apply --------- Co-authored-by: Michael McCulloch --- .github/workflows/ci.yml | 3 +- README.md | 21 +++++++++- defaults/main.yml | 1 + molecule/default/cilium.yml | 74 +++++++++++++++++++++++++++++++++++ tasks/control-plane-setup.yml | 51 ++++++++++++++++-------- tasks/hubble-client-setup.yml | 53 +++++++++++++++++++++++++ tasks/main.yml | 11 +++++- 7 files changed, 194 insertions(+), 20 deletions(-) create mode 100644 molecule/default/cilium.yml create mode 100644 tasks/hubble-client-setup.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2cf1bbd..05c1dd1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,7 +47,8 @@ jobs: playbook: converge.yml - distro: debian11 playbook: converge.yml - + - distro: debian11 + playbook: cilium.yml - distro: debian11 playbook: calico.yml diff --git a/README.md b/README.md index 05b760d..4b3f653 100644 --- a/README.md +++ b/README.md @@ -169,14 +169,31 @@ Flannel manifest file to apply to the Kubernetes cluster to enable networking. Y kubernetes_calico_manifest_file: https://projectcalico.docs.tigera.io/manifests/calico.yaml ``` -Cilium Helm chart values can be specified under `kubernetes_cilium_values`. [Kube Proxy Replacement](https://docs.cilium.io/en/latest/network/kubernetes/kubeproxy-free/) is supported through this method. +Calico manifest file to apply to the Kubernetes cluster (if using Calico instead of Flannel). + +#### Cilium ```yaml +kubernetes_cilium_hubble_client: true kubernetes_cilium_values: kubeProxyReplacement: true + gatewayAPI: + enabled: true ``` -Calico manifest file to apply to the Kubernetes cluster (if using Calico instead of Flannel). +Cilium Helm chart values can be specified under `kubernetes_cilium_values`. + +##### Kube Proxy Replacement + +[Kube Proxy Replacement](https://docs.cilium.io/en/latest/network/kubernetes/kubeproxy-free/) is supported through this method. The is a prerequisite for Gateway API Support. + +##### Gateway API Support + +[Gateway API Support](https://docs.cilium.io/en/latest/network/servicemesh/gateway-api/gateway-api/#gs-gateway-api). This will install additional CRDs to support GatewayAPI. + +##### Hubble Observability + +In addition to setting up [Setting up Hubble Observability](https://docs.cilium.io/en/stable/gettingstarted/hubble_setup/#hubble-setup), you may choose to install the hubble client with `kubernetes_cilium_hubble_client: true` ## Dependencies diff --git a/defaults/main.yml b/defaults/main.yml index a3ade31..dc06019 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -13,6 +13,7 @@ kubernetes_version: '1.25' kubernetes_version_rhel_package: '1.25.1' kubernetes_cilium_version: '1.14.5' kubernetes_cilium_datapath: 'native' +kubernetes_cilium_hubble_client: false kubernetes_cilium_values: "" kubernetes_role: control_plane diff --git a/molecule/default/cilium.yml b/molecule/default/cilium.yml new file mode 100644 index 0000000..33c5041 --- /dev/null +++ b/molecule/default/cilium.yml @@ -0,0 +1,74 @@ +--- +- name: Converge + hosts: all + become: true + + vars: + kubernetes_cilium_hubble_client: true + kubernetes_cilium_values: + envoy: + enabled: true + kubeProxyReplacement: true + l7Proxy: true + loadBalancer: + l7: + backend: envoy + ingressController: + enabled: true + loadbalancerMode: dedicated + default: true + hubble: + relay: + enabled: true + ui: + enabled: true + gatewayAPI: + enabled: true + kubernetes_pod_network: + cni: 'cilium' + cidr: 10.244.0.0/16 + + # Allow swap in test environments (hard to control in some envs). + kubernetes_config_kubelet_configuration: + cgroupDriver: "systemd" + failSwapOn: false + cgroupsPerQOS: true + enforceNodeAllocatable: ['pods'] + containerd_config_cgroup_driver_systemd: true + + pre_tasks: + - name: Update apt cache. + apt: update_cache=true cache_valid_time=600 + when: ansible_os_family == 'Debian' + + - name: Ensure test dependencies are installed (RedHat). + package: name=iproute state=present + when: ansible_os_family == 'RedHat' + + - name: Ensure test dependencies are installed (Debian). + package: name=iproute2 state=present + when: ansible_os_family == 'Debian' + + - name: Gather facts. + action: setup + + roles: + - role: geerlingguy.containerd + - role: geerlingguy.kubernetes + + post_tasks: + - name: Get cluster info. + command: kubectl cluster-info + changed_when: false + register: kubernetes_info + + - name: Print cluster info. + debug: var=kubernetes_info.stdout + + - name: Get all running pods. + command: kubectl get pods --all-namespaces + changed_when: false + register: kubernetes_pods + + - name: Print list of running pods. + debug: var=kubernetes_pods.stdout diff --git a/tasks/control-plane-setup.yml b/tasks/control-plane-setup.yml index 36386cb..2e86b16 100644 --- a/tasks/control-plane-setup.yml +++ b/tasks/control-plane-setup.yml @@ -16,16 +16,22 @@ (kubernetes_cilium_values.kubeProxyReplacement is defined) and (kubernetes_cilium_values.kubeProxyReplacement) | bool }}" +- name: Determine if we installing Gateway API + set_fact: + install_gateway_api="{{ (replace_kube_proxy) and + (kubernetes_cilium_values.gatewayAPI.enabled is defined) and + (kubernetes_cilium_values.gatewayAPI.enabled)| bool }}" + - name: Initialize Kubernetes control plane with kubeadm init command: > kubeadm init --config {{ kubernetes_kubeadm_kubelet_config_file_path }} {{ kubernetes_kubeadm_init_extra_opts }} register: kubeadmin_init - when: > - (not kubernetes_init_stat.stat.exists) and - (kubernetes_ignore_preflight_errors is not defined) and - (not (replace_kube_proxy)) + when: + - not kubernetes_init_stat.stat.exists + - kubernetes_ignore_preflight_errors is not defined + - not replace_kube_proxy - name: Initialize Kubernetes control plane with kubeadm init and ignore_preflight_errors command: > @@ -34,10 +40,10 @@ --ignore-preflight-errors={{ kubernetes_ignore_preflight_errors }} {{ kubernetes_kubeadm_init_extra_opts }} register: kubeadmin_init - when: > - (not kubernetes_init_stat.stat.exists) and - (kubernetes_ignore_preflight_errors is defined) and - (not (replace_kube_proxy)) + when: + - not kubernetes_init_stat.stat.exists + - kubernetes_ignore_preflight_errors is defined + - not replace_kube_proxy - name: Initialize Kubernetes control plane with kubeadm init without kube-proxy command: > @@ -46,10 +52,10 @@ --skip-phases=addon/kube-proxy {{ kubernetes_kubeadm_init_extra_opts }} register: kubeadmin_init - when: > - (not kubernetes_init_stat.stat.exists) and - (kubernetes_ignore_preflight_errors is not defined) and - (replace_kube_proxy) + when: + - not kubernetes_init_stat.stat.exists + - kubernetes_ignore_preflight_errors is not defined + - replace_kube_proxy - name: Initialize Kubernetes control plane with kubeadm init without kube-proxy and ignore_preflight_errors command: > @@ -59,10 +65,10 @@ --skip-phases=addon/kube-proxy {{ kubernetes_kubeadm_init_extra_opts }} register: kubeadmin_init - when: > - (not kubernetes_init_stat.stat.exists) and - (kubernetes_ignore_preflight_errors is defined) and - (replace_kube_proxy) + when: + - not kubernetes_init_stat.stat.exists + - kubernetes_ignore_preflight_errors is defined + - replace_kube_proxy - name: Print the init output to screen. debug: @@ -102,6 +108,19 @@ retries: 12 delay: 5 +- name: Install Prerequisite CRDs for Cilium Gateway API support. + when: install_gateway_api + register: gateway_crds + changed_when: "'created' in gateway_crds.stdout" + command: "kubectl apply -f {{ item }}" + loop: + - https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/v1.0.0/config/crd/standard/gateway.networking.k8s.io_gatewayclasses.yaml + - https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/v1.0.0/config/crd/standard/gateway.networking.k8s.io_gateways.yaml + - https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/v1.0.0/config/crd/standard/gateway.networking.k8s.io_httproutes.yaml + - https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/v1.0.0/config/crd/standard/gateway.networking.k8s.io_referencegrants.yaml + - https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/v1.0.0/config/crd/experimental/gateway.networking.k8s.io_grpcroutes.yaml + - https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/v1.0.0/config/crd/experimental/gateway.networking.k8s.io_tlsroutes.yaml + - name: Configure Cilium networking command: > /usr/local/bin/cilium install diff --git a/tasks/hubble-client-setup.yml b/tasks/hubble-client-setup.yml new file mode 100644 index 0000000..41bd634 --- /dev/null +++ b/tasks/hubble-client-setup.yml @@ -0,0 +1,53 @@ +--- +- name: Check if Hubble CLI has already been Installed. + stat: + path: /usr/local/bin/hubble + register: hubble_init_stat + when: + - kubernetes_pod_network.cni == 'cilium' + +- name: Install Hubble CLI + when: + - kubernetes_pod_network.cni == 'cilium' + - not hubble_init_stat.stat.exists + block: + - name: Get Hubble CLI version + shell: curl -s https://raw.githubusercontent.com/cilium/hubble/master/stable.txt + register: hubble_cli_version + changed_when: false + + - name: Set CLI architecture + set_fact: + cli_arch: "{{ 'arm64' if ansible_architecture == 'aarch64' else 'amd64' }}" + + + - name: Download Hubble CLI + get_url: + url: "https://github.com/cilium/hubble/releases/download/{{ hubble_cli_version.stdout }}/hubble-linux-{{ cli_arch }}.tar.gz" + dest: "/tmp/hubble-linux-{{ cli_arch }}.tar.gz" + mode: '0644' + + - name: Download Hubble CLI checksum + get_url: + url: "https://github.com/cilium/hubble/releases/download/{{ hubble_cli_version.stdout }}/hubble-linux-{{ cli_arch }}.tar.gz.sha256sum" + dest: "/tmp/hubble-linux-{{ cli_arch }}.tar.gz.sha256sum" + mode: '0644' + + - name: Verify Hubble CLI checksum + shell: sha256sum --check /tmp/hubble-linux-{{ cli_arch }}.tar.gz.sha256sum + args: + chdir: /tmp + + - name: Extract Hubble CLI + unarchive: + src: "/tmp/hubble-linux-{{ cli_arch }}.tar.gz" + dest: /usr/local/bin + remote_src: true + + - name: Remove downloaded files + file: + path: "/tmp/hubble-linux-{{ cli_arch }}.tar.gz{{ item }}" + state: absent + loop: + - '' + - '.sha256sum' diff --git a/tasks/main.yml b/tasks/main.yml index 245fe65..b507fc0 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -37,8 +37,17 @@ # Set up Cilium Client. - include_tasks: cilium-client-setup.yml when: - - kubernetes_pod_network.cni == 'cilium' - kubernetes_role == 'control_plane' + - kubernetes_pod_network.cni == 'cilium' + +# Set up hubble Client. +- include_tasks: hubble-client-setup.yml + when: + - kubernetes_role == 'control_plane' + - kubernetes_pod_network.cni == 'cilium' + - kubernetes_cilium_hubble_client + - kubernetes_cilium_values.hubble.relay.enabled is defined + - kubernetes_cilium_values.hubble.relay.enabled # Set up control plane. - include_tasks: control-plane-setup.yml