Hubble client (#2)

* Optionally include the hubble cli

* Add gateway API Support

* Add convergeance test for cilium

* and run it.

* Idempotent CRD Apply

---------

Co-authored-by: Michael McCulloch <mjm.gitlab@fastmail.com>

.github/workflows/ci.yml

@@ -47,7 +47,8 @@ jobs:
             playbook: converge.yml
           - distro: debian11
             playbook: converge.yml
-
+          - distro: debian11
+            playbook: cilium.yml
           - distro: debian11
             playbook: calico.yml
 

README.md

@@ -169,14 +169,31 @@ Flannel manifest file to apply to the Kubernetes cluster to enable networking. Y
 kubernetes_calico_manifest_file: https://projectcalico.docs.tigera.io/manifests/calico.yaml
 ```
 
-Cilium Helm chart values can be specified under `kubernetes_cilium_values`. [Kube Proxy Replacement](https://docs.cilium.io/en/latest/network/kubernetes/kubeproxy-free/) is supported through this method.
+Calico manifest file to apply to the Kubernetes cluster (if using Calico instead of Flannel).
+
+#### Cilium
 
 ```yaml
+kubernetes_cilium_hubble_client: true
 kubernetes_cilium_values:
   kubeProxyReplacement: true
+  gatewayAPI:
+    enabled: true
 ```
 
-Calico manifest file to apply to the Kubernetes cluster (if using Calico instead of Flannel).
+Cilium Helm chart values can be specified under `kubernetes_cilium_values`.
+
+##### Kube Proxy Replacement
+
+[Kube Proxy Replacement](https://docs.cilium.io/en/latest/network/kubernetes/kubeproxy-free/) is supported through this method. The is a prerequisite for Gateway API Support.
+
+##### Gateway API Support
+
+[Gateway API Support](https://docs.cilium.io/en/latest/network/servicemesh/gateway-api/gateway-api/#gs-gateway-api). This will install additional CRDs to support GatewayAPI.
+
+##### Hubble Observability
+
+In addition to setting up [Setting up Hubble Observability](https://docs.cilium.io/en/stable/gettingstarted/hubble_setup/#hubble-setup), you may choose to install the hubble client with `kubernetes_cilium_hubble_client: true`
 
 ## Dependencies
 

defaults/main.yml

@@ -13,6 +13,7 @@ kubernetes_version: '1.25'
 kubernetes_version_rhel_package: '1.25.1'
 kubernetes_cilium_version: '1.14.5'
 kubernetes_cilium_datapath: 'native'
+kubernetes_cilium_hubble_client: false
 kubernetes_cilium_values: ""
 
 kubernetes_role: control_plane

molecule/default/cilium.yml

@@ -0,0 +1,74 @@
+---
+- name: Converge
+  hosts: all
+  become: true
+
+  vars:
+    kubernetes_cilium_hubble_client: true
+    kubernetes_cilium_values:
+      envoy:
+        enabled: true
+      kubeProxyReplacement: true
+      l7Proxy: true
+      loadBalancer:
+        l7:
+          backend: envoy
+      ingressController:
+        enabled: true
+        loadbalancerMode: dedicated
+        default: true
+      hubble:
+        relay:
+          enabled: true
+        ui:
+          enabled: true
+      gatewayAPI:
+        enabled: true
+    kubernetes_pod_network:
+      cni: 'cilium'
+      cidr: 10.244.0.0/16
+
+    # Allow swap in test environments (hard to control in some envs).
+    kubernetes_config_kubelet_configuration:
+      cgroupDriver: "systemd"
+      failSwapOn: false
+      cgroupsPerQOS: true
+      enforceNodeAllocatable: ['pods']
+    containerd_config_cgroup_driver_systemd: true
+
+  pre_tasks:
+    - name: Update apt cache.
+      apt: update_cache=true cache_valid_time=600
+      when: ansible_os_family == 'Debian'
+
+    - name: Ensure test dependencies are installed (RedHat).
+      package: name=iproute state=present
+      when: ansible_os_family == 'RedHat'
+
+    - name: Ensure test dependencies are installed (Debian).
+      package: name=iproute2 state=present
+      when: ansible_os_family == 'Debian'
+
+    - name: Gather facts.
+      action: setup
+
+  roles:
+    - role: geerlingguy.containerd
+    - role: geerlingguy.kubernetes
+
+  post_tasks:
+    - name: Get cluster info.
+      command: kubectl cluster-info
+      changed_when: false
+      register: kubernetes_info
+
+    - name: Print cluster info.
+      debug: var=kubernetes_info.stdout
+
+    - name: Get all running pods.
+      command: kubectl get pods --all-namespaces
+      changed_when: false
+      register: kubernetes_pods
+
+    - name: Print list of running pods.
+      debug: var=kubernetes_pods.stdout

tasks/control-plane-setup.yml

@@ -16,16 +16,22 @@
         (kubernetes_cilium_values.kubeProxyReplacement is defined) and
         (kubernetes_cilium_values.kubeProxyReplacement) | bool }}"
 
+- name: Determine if we installing Gateway API
+  set_fact:
+    install_gateway_api="{{ (replace_kube_proxy) and
+        (kubernetes_cilium_values.gatewayAPI.enabled is defined) and
+        (kubernetes_cilium_values.gatewayAPI.enabled)| bool }}"
+
 - name: Initialize Kubernetes control plane with kubeadm init
   command: >
     kubeadm init
     --config {{ kubernetes_kubeadm_kubelet_config_file_path }}
     {{ kubernetes_kubeadm_init_extra_opts }}
   register: kubeadmin_init
-  when: >
+  when:
-    (not kubernetes_init_stat.stat.exists) and
+    - not kubernetes_init_stat.stat.exists
-    (kubernetes_ignore_preflight_errors is not defined) and
+    - kubernetes_ignore_preflight_errors is not defined
-    (not (replace_kube_proxy))
+    - not replace_kube_proxy
 
 - name: Initialize Kubernetes control plane with kubeadm init and ignore_preflight_errors
   command: >
@@ -34,10 +40,10 @@
     --ignore-preflight-errors={{ kubernetes_ignore_preflight_errors }}
     {{ kubernetes_kubeadm_init_extra_opts }}
   register: kubeadmin_init
-  when: >
+  when:
-    (not kubernetes_init_stat.stat.exists) and
+    - not kubernetes_init_stat.stat.exists
-    (kubernetes_ignore_preflight_errors is defined) and
+    - kubernetes_ignore_preflight_errors is defined
-    (not (replace_kube_proxy))
+    - not replace_kube_proxy
 
 - name: Initialize Kubernetes control plane with kubeadm init without kube-proxy
   command: >
@@ -46,10 +52,10 @@
     --skip-phases=addon/kube-proxy
     {{ kubernetes_kubeadm_init_extra_opts }}
   register: kubeadmin_init
-  when: >
+  when:
-    (not kubernetes_init_stat.stat.exists) and
+    - not kubernetes_init_stat.stat.exists
-    (kubernetes_ignore_preflight_errors is not defined) and
+    - kubernetes_ignore_preflight_errors is not defined
-    (replace_kube_proxy)
+    - replace_kube_proxy
 
 - name: Initialize Kubernetes control plane with kubeadm init without kube-proxy and ignore_preflight_errors
   command: >
@@ -59,10 +65,10 @@
     --skip-phases=addon/kube-proxy
     {{ kubernetes_kubeadm_init_extra_opts }}
   register: kubeadmin_init
-  when: >
+  when:
-    (not kubernetes_init_stat.stat.exists) and
+    - not kubernetes_init_stat.stat.exists
-    (kubernetes_ignore_preflight_errors is defined) and
+    - kubernetes_ignore_preflight_errors is defined
-    (replace_kube_proxy)
+    - replace_kube_proxy
 
 - name: Print the init output to screen.
   debug:
@@ -102,6 +108,19 @@
   retries: 12
   delay: 5
 
+- name: Install Prerequisite CRDs for Cilium Gateway API support.
+  when: install_gateway_api
+  register: gateway_crds
+  changed_when: "'created' in gateway_crds.stdout"
+  command: "kubectl apply -f {{ item }}"
+  loop:
+    - https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/v1.0.0/config/crd/standard/gateway.networking.k8s.io_gatewayclasses.yaml
+    - https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/v1.0.0/config/crd/standard/gateway.networking.k8s.io_gateways.yaml
+    - https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/v1.0.0/config/crd/standard/gateway.networking.k8s.io_httproutes.yaml
+    - https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/v1.0.0/config/crd/standard/gateway.networking.k8s.io_referencegrants.yaml
+    - https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/v1.0.0/config/crd/experimental/gateway.networking.k8s.io_grpcroutes.yaml
+    - https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/v1.0.0/config/crd/experimental/gateway.networking.k8s.io_tlsroutes.yaml
+
 - name: Configure Cilium networking
   command: >
     /usr/local/bin/cilium install

tasks/hubble-client-setup.yml

@@ -0,0 +1,53 @@
+---
+- name: Check if Hubble CLI has already been Installed.
+  stat:
+    path: /usr/local/bin/hubble
+  register: hubble_init_stat
+  when:
+    - kubernetes_pod_network.cni == 'cilium'
+
+- name: Install Hubble CLI
+  when:
+    - kubernetes_pod_network.cni == 'cilium'
+    - not hubble_init_stat.stat.exists
+  block:
+    - name: Get Hubble CLI version
+      shell: curl -s https://raw.githubusercontent.com/cilium/hubble/master/stable.txt
+      register: hubble_cli_version
+      changed_when: false
+
+    - name: Set CLI architecture
+      set_fact:
+        cli_arch: "{{ 'arm64' if ansible_architecture == 'aarch64' else 'amd64' }}"
+
+
+    - name: Download Hubble CLI
+      get_url:
+        url: "https://github.com/cilium/hubble/releases/download/{{ hubble_cli_version.stdout }}/hubble-linux-{{ cli_arch }}.tar.gz"
+        dest: "/tmp/hubble-linux-{{ cli_arch }}.tar.gz"
+        mode: '0644'
+
+    - name: Download Hubble CLI checksum
+      get_url:
+        url: "https://github.com/cilium/hubble/releases/download/{{ hubble_cli_version.stdout }}/hubble-linux-{{ cli_arch }}.tar.gz.sha256sum"
+        dest: "/tmp/hubble-linux-{{ cli_arch }}.tar.gz.sha256sum"
+        mode: '0644'
+
+    - name: Verify Hubble CLI checksum
+      shell: sha256sum --check /tmp/hubble-linux-{{ cli_arch }}.tar.gz.sha256sum
+      args:
+        chdir: /tmp
+
+    - name: Extract Hubble CLI
+      unarchive:
+        src: "/tmp/hubble-linux-{{ cli_arch }}.tar.gz"
+        dest: /usr/local/bin
+        remote_src: true
+
+    - name: Remove downloaded files
+      file:
+        path: "/tmp/hubble-linux-{{ cli_arch }}.tar.gz{{ item }}"
+        state: absent
+      loop:
+        - ''
+        - '.sha256sum'

tasks/main.yml

@@ -37,8 +37,17 @@
 # Set up Cilium Client.
 - include_tasks: cilium-client-setup.yml
   when:
-    - kubernetes_pod_network.cni == 'cilium'
     - kubernetes_role == 'control_plane'
+    - kubernetes_pod_network.cni == 'cilium'
+
+# Set up hubble Client.
+- include_tasks: hubble-client-setup.yml
+  when:
+    - kubernetes_role == 'control_plane'
+    - kubernetes_pod_network.cni == 'cilium'
+    - kubernetes_cilium_hubble_client
+    - kubernetes_cilium_values.hubble.relay.enabled is defined
+    - kubernetes_cilium_values.hubble.relay.enabled
 
 # Set up control plane.
 - include_tasks: control-plane-setup.yml