From 6bca7febf5956e41d052c27ac49a2552940bf471 Mon Sep 17 00:00:00 2001 From: Oliver Welter Date: Sat, 19 Jul 2025 19:44:23 +0200 Subject: [PATCH 01/10] Use IPv6 IP address in case no IPv4 is available --- defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/defaults/main.yml b/defaults/main.yml index 4023438..d52596c 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -34,7 +34,7 @@ kubernetes_config_kubelet_configuration: kubernetes_config_init_configuration: localAPIEndpoint: - advertiseAddress: "{{ kubernetes_apiserver_advertise_address | default(ansible_default_ipv4.address, true) }}" + advertiseAddress: "{{ kubernetes_apiserver_advertise_address | default(ansible_default_ipv4.address, true) | default(ansible_default_ipv6.address, true) }}" # if you use the next lines, remove the command line argument below # nodeRegistration: # ignorePreflightErrors: From fac7b631df881518ada9ecfccb951fee0645eb3b Mon Sep 17 00:00:00 2001 From: Oliver Welter Date: Sun, 20 Jul 2025 09:14:00 +0200 Subject: [PATCH 02/10] Add specification of service subnet CIDR --- defaults/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/defaults/main.yml b/defaults/main.yml index d52596c..b357a78 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -24,6 +24,7 @@ kubernetes_pod_network: # Flannel CNI. cni: 'flannel' cidr: '10.244.0.0/16' + serviceCidr: '10.96.0.0/12' # Calico CNI. # cni: 'calico' # cidr: '192.168.0.0/16' @@ -43,6 +44,7 @@ kubernetes_config_init_configuration: kubernetes_config_cluster_configuration: networking: podSubnet: "{{ kubernetes_pod_network.cidr }}" + serviceSubnet: "{{ kubernetes_pod_network.serviceCidr }}" kubernetesVersion: "{{ kubernetes_version_kubeadm }}" kubernetes_config_kube_proxy_configuration: {} From 8325de59af8508f7405813d104f78d2d98d493f6 Mon Sep 17 00:00:00 2001 From: Oliver Welter Date: Wed, 23 Jul 2025 12:33:33 +0200 Subject: [PATCH 03/10] Set kernel parameters for forwarding --- tasks/sysctl-setup.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tasks/sysctl-setup.yml b/tasks/sysctl-setup.yml index 174ebca..7e813b6 100644 --- a/tasks/sysctl-setup.yml +++ b/tasks/sysctl-setup.yml @@ -7,6 +7,15 @@ ansible_distribution != 'Debian' or ansible_distribution_major_version | int < 10 +- name: Activate forwarding + sysctl: + name: "{{ item }}" + value: '1' + state: present + loop: + - net.ipv4.ip_forward + - net.ipv6.conf.all.forwarding + # See: https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/#letting-iptables-see-bridged-traffic - name: Let iptables see bridged traffic. sysctl: From 98b3e168487fd2609d6b3c77aee73cd4526d8923 Mon Sep 17 00:00:00 2001 From: Oliver Welter Date: Wed, 23 Jul 2025 12:34:02 +0200 Subject: [PATCH 04/10] Ignore preflight for joining nodes --- tasks/main.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tasks/main.yml b/tasks/main.yml index 3fa409a..3066b2c 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -50,7 +50,18 @@ kubernetes_join_command: > {{ kubernetes_join_command_result.stdout }} {{ kubernetes_join_command_extra_opts }} - when: kubernetes_join_command_result.stdout is defined + when: (kubernetes_join_command_result.stdout is defined) and (kubernetes_ignore_preflight_errors is not defined) + delegate_to: "{{ item }}" + delegate_facts: true + with_items: "{{ groups['all'] }}" + +- name: Set the kubeadm join command with ignore preflight errors globally. + set_fact: + kubernetes_join_command: > + {{ kubernetes_join_command_result.stdout }} + --ignore-preflight-errors={{ kubernetes_ignore_preflight_errors }} + {{ kubernetes_join_command_extra_opts }} + when: (kubernetes_join_command_result.stdout is defined) and (kubernetes_ignore_preflight_errors is defined) delegate_to: "{{ item }}" delegate_facts: true with_items: "{{ groups['all'] }}" From 82e61a909c63a9dbf61d99ab5c7650b63cbccd44 Mon Sep 17 00:00:00 2001 From: Oliver Welter Date: Wed, 23 Jul 2025 13:30:36 +0200 Subject: [PATCH 05/10] Load br_netfilter module --- tasks/sysctl-setup.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tasks/sysctl-setup.yml b/tasks/sysctl-setup.yml index 7e813b6..d548932 100644 --- a/tasks/sysctl-setup.yml +++ b/tasks/sysctl-setup.yml @@ -16,6 +16,12 @@ - net.ipv4.ip_forward - net.ipv6.conf.all.forwarding +- name: Load module br_netfilter + modprobe: + name: br_netfilter + state: present + persistent: present + # See: https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/#letting-iptables-see-bridged-traffic - name: Let iptables see bridged traffic. sysctl: From b80c796d9d183f0a08e589e236d8cd61406bd672 Mon Sep 17 00:00:00 2001 From: Oliver Welter Date: Wed, 23 Jul 2025 14:01:47 +0200 Subject: [PATCH 06/10] Extend documentation with example for IPv6 only --- README.md | 274 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 274 insertions(+) diff --git a/README.md b/README.md index 99c88f5..ecfe625 100644 --- a/README.md +++ b/README.md @@ -213,6 +213,280 @@ Playbook: - geerlingguy.kubernetes ``` +### IPv6 only two or more nodes (single control-plane) cluster using containerd + +Download default config file for flannel from https://raw.githubusercontent.com/flannel-io/flannel/master/Documentation/kube-flannel.yml and store it in files/kube-flannel-config.yaml next to ansible scripts. + +Update net-conf.json in property: + +```yaml +--- +kind: Namespace +apiVersion: v1 +metadata: + name: kube-flannel + labels: + k8s-app: flannel + pod-security.kubernetes.io/enforce: privileged +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + labels: + k8s-app: flannel + name: flannel +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + labels: + k8s-app: flannel + name: flannel +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: flannel +subjects: +- kind: ServiceAccount + name: flannel + namespace: kube-flannel +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + k8s-app: flannel + name: flannel + namespace: kube-flannel +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: kube-flannel-cfg + namespace: kube-flannel + labels: + tier: node + k8s-app: flannel + app: flannel +data: + cni-conf.json: | + { + "name": "cbr0", + "cniVersion": "0.3.1", + "plugins": [ + { + "type": "flannel", + "delegate": { + "hairpinMode": true, + "isDefaultGateway": true + } + }, + { + "type": "portmap", + "capabilities": { + "portMappings": true + } + } + ] + } + net-conf.json: | + { + "EnableIPv4": false, + "EnableIPv6": true, + "IPv6Network": "fd76:cac2:6150::/56", + "IPv6Subnet": "fd76:cac2:6150::1/64", + "EnableNFTables": false, + "Backend": { + "Type": "vxlan" + } + } +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: kube-flannel-ds + namespace: kube-flannel + labels: + tier: node + app: flannel + k8s-app: flannel +spec: + selector: + matchLabels: + app: flannel + template: + metadata: + labels: + tier: node + app: flannel + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/os + operator: In + values: + - linux + hostNetwork: true + priorityClassName: system-node-critical + tolerations: + - operator: Exists + effect: NoSchedule + serviceAccountName: flannel + initContainers: + - name: install-cni-plugin + image: ghcr.io/flannel-io/flannel-cni-plugin:v1.7.1-flannel1 + command: + - cp + args: + - -f + - /flannel + - /opt/cni/bin/flannel + volumeMounts: + - name: cni-plugin + mountPath: /opt/cni/bin + - name: install-cni + image: ghcr.io/flannel-io/flannel:v0.27.0 + command: + - cp + args: + - -f + - /etc/kube-flannel/cni-conf.json + - /etc/cni/net.d/10-flannel.conflist + volumeMounts: + - name: cni + mountPath: /etc/cni/net.d + - name: flannel-cfg + mountPath: /etc/kube-flannel/ + containers: + - name: kube-flannel + image: ghcr.io/flannel-io/flannel:v0.27.0 + command: + - /opt/bin/flanneld + args: + - --ip-masq + - --kube-subnet-mgr + resources: + requests: + cpu: "100m" + memory: "50Mi" + securityContext: + privileged: false + capabilities: + add: ["NET_ADMIN", "NET_RAW"] + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: EVENT_QUEUE_DEPTH + value: "5000" + - name: CONT_WHEN_CACHE_NOT_READY + value: "false" + volumeMounts: + - name: run + mountPath: /run/flannel + - name: flannel-cfg + mountPath: /etc/kube-flannel/ + - name: xtables-lock + mountPath: /run/xtables.lock + volumes: + - name: run + hostPath: + path: /run/flannel + - name: cni-plugin + hostPath: + path: /opt/cni/bin + - name: cni + hostPath: + path: /etc/cni/net.d + - name: flannel-cfg + configMap: + name: kube-flannel-cfg + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate +``` + +Control plane inventory vars: + +```yaml +kubernetes_role: "control_plane" +``` + +Node(s) inventory vars: + +```yaml +kubernetes_role: "node" +``` + +Playbook: + +```yaml + +- hosts: all + + vars: + kubernetes_flannel_manifest_file: '/etc/kubernetes/kube-flannel-config.yaml' + + tasks: + - name: Create the directory for the kubernetes_config_file + ansible.builtin.file: + path: "{{ kubernetes_flannel_manifest_file | dirname }}" + state: directory + + - name: Copy 'kube-flannel-config.yaml' + ansible.builtin.copy: + src: files/kube-flannel-config.yaml + dest: "{{ kubernetes_flannel_manifest_file }}" + owner: root + group: root + mode: u+rw + +- hosts: all + + vars: + containerd_config_cgroup_driver_systemd: true + kubernetes_allow_pods_on_control_plane: true + kubernetes_flannel_manifest_file: '/etc/kubernetes/kube-flannel-config.yaml' + kubernetes_pod_network: + cni: 'flannel' + cidr: 'fd76:cac2:6150::/56' + serviceCidr: 'fd76:cac2:6150:1::/112' + + roles: + - geerlingguy.containerd + - geerlingguy.kubernetes +``` + Then, log into the Kubernetes control plane, and run `kubectl get nodes` as root, and you should see a list of all the servers. ## License From d8dd17c34d74c682fc0e7f2eb6c760304c71adb2 Mon Sep 17 00:00:00 2001 From: Oliver Welter Date: Fri, 1 Aug 2025 10:26:08 +0200 Subject: [PATCH 07/10] Update kubernetes version to 1.33 in documentation --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ecfe625..dd5cdbe 100644 --- a/README.md +++ b/README.md @@ -27,8 +27,8 @@ kubernetes_packages: Kubernetes packages to be installed on the server. You can either provide a list of package names, or set `name` and `state` to have more control over whether the package is `present`, `absent`, `latest`, etc. ```yaml -kubernetes_version: '1.32' -kubernetes_version_rhel_package: '1.32' +kubernetes_version: '1.33' +kubernetes_version_rhel_package: '1.33' ``` The minor version of Kubernetes to install. The plain `kubernetes_version` is used to pin an apt package version on Debian, and as the Kubernetes version passed into the `kubeadm init` command (see `kubernetes_version_kubeadm`). The `kubernetes_version_rhel_package` variable must be a specific Kubernetes release, and is used to pin the version on Red Hat / CentOS servers. From 8cd592b0229efa5cc128c4e0b956735bcc133d07 Mon Sep 17 00:00:00 2001 From: Oliver Welter Date: Fri, 1 Aug 2025 10:32:29 +0200 Subject: [PATCH 08/10] Run CI on all branches --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a3578b9..55b08b2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,7 +4,7 @@ name: CI pull_request: push: branches: - - master + - '*' schedule: - cron: "0 4 * * 3" From 2fb2814de4a42239221abcca0b01bbd6a592a1d6 Mon Sep 17 00:00:00 2001 From: Oliver Welter Date: Fri, 1 Aug 2025 10:59:48 +0200 Subject: [PATCH 09/10] Explicitly set CIDR for service network --- defaults/main.yml | 1 + molecule/default/calico.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/defaults/main.yml b/defaults/main.yml index bb91657..244ba0b 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -28,6 +28,7 @@ kubernetes_pod_network: # Calico CNI. # cni: 'calico' # cidr: '192.168.0.0/16' + # serviceCidr: '10.96.0.0/12' kubernetes_kubeadm_kubelet_config_file_path: '/etc/kubernetes/kubeadm-kubelet-config.yaml' diff --git a/molecule/default/calico.yml b/molecule/default/calico.yml index c20243d..ddf4614 100644 --- a/molecule/default/calico.yml +++ b/molecule/default/calico.yml @@ -7,6 +7,7 @@ kubernetes_pod_network: cni: 'calico' cidr: '192.168.0.0/16' + serviceCidr: '10.96.0.0/12' # Allow swap in test environments (hard to control in some envs). kubernetes_config_kubelet_configuration: From df6f42752b329469a00c8552d82f599936844080 Mon Sep 17 00:00:00 2001 From: Oliver Welter Date: Sat, 23 Aug 2025 16:19:49 +0200 Subject: [PATCH 10/10] Add .ansible directory to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 8840c8f..63f795c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ *.retry */__pycache__ *.pyc +.ansible .cache