From 6798a05d0b985759d2ec3d56ecba82f920599094 Mon Sep 17 00:00:00 2001 From: Jeff Geerling Date: Thu, 27 Sep 2018 22:17:50 -0500 Subject: [PATCH 1/4] Switch tests to use Molecule. --- .gitignore | 3 +- .travis.yml | 34 +++++++++++-------- README.md | 8 ++--- defaults/main.yml | 8 ++--- meta/main.yml | 20 +++++------ molecule/default/molecule.yml | 27 +++++++++++++++ .../test.yml => molecule/default/playbook.yml | 15 ++++---- {tests => molecule/default}/requirements.yml | 0 molecule/default/tests/test_default.py | 14 ++++++++ molecule/default/yaml-lint.yml | 6 ++++ tasks/kubelet-setup.yml | 4 +-- tasks/main.yml | 8 ++--- tasks/master-setup.yml | 12 +++---- tasks/setup-Debian.yml | 9 +++-- tasks/setup-RedHat.yml | 8 ++--- tests/README.md | 11 ------ 16 files changed, 115 insertions(+), 72 deletions(-) create mode 100644 molecule/default/molecule.yml rename tests/test.yml => molecule/default/playbook.yml (69%) rename {tests => molecule/default}/requirements.yml (100%) create mode 100644 molecule/default/tests/test_default.py create mode 100644 molecule/default/yaml-lint.yml delete mode 100644 tests/README.md diff --git a/.gitignore b/.gitignore index c9b2377..f56f5b5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ *.retry -tests/test.sh +*/__pycache__ +*.pyc diff --git a/.travis.yml b/.travis.yml index 23de055..69295e5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,25 +1,29 @@ --- +language: python services: docker env: - - distro: centos7 - - distro: ubuntu1604 - - distro: debian9 + global: + - ROLE_NAME: kubernetes + matrix: + - MOLECULE_DISTRO: centos7 + MOLECULE_DOCKER_COMMAND: /usr/lib/systemd/systemd + - MOLECULE_DISTRO: ubuntu1604 + - MOLECULE_DISTRO: debian9 + +install: + # Install test dependencies. + - pip install molecule docker + +before_script: + # Use actual Ansible Galaxy role name for the project directory. + - cd ../ + - mv ansible-role-$ROLE_NAME geerlingguy.$ROLE_NAME + - cd geerlingguy.$ROLE_NAME script: - # Configure test script so we can run extra tests after playbook is run. - - export container_id=$(date +%s) - - export cleanup=false - - # Download test shim. - - wget -O ${PWD}/tests/test.sh https://gist.githubusercontent.com/geerlingguy/73ef1e5ee45d8694570f334be385e181/raw/ - - chmod +x ${PWD}/tests/test.sh - # Run tests. - - ${PWD}/tests/test.sh - - # Test whether Kubernetes is running correctly. - # - docker exec --tty ${container_id} command-goes-here + - molecule test notifications: webhooks: https://galaxy.ansible.com/api/v1/notifications/ diff --git a/README.md b/README.md index e93d6b2..3ae0785 100644 --- a/README.md +++ b/README.md @@ -31,15 +31,15 @@ Whether the particular server will serve as a Kubernetes `master` (default) or ` Extra args to pass to `kubelet` during startup. E.g. to allow `kubelet` to start up even if there is swap is enabled on your server, set this to: `"--fail-swap-on=false"`. Or to specify the node-ip advertised by `kubelet`, set this to `"--node-ip={{ ansible_host }}"`. - kubernetes_allow_pods_on_master: True + kubernetes_allow_pods_on_master: true Whether to remove the taint that denies pods from being deployed to the Kubernetes master. If you have a single-node cluster, this should definitely be `True`. Otherwise, set to `False` if you want a dedicated Kubernetes master which doesn't run any other pods. - kubernetes_enable_web_ui: False + kubernetes_enable_web_ui: false Whether to enable the Kubernetes web dashboard UI (only accessible on the master itself, or proxied). - kuberenetes_debug: False + kuberenetes_debug: false Whether to show extra debug info in Ansible's logs (e.g. the output of the `kubeadm init` command). @@ -52,7 +52,7 @@ Options passed to `kubeadm init` when initializing the Kubernetes master. The `a kubernetes_apt_release_channel: main kubernetes_apt_repository: "deb http://apt.kubernetes.io/ kubernetes-xenial {{ kubernetes_apt_release_channel }}" - kubernetes_apt_ignore_key_error: False + kubernetes_apt_ignore_key_error: false Apt repository options for Kubernetes installation. diff --git a/defaults/main.yml b/defaults/main.yml index 7804c12..35e2d4e 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -13,9 +13,9 @@ kubernetes_role: master kubernetes_kubelet_extra_args: "" -kubernetes_allow_pods_on_master: True -kubernetes_enable_web_ui: True -kuberenetes_debug: False +kubernetes_allow_pods_on_master: true +kubernetes_enable_web_ui: true +kuberenetes_debug: false kubernetes_pod_network_cidr: '10.0.1.0/16' kubernetes_apiserver_advertise_address: '' @@ -25,7 +25,7 @@ kubernetes_ignore_preflight_errors: 'all' kubernetes_apt_release_channel: main # Note that xenial repo is used for all Debian derivatives at this time. kubernetes_apt_repository: "deb http://apt.kubernetes.io/ kubernetes-xenial {{ kubernetes_apt_release_channel }}" -kubernetes_apt_ignore_key_error: False +kubernetes_apt_ignore_key_error: false kubernetes_yum_arch: x86_64 diff --git a/meta/main.yml b/meta/main.yml index ea13032..992eac3 100644 --- a/meta/main.yml +++ b/meta/main.yml @@ -8,16 +8,16 @@ galaxy_info: license: "license (BSD, MIT)" min_ansible_version: 2.4 platforms: - - name: EL - versions: - - 7 - - name: Debian - versions: - - stretch - - name: Ubuntu - versions: - - xenial - - bionic + - name: EL + versions: + - 7 + - name: Debian + versions: + - stretch + - name: Ubuntu + versions: + - xenial + - bionic galaxy_tags: - system - containers diff --git a/molecule/default/molecule.yml b/molecule/default/molecule.yml new file mode 100644 index 0000000..0339702 --- /dev/null +++ b/molecule/default/molecule.yml @@ -0,0 +1,27 @@ +--- +dependency: + name: galaxy +driver: + name: docker +lint: + name: yamllint + options: + config-file: molecule/default/yaml-lint.yml +platforms: + - name: instance + image: geerlingguy/docker-${MOLECULE_DISTRO:-centos7}-ansible + command: ${MOLECULE_DOCKER_COMMAND:-"sleep infinity"} + privileged: true + pre_build_image: true +provisioner: + name: ansible + lint: + name: ansible-lint + playbooks: + converge: ${MOLECULE_PLAYBOOK:-playbook.yml} +scenario: + name: default +verifier: + name: testinfra + lint: + name: flake8 diff --git a/tests/test.yml b/molecule/default/playbook.yml similarity index 69% rename from tests/test.yml rename to molecule/default/playbook.yml index 4b51872..c21702e 100644 --- a/tests/test.yml +++ b/molecule/default/playbook.yml @@ -1,20 +1,23 @@ --- -- hosts: all +- name: Converge + hosts: all + become: true vars: # Allow swap in test environments (hard to control in some Docker envs). kubernetes_kubelet_extra_args: "--fail-swap-on=false" - docker_install_compose: False + docker_install_compose: false pre_tasks: - name: Update apt cache. - apt: update_cache=yes cache_valid_time=600 + apt: update_cache=true cache_valid_time=600 when: ansible_os_family == 'Debian' - name: Ensure test dependencies are installed. package: name=iproute state=present - - action: setup + - name: Gather facts. + action: setup - name: Use cgroupfs cgroup driver instead of systemd (Red Hat). set_fact: @@ -22,5 +25,5 @@ when: ansible_os_family == 'RedHat' roles: - - geerlingguy.docker - - role_under_test + - role: geerlingguy.docker + - role: geerlingguy.kubernetes diff --git a/tests/requirements.yml b/molecule/default/requirements.yml similarity index 100% rename from tests/requirements.yml rename to molecule/default/requirements.yml diff --git a/molecule/default/tests/test_default.py b/molecule/default/tests/test_default.py new file mode 100644 index 0000000..eedd64a --- /dev/null +++ b/molecule/default/tests/test_default.py @@ -0,0 +1,14 @@ +import os + +import testinfra.utils.ansible_runner + +testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( + os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') + + +def test_hosts_file(host): + f = host.file('/etc/hosts') + + assert f.exists + assert f.user == 'root' + assert f.group == 'root' diff --git a/molecule/default/yaml-lint.yml b/molecule/default/yaml-lint.yml new file mode 100644 index 0000000..c9aab08 --- /dev/null +++ b/molecule/default/yaml-lint.yml @@ -0,0 +1,6 @@ +--- +extends: default +rules: + line-length: + max: 150 + level: warning diff --git a/tasks/kubelet-setup.yml b/tasks/kubelet-setup.yml index b016bd7..13f161f 100644 --- a/tasks/kubelet-setup.yml +++ b/tasks/kubelet-setup.yml @@ -16,7 +16,7 @@ kubelet_args_path: '/etc/systemd/system/kubelet.service.d/10-kubeadm.conf' kubelet_args_line: "{{ 'Environment=\"KUBELET_EXTRA_ARGS=' + kubernetes_kubelet_extra_args + '\"' }}" kubelet_args_regexp: '^Environment=' - when: kubelet_environment_file.stat.exists == False + when: kubelet_environment_file.stat.exists == false - name: Configure KUBELET_EXTRA_ARGS. lineinfile: @@ -29,6 +29,6 @@ - name: Reload systemd unit if args were changed. systemd: state: restarted - daemon_reload: yes + daemon_reload: true name: kubelet when: kubelet_config_file is changed diff --git a/tasks/main.yml b/tasks/main.yml index 5d709e2..fe6d999 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -24,7 +24,7 @@ service: name: kubelet state: started - enabled: yes + enabled: true - name: Check if Kubernetes has already been initialized. stat: @@ -37,8 +37,8 @@ # Set up nodes. - name: Get the kubeadm join command from the Kubernetes master. - shell: kubeadm token create --print-join-command - changed_when: False + command: kubeadm token create --print-join-command + changed_when: false when: kubernetes_role == 'master' register: kubernetes_join_command_result @@ -47,7 +47,7 @@ kubernetes_join_command: "{{ kubernetes_join_command_result.stdout }}" when: kubernetes_join_command_result.stdout is defined delegate_to: "{{ item }}" - delegate_facts: True + delegate_facts: true with_items: "{{ groups['all'] }}" - include_tasks: node-setup.yml diff --git a/tasks/master-setup.yml b/tasks/master-setup.yml index 416c6c2..7813dbf 100644 --- a/tasks/master-setup.yml +++ b/tasks/master-setup.yml @@ -7,14 +7,14 @@ --kubernetes-version {{ kubernetes_version }} --ignore-preflight-errors={{ kubernetes_ignore_preflight_errors }} register: kubeadmin_init - failed_when: False - when: kubernetes_init_stat.stat.exists == False + failed_when: false + when: kubernetes_init_stat.stat.exists == false - name: Print the init output to screen. debug: var=kubeadmin_init.stdout when: - kuberenetes_debug - - kubernetes_init_stat.stat.exists == False + - kubernetes_init_stat.stat.exists == false - name: Ensure .kube directory exists. file: @@ -41,12 +41,12 @@ command: "kubectl taint nodes --all node-role.kubernetes.io/master-" when: - kubernetes_allow_pods_on_master - - kubernetes_init_stat.stat.exists == False + - kubernetes_init_stat.stat.exists == false - name: Check if Kubernetes Dashboard UI service already exists. shell: kubectl get services --namespace kube-system | grep -q kubernetes-dashboard - changed_when: False - failed_when: False + changed_when: false + failed_when: false register: kubernetes_dashboard_service when: kubernetes_enable_web_ui diff --git a/tasks/setup-Debian.yml b/tasks/setup-Debian.yml index 3604a1e..3b9c72a 100644 --- a/tasks/setup-Debian.yml +++ b/tasks/setup-Debian.yml @@ -1,11 +1,10 @@ --- - name: Ensure dependencies are installed. apt: - name: "{{ item }}" + name: + - apt-transport-https + - ca-certificates state: present - with_items: - - apt-transport-https - - ca-certificates - name: Add Kubernetes apt key. apt_key: @@ -18,4 +17,4 @@ apt_repository: repo: "{{ kubernetes_apt_repository }}" state: present - update_cache: yes + update_cache: false diff --git a/tasks/setup-RedHat.yml b/tasks/setup-RedHat.yml index 09fea09..83335d9 100644 --- a/tasks/setup-RedHat.yml +++ b/tasks/setup-RedHat.yml @@ -3,9 +3,9 @@ yum_repository: name: kubernetes description: Kubernetes - enabled: yes - gpgcheck: yes - repo_gpgcheck: yes + enabled: true + gpgcheck: true + repo_gpgcheck: true baseurl: https://packages.cloud.google.com/yum/repos/kubernetes-el7-{{ kubernetes_yum_arch }} gpgkey: - https://packages.cloud.google.com/yum/doc/yum-key.gpg @@ -24,4 +24,4 @@ command: "yum -q makecache -y --disablerepo='*' --enablerepo='kubernetes'" when: kubernetes_rpm_key is changed args: - warn: no + warn: false diff --git a/tests/README.md b/tests/README.md deleted file mode 100644 index 6fb2117..0000000 --- a/tests/README.md +++ /dev/null @@ -1,11 +0,0 @@ -# Ansible Role tests - -To run the test playbook(s) in this directory: - - 1. Install and start Docker. - 1. Download the test shim (see .travis.yml file for the URL) into `tests/test.sh`: - - `wget -O tests/test.sh https://gist.githubusercontent.com/geerlingguy/73ef1e5ee45d8694570f334be385e181/raw/` - 1. Make the test shim executable: `chmod +x tests/test.sh`. - 1. Run (from the role root directory) `distro=[distro] playbook=[playbook] ./tests/test.sh` - -If you don't want the container to be automatically deleted after the test playbook is run, add the following environment variables: `cleanup=false container_id=$(date +%s)` From 0bb0604b618ada8f061d5ccc84b4f9b9944db7b1 Mon Sep 17 00:00:00 2001 From: Jeff Geerling Date: Thu, 27 Sep 2018 22:46:20 -0500 Subject: [PATCH 2/4] Issue #17: Attempt to fix installation on CentOS. --- README.md | 2 +- defaults/main.yml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 3ae0785..3fe24dd 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ Whether to enable the Kubernetes web dashboard UI (only accessible on the master Whether to show extra debug info in Ansible's logs (e.g. the output of the `kubeadm init` command). - kubernetes_pod_network_cidr: '10.0.1.0/16' + kubernetes_pod_network_cidr: '10.244.0.0/16' kubernetes_apiserver_advertise_address: '' kubernetes_version: 'stable-1.11' kubernetes_ignore_preflight_errors: 'all' diff --git a/defaults/main.yml b/defaults/main.yml index 35e2d4e..1bc3866 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -2,10 +2,10 @@ kubernetes_packages: - name: kubelet state: present - - name: kubeadm - state: present - name: kubectl state: present + - name: kubeadm + state: present - name: kubernetes-cni state: present @@ -17,7 +17,7 @@ kubernetes_allow_pods_on_master: true kubernetes_enable_web_ui: true kuberenetes_debug: false -kubernetes_pod_network_cidr: '10.0.1.0/16' +kubernetes_pod_network_cidr: '10.244.0.0/16' kubernetes_apiserver_advertise_address: '' kubernetes_version: 'stable-1.11' kubernetes_ignore_preflight_errors: 'all' From fad8281b9865dc1e262429da4ebc624f2c272322 Mon Sep 17 00:00:00 2001 From: Jeff Geerling Date: Thu, 27 Sep 2018 23:19:56 -0500 Subject: [PATCH 3/4] Issue #18: Attempt to fix version pinning issues on RedHat and Debian. --- .travis.yml | 2 ++ README.md | 9 ++++++++- defaults/main.yml | 5 ++++- molecule/default/molecule.yml | 3 +++ tasks/master-setup.yml | 2 +- tasks/setup-Debian.yml | 5 +++++ templates/apt-preferences-kubernetes.j2 | 11 +++++++++++ vars/RedHat.yml | 9 +++++++++ 8 files changed, 43 insertions(+), 3 deletions(-) create mode 100644 templates/apt-preferences-kubernetes.j2 diff --git a/.travis.yml b/.travis.yml index 69295e5..91594fe 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,9 @@ env: - MOLECULE_DISTRO: centos7 MOLECULE_DOCKER_COMMAND: /usr/lib/systemd/systemd - MOLECULE_DISTRO: ubuntu1604 + MOLECULE_DOCKER_COMMAND: /lib/systemd/systemd - MOLECULE_DISTRO: debian9 + MOLECULE_DOCKER_COMMAND: /lib/systemd/systemd install: # Install test dependencies. diff --git a/README.md b/README.md index 3fe24dd..358eb13 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,8 @@ Available variables are listed below, along with default values (see `defaults/m kubernetes_packages: - name: kubelet state: present + - name: kubectl + state: present - name: kubeadm state: present - name: kubernetes-cni @@ -22,6 +24,11 @@ Available variables are listed below, along with default values (see `defaults/m Kubernetes packages to be installed on the server. You can either provide a list of package names, or set `name` and `state` to have more control over whether the package is `present`, `absent`, `latest`, etc. + kubernetes_version: '1.11' + kubernetes_version_rhel_package: '1.11.3' + +The minor version of Kubernetes to install. The plain `kubernetes_version` is used to pin an apt package version on Debian, and as the Kubernetes version passed into the `kubeadm init` command (see `kubernetes_version_kubeadm`). The `kubernetes_version_rhel_package` variable must be a specific Kubernetes release, and is used to pin the version on Red Hat / CentOS servers. + kubernetes_role: master Whether the particular server will serve as a Kubernetes `master` (default) or `node`. The master will have `kubeadm init` run on it to intialize the entire K8s control plane, while `node`s will have `kubeadm join` run on them to join them to the `master`. @@ -45,7 +52,7 @@ Whether to show extra debug info in Ansible's logs (e.g. the output of the `kube kubernetes_pod_network_cidr: '10.244.0.0/16' kubernetes_apiserver_advertise_address: '' - kubernetes_version: 'stable-1.11' + kubernetes_version_kubeadm: 'stable-{{ kubernetes_version }}' kubernetes_ignore_preflight_errors: 'all' Options passed to `kubeadm init` when initializing the Kubernetes master. The `apiserver_advertise_address` defaults to `ansible_default_ipv4.address` if it's left empty. diff --git a/defaults/main.yml b/defaults/main.yml index 1bc3866..a96d358 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -9,6 +9,9 @@ kubernetes_packages: - name: kubernetes-cni state: present +kubernetes_version: '1.11' +kubernetes_version_rhel_package: '1.11.3' + kubernetes_role: master kubernetes_kubelet_extra_args: "" @@ -19,7 +22,7 @@ kuberenetes_debug: false kubernetes_pod_network_cidr: '10.244.0.0/16' kubernetes_apiserver_advertise_address: '' -kubernetes_version: 'stable-1.11' +kubernetes_version_kubeadm: 'stable-{{ kubernetes_version }}' kubernetes_ignore_preflight_errors: 'all' kubernetes_apt_release_channel: main diff --git a/molecule/default/molecule.yml b/molecule/default/molecule.yml index 0339702..ef11d53 100644 --- a/molecule/default/molecule.yml +++ b/molecule/default/molecule.yml @@ -11,6 +11,9 @@ platforms: - name: instance image: geerlingguy/docker-${MOLECULE_DISTRO:-centos7}-ansible command: ${MOLECULE_DOCKER_COMMAND:-"sleep infinity"} + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:ro + - /var/lib/docker privileged: true pre_build_image: true provisioner: diff --git a/tasks/master-setup.yml b/tasks/master-setup.yml index 7813dbf..2294ffc 100644 --- a/tasks/master-setup.yml +++ b/tasks/master-setup.yml @@ -4,7 +4,7 @@ kubeadm init --pod-network-cidr={{ kubernetes_pod_network_cidr }} --apiserver-advertise-address={{ kubernetes_apiserver_advertise_address | default(ansible_default_ipv4.address) }} - --kubernetes-version {{ kubernetes_version }} + --kubernetes-version {{ kubernetes_version_kubeadm }} --ignore-preflight-errors={{ kubernetes_ignore_preflight_errors }} register: kubeadmin_init failed_when: false diff --git a/tasks/setup-Debian.yml b/tasks/setup-Debian.yml index 3b9c72a..1aa5c20 100644 --- a/tasks/setup-Debian.yml +++ b/tasks/setup-Debian.yml @@ -18,3 +18,8 @@ repo: "{{ kubernetes_apt_repository }}" state: present update_cache: false + +- name: Add Kubernetes apt preferences file to pin a version. + template: + src: apt-preferences-kubernetes.j2 + dest: /etc/apt/preferences.d/kubernetes diff --git a/templates/apt-preferences-kubernetes.j2 b/templates/apt-preferences-kubernetes.j2 new file mode 100644 index 0000000..0c7d447 --- /dev/null +++ b/templates/apt-preferences-kubernetes.j2 @@ -0,0 +1,11 @@ +Package: kubectl +Pin: version {{ kubernetes_version }}.* +Pin-Priority: 1000 + +Package: kubeadm +Pin: version {{ kubernetes_version }}.* +Pin-Priority: 1000 + +Package: kubelet +Pin: version {{ kubernetes_version }}.* +Pin-Priority: 1000 diff --git a/vars/RedHat.yml b/vars/RedHat.yml index 04919de..db8b7ac 100644 --- a/vars/RedHat.yml +++ b/vars/RedHat.yml @@ -1,2 +1,11 @@ --- kubelet_environment_file_path: /etc/sysconfig/kubelet +kubernetes_packages: + - name: kubelet-{{ kubernetes_version_rhel_package }}-0 + state: present + - name: kubectl-{{ kubernetes_version_rhel_package }}-0 + state: present + - name: kubeadm-{{ kubernetes_version_rhel_package }}-0 + state: present + - name: kubernetes-cni + state: present From 0379cc47db361179cc094a1140e65763d3eec3a3 Mon Sep 17 00:00:00 2001 From: Jeff Geerling Date: Thu, 27 Sep 2018 23:30:25 -0500 Subject: [PATCH 4/4] Fix boolean on Debian setup, add more tests. --- molecule/default/playbook.yml | 17 +++++++++++++++++ tasks/setup-Debian.yml | 2 +- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/molecule/default/playbook.yml b/molecule/default/playbook.yml index c21702e..cc3bf1c 100644 --- a/molecule/default/playbook.yml +++ b/molecule/default/playbook.yml @@ -27,3 +27,20 @@ roles: - role: geerlingguy.docker - role: geerlingguy.kubernetes + + post_tasks: + - name: Get cluster info. + command: kubectl cluster-info + changed_when: false + register: kubernetes_info + + - name: Print cluster info. + debug: var=kubernetes_info.stdout + + - name: Get all running pods. + command: kubectl get pods --all-namespaces + changed_when: false + register: kubernetes_pods + + - name: Print list of running pods. + debug: var=kubernetes_pods.stdout diff --git a/tasks/setup-Debian.yml b/tasks/setup-Debian.yml index 1aa5c20..2d07fa5 100644 --- a/tasks/setup-Debian.yml +++ b/tasks/setup-Debian.yml @@ -17,7 +17,7 @@ apt_repository: repo: "{{ kubernetes_apt_repository }}" state: present - update_cache: false + update_cache: true - name: Add Kubernetes apt preferences file to pin a version. template: