From 7772b5cb777b7363e493823d6e5aeff6e56243a5 Mon Sep 17 00:00:00 2001 From: Jonathan Agmon Date: Sun, 19 Apr 2026 00:12:26 +0300 Subject: [PATCH] Initial commit --- ansible/group_vars/vault-example.yml | 5 + ansible/inv-example.yml | 15 ++ ansible/playbooks/k3s-check.yml | 23 +++ .../playbooks/k3s-distribute-kubeconfig.yml | 40 +++++ ansible/playbooks/k3s-server-join-dry-run.yml | 156 ++++++++++++++++ ansible/playbooks/k3s-server-join.yml | 170 ++++++++++++++++++ ansible/playbooks/k3s-uninstall.yml | 55 ++++++ .../playbooks/templates/config-example.yaml | 34 ++++ 8 files changed, 498 insertions(+) create mode 100644 ansible/group_vars/vault-example.yml create mode 100644 ansible/inv-example.yml create mode 100644 ansible/playbooks/k3s-check.yml create mode 100644 ansible/playbooks/k3s-distribute-kubeconfig.yml create mode 100644 ansible/playbooks/k3s-server-join-dry-run.yml create mode 100644 ansible/playbooks/k3s-server-join.yml create mode 100644 ansible/playbooks/k3s-uninstall.yml create mode 100644 ansible/playbooks/templates/config-example.yaml diff --git a/ansible/group_vars/vault-example.yml b/ansible/group_vars/vault-example.yml new file mode 100644 index 0000000..6ec7877 --- /dev/null +++ b/ansible/group_vars/vault-example.yml @@ -0,0 +1,5 @@ +vault_ansible_become_pass: +# get token with: sudo cat /var/lib/rancher/k3s/server/token on the first server node +k3s_token: +k3s_first_server_ip: +k3s_first_server_hostname: \ No newline at end of file diff --git a/ansible/inv-example.yml b/ansible/inv-example.yml new file mode 100644 index 0000000..50a2250 --- /dev/null +++ b/ansible/inv-example.yml @@ -0,0 +1,15 @@ +--- +group_name: + hosts: + node1_name: + ansible_host: {IP_ADDRESS_1} + node2_name: + ansible_host: {IP_ADDRESS_2} + vars: + ansible_user: {USERNAME} + ansible_ssh_private_key_file: ~/.ssh/{SSH_KEY_NAME} + ansible_python_interpreter: /usr/bin/python3 + ansible_become: true + ansible_become_method: sudo + ansible_become_user: root + ansible_become_pass: "{{ vault_ansible_become_pass }}" \ No newline at end of file diff --git a/ansible/playbooks/k3s-check.yml b/ansible/playbooks/k3s-check.yml new file mode 100644 index 0000000..82f7528 --- /dev/null +++ b/ansible/playbooks/k3s-check.yml @@ -0,0 +1,23 @@ +--- +- name: Check k3s installation + hosts: k3s + become: true + vars_files: + - group_vars/k3s.yml + + tasks: + - name: Check if k3s is installed + ansible.builtin.command: k3s --version + register: k3s_version + changed_when: false + failed_when: false + + - name: Display k3s version + ansible.builtin.debug: + msg: "k3s version: {{ k3s_version.stdout }}" + when: k3s_version.rc == 0 + + - name: Display message if k3s is not installed + ansible.builtin.debug: + msg: "k3s is NOT installed on this host" + when: k3s_version.rc != 0 diff --git a/ansible/playbooks/k3s-distribute-kubeconfig.yml b/ansible/playbooks/k3s-distribute-kubeconfig.yml new file mode 100644 index 0000000..8a3a359 --- /dev/null +++ b/ansible/playbooks/k3s-distribute-kubeconfig.yml @@ -0,0 +1,40 @@ +--- +- name: Distribute kubeconfig to k3s server nodes + hosts: k3s + become: true + + tasks: + - name: Create .kube directory + ansible.builtin.file: + path: "~{{ ansible_user }}/.kube" + state: directory + mode: '0755' + owner: "{{ ansible_user }}" + group: "{{ ansible_user }}" + + - name: Copy kubeconfig from local machine to remote host + ansible.builtin.copy: + src: "~/.kube/config" + dest: "~{{ ansible_user }}/.kube/config" + mode: '0600' + owner: "{{ ansible_user }}" + group: "{{ ansible_user }}" + + - name: Set KUBECONFIG in ansible_user profile + ansible.builtin.lineinfile: + path: "~{{ ansible_user }}/.profile" + line: "export KUBECONFIG=~{{ ansible_user }}/.kube/config" + regexp: "^export KUBECONFIG=.*" + create: true + mode: '0644' + owner: "{{ ansible_user }}" + group: "{{ ansible_user }}" + + - name: Verify kubectl works on remote host + ansible.builtin.command: k3s kubectl get nodes + register: nodes_result + changed_when: false + + - name: Display cluster nodes + ansible.builtin.debug: + msg: "{{ nodes_result.stdout_lines }}" diff --git a/ansible/playbooks/k3s-server-join-dry-run.yml b/ansible/playbooks/k3s-server-join-dry-run.yml new file mode 100644 index 0000000..42e8d34 --- /dev/null +++ b/ansible/playbooks/k3s-server-join-dry-run.yml @@ -0,0 +1,156 @@ +--- +- name: Dry run - Join a server node to an existing k3s HA cluster + hosts: k3s + become: true + vars: + k3s_server_url: "https://{{ k3s_first_server_ip }}:6443" + + pre_tasks: + - name: "[DRY RUN] Detect network interface for ansible_host" + ansible.builtin.shell: | + set -o pipefail + ip route get {{ ansible_host }} | grep -oP 'dev \K\w+' | head -1 + register: detected_iface + changed_when: false + failed_when: false + + - name: "[DRY RUN] Set fact for node interface" + ansible.builtin.set_fact: + k3s_node_iface: "{{ detected_iface.stdout | default('') }}" + when: detected_iface.rc == 0 and detected_iface.stdout | length > 0 + + - name: "[DRY RUN] Warn if mac-shim interface detected" + ansible.builtin.debug: + msg: "WARNING: mac-shim-like interface detected on {{ inventory_hostname }} (iface: {{ k3s_node_iface }}). This can cause k3s cluster join issues." + when: + - k3s_node_iface is defined + - "'mac' in k3s_node_iface or 'shim' in k3s_node_iface" + + - name: "[DRY RUN] Check if k3s is installed" + ansible.builtin.command: k3s --version + register: k3s_installed + changed_when: false + failed_when: false + + - name: "[DRY RUN] Display k3s installation status" + ansible.builtin.debug: + msg: "k3s installation: {{ 'FOUND' if k3s_installed.rc == 0 else 'NOT FOUND' }}" + + - name: "[DRY RUN] Check if k3s service is active" + ansible.builtin.systemd: + name: k3s + register: k3s_service_active + changed_when: false + failed_when: false + + - name: "[DRY RUN] Display k3s service status" + ansible.builtin.debug: + msg: "k3s service: {{ 'ACTIVE' if k3s_service_active.status.ActiveState == 'active' else 'NOT ACTIVE' }}" + + - name: "[DRY RUN] Check if node is already in cluster" + ansible.builtin.shell: | + set -o pipefail + k3s kubectl get nodes --no-headers | grep -w "{{ ansible_hostname }}" || true + register: node_in_cluster + changed_when: false + failed_when: false + when: k3s_service_active.status.ActiveState == 'active' + + - name: "[DRY RUN] Display cluster membership status" + ansible.builtin.debug: + msg: "Node in cluster: {{ 'YES' if node_in_cluster.stdout | default('') | length > 0 else 'NO' }}" + when: k3s_service_active.status.ActiveState == 'active' + + - name: "[DRY RUN] Validate required vault variables" + ansible.builtin.assert: + that: + - k3s_token is defined + - k3s_token | length > 0 + fail_msg: "k3s_token is not defined. Create vault/k3s-cluster.yml with 'ansible-vault create vault/k3s-cluster.yml' and set k3s_token." + when: not (k3s_installed.rc == 0 and k3s_service_active.rc == 0 and node_in_cluster.stdout | default('') | length > 0) + + - name: "[DRY RUN] Validate vault variables when already joined" + ansible.builtin.assert: + that: + - k3s_token is defined + - k3s_token | length > 0 + fail_msg: "k3s_token is not defined or empty in vault" + success_msg: "k3s_token is defined and accessible" + when: k3s_installed.rc == 0 and k3s_service_active.status.ActiveState == 'active' and node_in_cluster.stdout | default('') | length > 0 + + tasks: + - name: "[DRY RUN] Check if node would be skipped" + ansible.builtin.debug: + msg: "Node {{ inventory_hostname }} is already installed and joined - would be SKIPPED" + when: + - k3s_installed.rc == 0 + - k3s_service_active.status.ActiveState == 'active' + - node_in_cluster.stdout | default('') | length > 0 + + - name: "[DRY RUN] Display what would be executed" + ansible.builtin.debug: + msg: + - "Would execute:" + - " curl -sfL https://get.k3s.io | K3S_TOKEN='' K3S_CONFIG=/etc/rancher/k3s/config.yaml sh -s - server \\" + - " --server {{ k3s_server_url }} \\" + - " --node-ip {{ ansible_host }} \\" + - " {% if k3s_node_iface is defined and k3s_node_iface | length > 0 %}--flannel-iface {{ k3s_node_iface }}{% endif %}" + when: + - not (k3s_installed.rc == 0 and k3s_service_active.status.ActiveState == 'active' and node_in_cluster.stdout | default('') | length > 0) + + - name: "[DRY RUN] Check connectivity to existing k3s server" + ansible.builtin.wait_for: + host: "{{ k3s_first_server_ip }}" + port: 6443 + timeout: 5 + register: server_connectivity + ignore_errors: true + when: + - not (k3s_installed.rc == 0 and k3s_service_active.status.ActiveState == 'active' and node_in_cluster.stdout | default('') | length > 0) + + - name: "[DRY RUN] Display server connectivity" + ansible.builtin.debug: + msg: "Connectivity to {{ k3s_first_server_ip }}:6443: {{ 'REACHABLE' if server_connectivity is succeeded else 'NOT REACHABLE' }}" + when: + - not (k3s_installed.rc == 0 and k3s_service_active.status.ActiveState == 'active' and node_in_cluster.stdout | default('') | length > 0) + + - name: "[DRY RUN] Display current cluster nodes (if accessible)" + ansible.builtin.command: k3s kubectl get nodes + register: current_nodes + changed_when: false + failed_when: false + when: k3s_service_active.status.ActiveState == 'active' + + - name: "[DRY RUN] Show cluster nodes" + ansible.builtin.debug: + msg: "{{ current_nodes.stdout_lines | default(['Unable to retrieve cluster nodes']) }}" + when: k3s_service_active.status.ActiveState == 'active' + + - name: "[DRY RUN] Check encryption status" + ansible.builtin.command: k3s secrets-encrypt status + register: encryption_status + changed_when: false + failed_when: false + when: k3s_service_active.status.ActiveState == 'active' + + - name: "[DRY RUN] Display encryption status" + ansible.builtin.debug: + msg: "{{ encryption_status.stdout_lines }}" + when: + - k3s_service_active.status.ActiveState == 'active' + - encryption_status is defined + - encryption_status.stdout_lines is defined + + - name: "[DRY RUN] Summary" + ansible.builtin.debug: + msg: + - "=== DRY RUN SUMMARY ===" + - "Host: {{ inventory_hostname }}" + - "Network interface: {{ k3s_node_iface | default('auto') }}" + - "k3s installed: {{ 'YES' if k3s_installed.rc == 0 else 'NO' }}" + - "k3s active: {{ 'YES' if k3s_service_active.status.ActiveState == 'active' else 'NO' }}" + - "In cluster: {{ 'YES' if node_in_cluster.stdout | default('') | length > 0 else 'NO' }}" + - |- + Action: {%- if k3s_installed.rc == 0 and k3s_service_active.status.ActiveState == 'active' + and node_in_cluster.stdout | default('') | length > 0 %} SKIP (already joined) + {%- else %} JOIN CLUSTER {%- endif %} diff --git a/ansible/playbooks/k3s-server-join.yml b/ansible/playbooks/k3s-server-join.yml new file mode 100644 index 0000000..2cf0af5 --- /dev/null +++ b/ansible/playbooks/k3s-server-join.yml @@ -0,0 +1,170 @@ +--- +- name: Join a server node to an existing k3s HA cluster + hosts: k3s + become: true + serial: 1 + vars: + k3s_server_url: "https://{{ k3s_first_server_ip }}:6443" + + pre_tasks: + - name: Detect network interface for ansible_host + ansible.builtin.shell: | + set -o pipefail + ip route get {{ ansible_host }} | grep -oP 'dev \K\w+' | head -1 + register: detected_iface + changed_when: false + failed_when: false + + - name: Set fact for node interface + ansible.builtin.set_fact: + k3s_node_iface: "{{ detected_iface.stdout | default('') }}" + when: detected_iface.rc == 0 and detected_iface.stdout | length > 0 + + - name: Warn if mac-shim interface detected + ansible.builtin.debug: + msg: "WARNING: mac-shim-like interface detected on {{ inventory_hostname }} (iface: {{ k3s_node_iface }}). This can cause k3s cluster join issues." + when: + - k3s_node_iface is defined + - "'mac' in k3s_node_iface or 'shim' in k3s_node_iface" + + - name: Check if k3s is installed + ansible.builtin.command: k3s --version + register: k3s_installed + changed_when: false + failed_when: false + + - name: Check if k3s service is active + ansible.builtin.systemd: + name: k3s + register: k3s_service_active + changed_when: false + failed_when: false + + - name: Check if node is already in the cluster + ansible.builtin.shell: | + set -o pipefail + k3s kubectl get nodes --no-headers | grep -w "{{ ansible_hostname }}" || true + register: node_in_cluster + changed_when: false + failed_when: false + when: k3s_service_active.status.ActiveState | default('') == 'active' + + - name: Set fact if already installed and joined + ansible.builtin.set_fact: + k3s_already_joined: true + when: + - k3s_installed.rc == 0 + - k3s_service_active.status.ActiveState | default('') == 'active' + - node_in_cluster.stdout | default('') | length > 0 + + - name: Display status if already joined + ansible.builtin.debug: + msg: "k3s is already installed and joined to the cluster on {{ inventory_hostname }}" + when: k3s_already_joined | default(false) | bool + + - name: Validate required vault variables + ansible.builtin.assert: + that: + - k3s_token is defined + - k3s_token | length > 0 + fail_msg: "k3s_token is not defined. Create vault/k3s-cluster.yml with 'ansible-vault create vault/k3s-cluster.yml' and set k3s_token." + success_msg: "k3s_token is defined and accessible" + when: not (k3s_already_joined | default(false) | bool) + + tasks: + - name: Create k3s config directory + ansible.builtin.file: + path: /etc/rancher/k3s + state: directory + mode: '0755' + when: not (k3s_already_joined | default(false) | bool) + + - name: Deploy k3s server configuration + ansible.builtin.template: + src: config.yaml + dest: /etc/rancher/k3s/config.yaml + mode: '0644' + when: not (k3s_already_joined | default(false) | bool) + + - name: Download k3s install script + ansible.builtin.get_url: + url: https://get.k3s.io + dest: /tmp/k3s-install.sh + mode: '0755' + force: true + when: not (k3s_already_joined | default(false) | bool) + + - name: Install and join server to the k3s cluster + ansible.builtin.command: + cmd: /tmp/k3s-install.sh server --server {{ k3s_server_url }} + environment: + K3S_TOKEN: "{{ k3s_token }}" + register: k3s_join_result + changed_when: "'already installed' not in k3s_join_result.stdout" + when: not (k3s_already_joined | default(false) | bool) + + - name: Wait for k3s service to become active + ansible.builtin.systemd: + name: k3s + state: started + enabled: true + register: k3s_service + when: not (k3s_already_joined | default(false) | bool) + + - name: Verify node has joined the cluster + ansible.builtin.command: k3s kubectl get nodes + register: nodes_result + changed_when: false + retries: 10 + delay: 10 + until: nodes_result.rc == 0 and ansible_hostname in nodes_result.stdout + when: not (k3s_already_joined | default(false) | bool) + + - name: Display cluster nodes + ansible.builtin.debug: + msg: "{{ nodes_result.stdout_lines }}" + when: not (k3s_already_joined | default(false) | bool) + + post_tasks: + - name: Create .kube directory in ansible_user home + ansible.builtin.file: + path: "~{{ ansible_user }}/.kube" + state: directory + mode: '0755' + owner: "{{ ansible_user }}" + group: "{{ ansible_user }}" + + - name: Copy kubeconfig to ansible_user home + ansible.builtin.copy: + src: /etc/rancher/k3s/k3s.yaml + dest: "~{{ ansible_user }}/.kube/config" + mode: '0644' + owner: "{{ ansible_user }}" + group: "{{ ansible_user }}" + remote_src: true + when: not (k3s_already_joined | default(false) | bool) + + - name: Set KUBECONFIG in ansible_user profile + ansible.builtin.lineinfile: + path: "~{{ ansible_user }}/.profile" + line: "export KUBECONFIG=~{{ ansible_user }}/.kube/config" + regexp: "^export KUBECONFIG=.*" + create: true + mode: '0644' + owner: "{{ ansible_user }}" + group: "{{ ansible_user }}" + + - name: Verify secrets encryption is enabled + ansible.builtin.command: k3s secrets-encrypt status + register: encryption_status + changed_when: false + failed_when: false + when: not (k3s_already_joined | default(false) | bool) + + - name: Display encryption status + ansible.builtin.debug: + msg: "{{ encryption_status.stdout_lines }}" + when: + - not (k3s_already_joined | default(false) | bool) + - encryption_status is defined + - encryption_status.stdout_lines is defined diff --git a/ansible/playbooks/k3s-uninstall.yml b/ansible/playbooks/k3s-uninstall.yml new file mode 100644 index 0000000..f8a41d1 --- /dev/null +++ b/ansible/playbooks/k3s-uninstall.yml @@ -0,0 +1,55 @@ +--- +- name: Uninstall k3s from cluster nodes + hosts: k3s + become: true + gather_facts: true + + pre_tasks: + - name: Check if k3s is installed + ansible.builtin.stat: + path: /usr/local/bin/k3s-uninstall.sh + register: uninstall_script + + - name: Display uninstall status + ansible.builtin.debug: + msg: "k3s uninstall script found: {{ 'YES' if uninstall_script.stat.exists else 'NO' }} on {{ inventory_hostname }}" + + tasks: + - name: Run k3s uninstall script + ansible.builtin.command: /usr/local/bin/k3s-uninstall.sh + register: uninstall_result + changed_when: uninstall_result.rc == 0 + when: uninstall_script.stat.exists + + - name: Display uninstall result + ansible.builtin.debug: + msg: "k3s uninstalled successfully on {{ inventory_hostname }}" + when: + - uninstall_script.stat.exists + - uninstall_result.rc == 0 + + - name: Display skip message + ansible.builtin.debug: + msg: "k3s was not installed on {{ inventory_hostname }} - nothing to do" + when: not uninstall_script.stat.exists + + - name: Clean up k3s configuration directory + ansible.builtin.file: + path: /etc/rancher/k3s + state: absent + + - name: Clean up k3s data directory + ansible.builtin.file: + path: /var/lib/rancher/k3s + state: absent + + post_tasks: + - name: Verify k3s is removed + ansible.builtin.command: k3s --version + register: k3s_check + changed_when: false + failed_when: false + + - name: Display verification result + ansible.builtin.debug: + msg: "k3s verification on {{ inventory_hostname }}: {{ 'STILL PRESENT' if k3s_check.rc == 0 else 'REMOVED' }}" diff --git a/ansible/playbooks/templates/config-example.yaml b/ansible/playbooks/templates/config-example.yaml new file mode 100644 index 0000000..100b9fc --- /dev/null +++ b/ansible/playbooks/templates/config-example.yaml @@ -0,0 +1,34 @@ +# K3s server configuration +# Enable secrets encryption at rest +secrets-encryption: true + +# Node configuration - force k3s to use the physical NIC IP +# instead of auto-detecting interfaces like mac-shim +node-ip: "{{ ansible_host }}" +advertise-address: "{{ ansible_host }}" +bind-address: "{{ ansible_host }}" +{% if k3s_node_iface is defined and k3s_node_iface | length > 0 %} +flannel-iface: "{{ k3s_node_iface }}" +{% endif %} + +# TLS configuration - add SANs for API server certificate +# This ensures the certificate is valid for these names/IPs +tls-san: + - "{{ k3s_first_server_ip }}" + - "{{ k3s_first_server_hostname }}" + - kubernetes.default.svc + - kubernetes.default.svc.cluster.local + +# Additional kube-apiserver arguments for TLS hardening +kube-apiserver-arg: + - "tls-min-version=VersionTLS12" + - "tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384" + +# Additional kubelet arguments for TLS +kubelet-arg: + - "tls-min-version=VersionTLS12" + +# Disable unnecessary components (optional - adjust as needed) +# disable: +# - traefik +# - servicelb