---
# Copyright kubeinit contributors
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

- name: Deploy the cluster nodes
  ansible.builtin.include_role:
    name: kubeinit.kubeinit.kubeinit_libvirt
    tasks_from: "deploy_{{ hostvars[kubeinit_deployment_node_name].os }}_guest.yml"
    public: yes
  loop: "{{ groups['all_cluster_nodes'] }}"
  loop_control:
    loop_var: kubeinit_deployment_node_name
  vars:
    kubeinit_deployment_delegate: "{{ hostvars[kubeinit_deployment_node_name].target }}"
  when: kubeinit_cluster_nodes_deployed is not defined or not kubeinit_cluster_nodes_deployed

- name: Setup the first controller node
  block:

    - name: Clean kubeadm and initialize Kubernetes cluster
      ansible.builtin.shell: |
        set -eo pipefail
        kubeadm reset -f || true
        kubeadm init \
            --control-plane-endpoint "api.{{ kubeinit_cluster_fqdn }}:6443" \
            --upload-certs \
            --pod-network-cidr={{ kubeinit_k8s_pod_network_cidr }}
      args:
        executable: /bin/bash
      register: _result_kubeadm_init_output
      changed_when: "_result_kubeadm_init_output.rc == 0"

    - name: Debug
      ansible.builtin.debug:
        var: _result_kubeadm_init_output

    - name: Get the controller join key
      ansible.builtin.set_fact:
        k8s_controller_join_key: "{{ item }}"
      loop: "{{ _result_kubeadm_init_output.stdout_lines }}"
      when: ('--control-plane --certificate-key' in item)

    - name: Debug
      ansible.builtin.debug:
        var: k8s_controller_join_key

    - name: Storing the controller join key to add the other controller nodes.
      ansible.builtin.copy:
        content: "{{ k8s_controller_join_key }}"
        dest: ~/k8s_controller_join_key
        mode: '0644'
      when: kubeinit_controller_count|int > 1

    - name: Create kube directory
      ansible.builtin.file:
        path: ~/.kube
        state: directory
        mode: '0644'

    - name: Copying required files
      ansible.builtin.shell: |
        cp -f /etc/kubernetes/admin.conf ~/.kube/config
        chown $(id -u):$(id -g) ~/.kube/config
      args:
        executable: /bin/bash
      register: _result
      changed_when: "_result.rc == 0"

    - name: Get the join command
      ansible.builtin.command: kubeadm token create --print-join-command
      register: _result_join_command
      changed_when: "_result_join_command.rc == 0"

    - name: Storing the join cluster command token.
      ansible.builtin.copy:
        content: "{{ _result_join_command.stdout }}"
        dest: ~/k8s_token
        mode: '0644'
      when: kubeinit_controller_count|int > 1

    - name: Render the flannel template
      ansible.builtin.template:
        src: "kube-flannel.yml.j2"
        dest: "~/kube-flannel.yml"
        mode: "0644"

    - name: Install the Flannel network Add-on
      ansible.builtin.command: kubectl apply -f ~/kube-flannel.yml
      register: _result
      changed_when: "_result.rc == 0"

  delegate_to: "{{ kubeinit_first_controller_node }}"

##
## Execute to configure all the other controller nodes
##

- name: Join additional controller nodes using the PK
  ansible.builtin.shell: |
    kubeadm reset -f || true
    echo "{{ _result_join_command.stdout }} {{ k8s_controller_join_key }}" > ~/k8s_controller_join_command.sh
    sh ~/k8s_controller_join_command.sh
  args:
    executable: /bin/bash
  register: _result
  changed_when: "_result.rc == 0"
  loop: "{{ groups['all_controller_nodes'] }}"
  loop_control:
    loop_var: controller_node
  delegate_to: "{{ controller_node }}"
  when: kubeinit_controller_count|int > 1 and controller_node not in kubeinit_first_controller_node

- name: Create kube directory
  ansible.builtin.file:
    path: ~/.kube
    state: directory
    mode: '0777'
  loop: "{{ groups['all_controller_nodes'] }}"
  loop_control:
    loop_var: controller_node
  delegate_to: "{{ controller_node }}"
  when: kubeinit_controller_count|int > 1 and controller_node not in kubeinit_first_controller_node

- name: Copying required files
  ansible.builtin.shell: |
    cp -f /etc/kubernetes/admin.conf ~/.kube/config
    chown $(id -u):$(id -g) ~/.kube/config
  args:
    executable: /bin/bash
  register: _result
  changed_when: "_result.rc == 0"
  loop: "{{ groups['all_controller_nodes'] }}"
  loop_control:
    loop_var: controller_node
  delegate_to: "{{ controller_node }}"
  when: kubeinit_controller_count|int > 1 and controller_node not in kubeinit_first_controller_node

- name: Join compute nodes with kubernetes control plane
  ansible.builtin.shell: |
    kubeadm reset -f || true
    echo "{{ _result_join_command.stdout }}" > ~/k8s_compute_join_command.sh
    sh ~/k8s_compute_join_command.sh
  args:
    executable: /bin/bash
  register: _result
  changed_when: "_result.rc == 0"
  loop: "{{ groups['all_compute_nodes'] | default([]) }}"
  loop_control:
    loop_var: compute_node
  delegate_to: "{{ compute_node }}"
  when: hostvars[compute_node].os != 'windows'

# The kubeconfig file is on the controller nodes so we run kubectl label on the first controller
- name: Label node
  ansible.builtin.command: |
    kubectl label node {{ hostvars[compute_node].fqdn }} node-role.kubernetes.io/worker=
  register: _result
  changed_when: "_result.rc == 0"
  loop: "{{ groups['all_compute_nodes'] | default([]) }}"
  loop_control:
    loop_var: compute_node
  delegate_to: "{{ kubeinit_first_controller_node }}"
  when: hostvars[compute_node].os != 'windows'

- name: Check if there is at least one compute Windows node
  ansible.builtin.set_fact:
    windows_compute_node_exists: true
  loop: "{{ groups['all_compute_nodes'] | default([]) }}"
  loop_control:
    loop_var: compute_node
  when: hostvars[compute_node].os == 'windows'

- name: Allow schedule workloads in controller nodes if there are no compute nodes
  ansible.builtin.shell: |
    set -o pipefail
    dnf install -y jq
    # Deprecated in 1.24
    for node in $(kubectl get nodes -o json | jq -r '.items[] | select(.spec.taints[]?.key=="node-role.kubernetes.io/master") | .metadata.labels."kubernetes.io/hostname"');
    do
      kubectl taint node ${node} node-role.kubernetes.io/master:NoSchedule-
    done
    # Working starting on 1.24
    for node in $(kubectl get nodes -o json | jq -r '.items[] | select(.spec.taints[]?.key=="node-role.kubernetes.io/control-plane") | .metadata.labels."kubernetes.io/hostname"');
    do
      kubectl taint node ${node} node-role.kubernetes.io/control-plane:NoSchedule-
    done
  args:
    executable: /bin/bash
  register: _result
  changed_when: "_result.rc == 0"
  when: not kubeinit_compute_count|int > 0 or (windows_compute_node_exists | default(false))
  delegate_to: "{{ kubeinit_first_controller_node }}"

# We fetch the kubeconfig from the first controller node
- name: Copying the kubeconfig to a variable
  ansible.builtin.slurp:
    src: ~/.kube/config
  register: _result_cluster_kubeconfig
  delegate_to: "{{ kubeinit_first_controller_node }}"

- name: Create kube directory
  ansible.builtin.file:
    path: ~/.kube
    state: directory
    mode: '0644'
  delegate_to: "{{ kubeinit_provision_service_node }}"

- name: Storing the master kubeconfig to the services machine.
  ansible.builtin.copy:
    content: "{{ _result_cluster_kubeconfig.content | default('Empty file') | b64decode }}"
    dest: ~/.kube/config
    mode: '0644'
  delegate_to: "{{ kubeinit_provision_service_node }}"

- name: Install kustomize
  ansible.builtin.shell: |
    curl -sL https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2Fv5.1.0/kustomize_v5.1.0_linux_amd64.tar.gz > kustomize.tar.gz
    tar xzf ./kustomize.tar.gz
    mv ./kustomize /bin/
  args:
    executable: /bin/bash
  register: _result
  changed_when: "_result.rc == 0"
  delegate_to: "{{ kubeinit_provision_service_node }}"

#
# Configure additional steps for including the Windows compute nodes
#

- name: Create the kubeinit folder
  ansible.windows.win_file:
    path: C:\k
    state: directory
  # TODO:FIXME: The following variables should be
  # added as group vars for those compute nodes which the
  # os is equals to 'windows'
  # TODO:FIXME: The usage of ansible_shell_type might
  # be different depending on the win_* task, the
  # supported values are [cmd | powershell]
  vars:
    ansible_shell_type: 'cmd'
    ansible_remote_tmp: 'C:\Windows\Temp'
  delegate_to: "{{ compute_node }}"
  loop: "{{ groups['all_compute_nodes'] | default([]) }}"
  loop_control:
    loop_var: compute_node
  when: hostvars[compute_node].os == 'windows'

- name: Copy the kubeconfig file
  ansible.windows.win_copy:
    content: "{{ _result_cluster_kubeconfig.content | default('Empty file') | b64decode }}"
    dest: C:\k\kube_config
  # TODO:FIXME: The following variables should be
  # added as group vars for those compute nodes which the
  # os is equals to 'windows'
  # TODO:FIXME: The usage of ansible_shell_type might
  # be different depending on the win_* task, the
  # supported values are [cmd | powershell]
  vars:
    ansible_shell_type: 'cmd'
    ansible_remote_tmp: 'C:\Windows\Temp'
  delegate_to: "{{ compute_node }}"
  loop: "{{ groups['all_compute_nodes'] | default([]) }}"
  loop_control:
    loop_var: compute_node
  when: hostvars[compute_node].os == 'windows'

- name: Copy the kubeconfig file
  ansible.windows.win_copy:
    content: "{{ _result_cluster_kubeconfig.content | default('Empty file') | b64decode }}"
    dest: C:\k\Kubeconfig
  # TODO:FIXME: The following variables should be
  # added as group vars for those compute nodes which the
  # os is equals to 'windows'
  # TODO:FIXME: The usage of ansible_shell_type might
  # be different depending on the win_* task, the
  # supported values are [cmd | powershell]
  vars:
    ansible_shell_type: 'cmd'
    ansible_remote_tmp: 'C:\Windows\Temp'
  delegate_to: "{{ compute_node }}"
  loop: "{{ groups['all_compute_nodes'] | default([]) }}"
  loop_control:
    loop_var: compute_node
  when: hostvars[compute_node].os == 'windows'

- name: Copy the kubeconfig file
  ansible.windows.win_copy:
    content: "{{ _result_cluster_kubeconfig.content | default('Empty file') | b64decode }}"
    dest: C:\k\config
  # TODO:FIXME: The following variables should be
  # added as group vars for those compute nodes which the
  # os is equals to 'windows'
  # TODO:FIXME: The usage of ansible_shell_type might
  # be different depending on the win_* task, the
  # supported values are [cmd | powershell]
  vars:
    ansible_shell_type: 'cmd'
    ansible_remote_tmp: 'C:\Windows\Temp'
  delegate_to: "{{ compute_node }}"
  loop: "{{ groups['all_compute_nodes'] | default([]) }}"
  loop_control:
    loop_var: compute_node
  when: hostvars[compute_node].os == 'windows'

- name: Write the join command
  ansible.windows.win_copy:
    content: "{{ _result_join_command.stdout | replace('kubeadm','c:\\k\\kubeadm.exe') }} --node-name {{ compute_node }}.{{ kubeinit_cluster_fqdn }} --cri-socket 'npipe:////./pipe/containerd-containerd' -v=10"
    dest: C:\k\k8s_join_command.ps1
  # TODO:FIXME: The following variables should be
  # added as group vars for those compute nodes which the
  # os is equals to 'windows'
  # TODO:FIXME: The usage of ansible_shell_type might
  # be different depending on the win_* task, the
  # supported values are [cmd | powershell]
  vars:
    ansible_shell_type: 'cmd'
    ansible_remote_tmp: 'C:\Windows\Temp'
  delegate_to: "{{ compute_node }}"
  loop: "{{ groups['all_compute_nodes'] | default([]) }}"
  loop_control:
    loop_var: compute_node
  when: hostvars[compute_node].os == 'windows'

- name: Install prereqs
  ansible.windows.win_powershell:
    script: |
      $env:Path += ";C:\k;C:\Program Files\containerd;C:\Program Files\nssm"
      powershell c:\k\PrepareRequirements.ps1
  # TODO:FIXME: The following variables should be
  # added as group vars for those compute nodes which the
  # os is equals to 'windows'
  # TODO:FIXME: The usage of ansible_shell_type might
  # be different depending on the win_* task, the
  # supported values are [cmd | powershell]
  vars:
    ansible_shell_type: 'cmd'
    ansible_remote_tmp: 'C:\Windows\Temp'
  delegate_to: "{{ compute_node }}"
  loop: "{{ groups['all_compute_nodes'] | default([]) }}"
  loop_control:
    loop_var: compute_node
  when: hostvars[compute_node].os == 'windows'

- name: Install containerd
  ansible.windows.win_powershell:
    script: |
      $env:Path += ";C:\k;C:\Program Files\containerd;C:\Program Files\nssm"
      powershell C:\k\Install-Containerd.ps1 -netAdapterName 'Ethernet' -ContainerDVersion '1.6.6'
  # TODO:FIXME: The following variables should be
  # added as group vars for those compute nodes which the
  # os is equals to 'windows'
  # TODO:FIXME: The usage of ansible_shell_type might
  # be different depending on the win_* task, the
  # supported values are [cmd | powershell]
  vars:
    ansible_shell_type: 'cmd'
    ansible_remote_tmp: 'C:\Windows\Temp'
  delegate_to: "{{ compute_node }}"
  loop: "{{ groups['all_compute_nodes'] | default([]) }}"
  loop_control:
    loop_var: compute_node
  when: hostvars[compute_node].os == 'windows'

- name: Prepare the Windows computes (register containerd and kubelet)
  ansible.windows.win_powershell:
    script: |
      $env:Path += ";C:\k;C:\Program Files\containerd;C:\Program Files\nssm"
      # The prepare node script will start kubelet as an nssm service
      # Make sure is like the following command, otherwise it will fail
      # for example: C:\k\kubelet.exe --container-runtime-endpoint=npipe:////./pipe/containerd-containerd --cert-dir=$env:SYSTEMDRIVE\var\lib\kubelet\pki --config=/var/lib/kubelet/config.yaml --kubeconfig=/k/config --hostname-override=compute-01.k8scluster.kubeinit.local --pod-infra-container-image=`"mcr.microsoft.com/oss/kubernetes/pause:3.6`" --enable-debugging-handlers --cgroups-per-qos=false --enforce-node-allocatable=`"`" --resolv-conf=`"`" --log-dir=/var/log/kubelet --logtostderr=true
      powershell C:\k\PrepareNode.ps1 -KubernetesVersion v1.24.2 -ContainerRuntime containerD
  # TODO:FIXME: The following variables should be
  # added as group vars for those compute nodes which the
  # os is equals to 'windows'
  # TODO:FIXME: The usage of ansible_shell_type might
  # be different depending on the win_* task, the
  # supported values are [cmd | powershell]
  vars:
    ansible_shell_type: 'cmd'
    ansible_remote_tmp: 'C:\Windows\Temp'
  delegate_to: "{{ compute_node }}"
  loop: "{{ groups['all_compute_nodes'] | default([]) }}"
  loop_control:
    loop_var: compute_node
  when: hostvars[compute_node].os == 'windows'

- name: Install prereqs and start script (register flannel)
  ansible.windows.win_powershell:
    script: |
      $env:Path += ";C:\k;C:\Program Files\containerd;C:\Program Files\nssm"
      # Default domain: controller-01.k8scluster.kubeinit.local
      # The management IP is the node's IP not the controller's IP.
      # for example: powershell C:\k\PrepareFlannel.ps1 -ManagementIP 10.0.0.2 -Hostname compute-01.k8scluster.kubeinit.local -NetworkMode overlay
      powershell C:\k\PrepareFlannel.ps1 -ManagementIP {{ hostvars[compute_node].ansible_host }}  -Hostname {{ compute_node }}.{{ kubeinit_cluster_fqdn }} -NetworkMode overlay
  # TODO:FIXME: The following variables should be
  # added as group vars for those compute nodes which the
  # os is equals to 'windows'
  # TODO:FIXME: The usage of ansible_shell_type might
  # be different depending on the win_* task, the
  # supported values are [cmd | powershell]
  vars:
    ansible_shell_type: 'cmd'
    ansible_remote_tmp: 'C:\Windows\Temp'
  delegate_to: "{{ compute_node }}"
  loop: "{{ groups['all_compute_nodes'] | default([]) }}"
  loop_control:
    loop_var: compute_node
  when: hostvars[compute_node].os == 'windows'

- name: Join the Windows computes in the cluster
  ansible.windows.win_powershell:
    script: |
      $env:Path += ";C:\k;C:\Program Files\containerd;C:\Program Files\nssm"
      # The kubelet service automatically joins the node to the cluster as kubelet is running as an nssm service
      # powershell C:\k\k8s_join_command.ps1
  # TODO:FIXME: The following variables should be
  # added as group vars for those compute nodes which the
  # os is equals to 'windows'
  # TODO:FIXME: The usage of ansible_shell_type might
  # be different depending on the win_* task, the
  # supported values are [cmd | powershell]
  vars:
    ansible_shell_type: 'cmd'
    ansible_remote_tmp: 'C:\Windows\Temp'
  delegate_to: "{{ compute_node }}"
  loop: "{{ groups['all_compute_nodes'] | default([]) }}"
  loop_control:
    loop_var: compute_node
  when: hostvars[compute_node].os == 'windows'

- name: Tag Windows compute nodes as workers
  ansible.builtin.shell: |
    set -o pipefail
    kubectl label node {{ compute_node }}.{{ kubeinit_cluster_fqdn }} node-role.kubernetes.io/worker=worker
  args:
    executable: /bin/bash
  register: _result
  changed_when: "_result.rc == 0"
  loop: "{{ groups['all_compute_nodes'] | default([]) }}"
  loop_control:
    loop_var: compute_node
  when: hostvars[compute_node].os == 'windows'
  delegate_to: "{{ kubeinit_first_controller_node }}"