0

I am trying to automate the creation of VM cloud image templates on my Proxmox server using Ansible. If I execute the commands in a shell script it always works. If I execute the same commands (with multiple item expansion) in the Ansible shell module, it fails, and I'm not sure why, my guess is timing.

Test.sh:

#!/bin/bash

set -e

qm unlock 1003 || true
qm destroy 1003 || true
qm create 1003 \
    --name test-template \
    --tags test1,test2 \
    --memory 4096 \
    --cores 2 \
    --net0 virtio,bridge=vmbr1 \
    --scsihw virtio-scsi-pci \
    --ciuser pieter \
    --cipassword Password1 \
    --searchdomain home.insanegenius.net \
    --sshkeys /data/install/images/ssh_keys \
    --ipconfig0 ip=dhcp \
    --ostype l26 \
    --agent 1
virt-customize -a /data/install/images/debian-12-genericcloud-amd64.qcow2 --install qemu-guest-agent
qm set 1003 --scsi0 vmdata:0,import-from=/data/install/images/debian-12-genericcloud-amd64.qcow2 --boot order=scsi0 --ide2 vmdata:cloudinit
qm resize 1003 scsi0 8G
qm template 1003

Run test.sh:

pieter@server-1:~/HomeAutomation$ sudo ./test.sh
[sudo] password for pieter: 
[   0.0] Examining the guest ...
[   4.0] Setting a random seed
virt-customize: warning: random seed could not be set for this type of 
guest
[   4.0] Installing packages: qemu-guest-agent
[   6.5] Finishing off
update VM 1003: -boot order=scsi0 -ide2 vmdata:cloudinit -scsi0 vmdata:0,import-from=/data/install/images/debian-12-genericcloud-amd64.qcow2
ide2: successfully created disk 'vmdata:vm-1003-cloudinit,media=cdrom'
transferred 0.0 B of 2.0 GiB (0.00%)
transferred 20.5 MiB of 2.0 GiB (1.00%)
...
transferred 2.0 GiB of 2.0 GiB (100.00%)
scsi0: successfully created disk 'vmdata:vm-1003-disk-0,size=2G'
generating cloud-init ISO
pieter@server-1:~/HomeAutomation$

Ansible playbook:

---
# Create Cloud-Init templates
# https://pve.proxmox.com/wiki/Cloud-Init_Support

# Install libguestfs
# https://docs.ansible.com/ansible/latest/modules/apt_module.html
- name: "Install libguestfs"
  apt:
    name: ["libguestfs-tools"]
    state: latest
    update_cache: yes
    cache_valid_time: 3600

# https://docs.ansible.com/ansible/latest/modules/file_module.html
- name: "Create Cloud Init directory"
  file:
    path: "{{ item }}"
    state: directory
    mode: "ugo+rwx"
    owner: nobody
    group: users
    recurse: true
  with_items:
    - "{{ cloud_init_dir }}"

# Download SSH keys and cloud images
# https://docs.ansible.com/ansible/latest/collections/ansible/builtin/get_url_module.html
- name: "Download cloud init artifacts"
  # Register changes
  register: download
  get_url:
    url: "{{ item.url }}"
    dest: "{{ item.dest }}"
    mode: "ugo+rwx"
    owner: nobody
    group: users
    timeout: 120
  with_items:
    - {
        url: "https://launchpad.net/~{{ launchpad_user }}/+sshkeys",
        dest: "{{ cloud_init_dir }}/ssh_keys",
      }
    - {
        url: "https://cloud-images.ubuntu.com/jammy/current/jammy-server-cloudimg-amd64.img",
        dest: "{{ cloud_init_dir }}/jammy-server-cloudimg-amd64.img",
      }
    - {
        url: "https://cloud.debian.org/images/cloud/bookworm/latest/debian-12-genericcloud-amd64.qcow2",
        dest: "{{ cloud_init_dir }}/debian-12-genericcloud-amd64.qcow2",
      }

# Create templates
# https://docs.ansible.com/ansible/latest/collections/ansible/builtin/shell_module.html
- name: "Create cloud image templates"
  # Recreate when images changed
  # when: download.changed
  ansible.builtin.shell: |
    qm unlock {{ item.id }} || true
    qm destroy {{ item.id }} || true
    qm create {{ item.id }} \
      --name {{ item.name }} \
      --tags {{ item.tags }} \
      --memory 4096 \
      --cores 2 \
      --net0 virtio,bridge=vmbr1 \
      --scsihw virtio-scsi-pci \
      --ciuser {{ cloud_init_user }} \
      --cipassword {{ cloud_init_password }} \
      --searchdomain {{ cloud_init_domain }} \
      --sshkeys {{ cloud_init_dir }}/ssh_keys \
      --ipconfig0 ip=dhcp \
      --ostype l26 \
      --agent 1
    virt-customize -a {{ item.image }} --install qemu-guest-agent
    qm set {{ item.id }} --scsi0 vmdata:0,import-from={{ item.image }} --boot order=scsi0 --ide2 vmdata:cloudinit
    qm resize {{ item.id }} scsi0 8G
    qm template {{ item.id }}
  with_items:
    - {
        id: "9001",
        name: "ubuntu-jammy-template",
        tags: "ubuntu,jammy,cloud-image",
        image: "{{ cloud_init_dir }}/jammy-server-cloudimg-amd64.img"
      }
    - {
        id: "9002",
        name: "debian-bookworm-template",
        tags: "debian,bookworm,cloud-image",
        image: "{{ cloud_init_dir }}/debian-12-genericcloud-amd64.qcow2"
      }

Run playbook:

pieter@server-1:~/HomeAutomation$ ansible-playbook -vvv --ask-become-pass ~/HomeAutomation/Ansible/test.yml
ansible-playbook [core 2.15.2]
  config file = None
  configured module search path = ['/home/pieter/.ansible/plugins/modules', '/usr/share/ansible/plugins/modules']
  ansible python module location = /usr/local/lib/python3.9/dist-packages/ansible
  ansible collection location = /home/pieter/.ansible/collections:/usr/share/ansible/collections
  executable location = /usr/local/bin/ansible-playbook
  python version = 3.9.2 (default, Feb 28 2021, 17:03:44) [GCC 10.2.1 20210110] (/usr/bin/python3)
  jinja version = 3.0.1
  libyaml = True
No config file found; using defaults
BECOME password: 
...
PLAYBOOK: test.yml ***********************************************************************************************************************************************************************************************************************************************************************************
1 plays in /home/pieter/HomeAutomation/Ansible/test.yml
...
TASK [Create cloud image templates] ******************************************************************************************************************************************************************************************************************************************************************
task path: /home/pieter/HomeAutomation/Ansible/tasks/cloud_init.yml:54
<127.0.0.1> ESTABLISH LOCAL CONNECTION FOR USER: pieter
<127.0.0.1> EXEC /bin/sh -c 'echo ~pieter && sleep 0'
<127.0.0.1> EXEC /bin/sh -c '( umask 77 && mkdir -p "` echo /home/pieter/.ansible/tmp `"&& mkdir "` echo /home/pieter/.ansible/tmp/ansible-tmp-1690944817.2016468-1824495-48439512100173 `" && echo ansible-tmp-1690944817.2016468-1824495-48439512100173="` echo /home/pieter/.ansible/tmp/ansible-tmp-1690944817.2016468-1824495-48439512100173 `" ) && sleep 0'
Using module file /usr/local/lib/python3.9/dist-packages/ansible/modules/command.py
<127.0.0.1> PUT /home/pieter/.ansible/tmp/ansible-local-18231187hwfn7n8/tmpt8mizpiv TO /home/pieter/.ansible/tmp/ansible-tmp-1690944817.2016468-1824495-48439512100173/AnsiballZ_command.py
<127.0.0.1> EXEC /bin/sh -c 'chmod u+x /home/pieter/.ansible/tmp/ansible-tmp-1690944817.2016468-1824495-48439512100173/ /home/pieter/.ansible/tmp/ansible-tmp-1690944817.2016468-1824495-48439512100173/AnsiballZ_command.py && sleep 0'
<127.0.0.1> EXEC /bin/sh -c 'sudo -H -S -p "[sudo via ansible, key=qqdobawkyyzlecwcixjocubtcgjgsaml] password:" -u root /bin/sh -c '"'"'echo BECOME-SUCCESS-qqdobawkyyzlecwcixjocubtcgjgsaml ; /usr/bin/python3 /home/pieter/.ansible/tmp/ansible-tmp-1690944817.2016468-1824495-48439512100173/AnsiballZ_command.py'"'"' && sleep 0'
<127.0.0.1> EXEC /bin/sh -c 'rm -f -r /home/pieter/.ansible/tmp/ansible-tmp-1690944817.2016468-1824495-48439512100173/ > /dev/null 2>&1 && sleep 0'
failed: [localhost] (item={'id': '9001', 'name': 'ubuntu-jammy-template', 'tags': 'ubuntu,jammy,cloud-image', 'image': '/data/install/images/jammy-server-cloudimg-amd64.img'}) => {
    "ansible_loop_var": "item",
    "changed": true,
    "cmd": "qm unlock 9001 || true\nqm destroy 9001 || true\nqm create 9001  --name ubuntu-jammy-template  --tags ubuntu,jammy,cloud-image  --memory 4096  --cores 2  --net0 virtio,bridge=vmbr1  --scsihw virtio-scsi-pci  --ciuser pieter  --cipassword Password1  --searchdomain home.insanegenius.net  --sshkeys /data/install/images/ssh_keys  --ipconfig0 ip=dhcp  --ostype l26  --agent 1\nvirt-customize -a /data/install/images/jammy-server-cloudimg-amd64.img --install qemu-guest-agent\nqm set 9001 --scsi0 vmdata:0,import-from=/data/install/images/jammy-server-cloudimg-amd64.img --boot order=scsi0 --ide2 vmdata:cloudinit\nqm resize 9001 scsi0 8G\nqm template 9001\n",
    "delta": "0:00:12.984944",
    "end": "2023-08-01 19:53:50.477484",
    "invocation": {
        "module_args": {
            "_raw_params": "qm unlock 9001 || true\nqm destroy 9001 || true\nqm create 9001  --name ubuntu-jammy-template  --tags ubuntu,jammy,cloud-image  --memory 4096  --cores 2  --net0 virtio,bridge=vmbr1  --scsihw virtio-scsi-pci  --ciuser pieter  --cipassword Password1  --searchdomain home.insanegenius.net  --sshkeys /data/install/images/ssh_keys  --ipconfig0 ip=dhcp  --ostype l26  --agent 1\nvirt-customize -a /data/install/images/jammy-server-cloudimg-amd64.img --install qemu-guest-agent\nqm set 9001 --scsi0 vmdata:0,import-from=/data/install/images/jammy-server-cloudimg-amd64.img --boot order=scsi0 --ide2 vmdata:cloudinit\nqm resize 9001 scsi0 8G\nqm template 9001\n",
            "_uses_shell": true,
            "argv": null,
            "chdir": null,
            "creates": null,
            "executable": null,
            "removes": null,
            "stdin": null,
            "stdin_add_newline": true,
            "strip_empty_ends": true
        }
    },
    "item": {
        "id": "9001",
        "image": "/data/install/images/jammy-server-cloudimg-amd64.img",
        "name": "ubuntu-jammy-template",
        "tags": "ubuntu,jammy,cloud-image"
    },
    "msg": "non-zero return code",
    "rc": 25,
    "start": "2023-08-01 19:53:37.492540",
    "stderr": "failed to tcsetpgrp: Inappropriate ioctl for device\ngot no worker upid - start worker failed\nunable to create VM 9001 - VM 9001 already exists on node 'server-1'\nzfs error: cannot create 'ssdpool/vmdata/vm-9001-cloudinit': dataset already exists\nfailed to tcsetpgrp: Inappropriate ioctl for device\ngot no worker upid - start worker failed",
    "stderr_lines": [
        "failed to tcsetpgrp: Inappropriate ioctl for device",
        "got no worker upid - start worker failed",
        "unable to create VM 9001 - VM 9001 already exists on node 'server-1'",
        "zfs error: cannot create 'ssdpool/vmdata/vm-9001-cloudinit': dataset already exists",
        "failed to tcsetpgrp: Inappropriate ioctl for device",
        "got no worker upid - start worker failed"
    ],
    "stdout": "[   0.0] Examining the guest ...\n[   4.0] Setting a random seed\nvirt-customize: warning: random seed could not be set for this type of \nguest\n[   4.0] Installing packages: qemu-guest-agent\n[   7.2] Finishing off\nupdate VM 9001: -boot order=scsi0 -ide2 vmdata:cloudinit -scsi0 vmdata:0,import-from=/data/install/images/jammy-server-cloudimg-amd64.img",
    "stdout_lines": [
        "[   0.0] Examining the guest ...",
        "[   4.0] Setting a random seed",
        "virt-customize: warning: random seed could not be set for this type of ",
        "guest",
        "[   4.0] Installing packages: qemu-guest-agent",
        "[   7.2] Finishing off",
        "update VM 9001: -boot order=scsi0 -ide2 vmdata:cloudinit -scsi0 vmdata:0,import-from=/data/install/images/jammy-server-cloudimg-amd64.img"
    ]
}
<127.0.0.1> EXEC /bin/sh -c 'echo ~pieter && sleep 0'
<127.0.0.1> EXEC /bin/sh -c '( umask 77 && mkdir -p "` echo /home/pieter/.ansible/tmp `"&& mkdir "` echo /home/pieter/.ansible/tmp/ansible-tmp-1690944830.5114603-1824495-279107019138629 `" && echo ansible-tmp-1690944830.5114603-1824495-279107019138629="` echo /home/pieter/.ansible/tmp/ansible-tmp-1690944830.5114603-1824495-279107019138629 `" ) && sleep 0'
Using module file /usr/local/lib/python3.9/dist-packages/ansible/modules/command.py
<127.0.0.1> PUT /home/pieter/.ansible/tmp/ansible-local-18231187hwfn7n8/tmpyct2nw3u TO /home/pieter/.ansible/tmp/ansible-tmp-1690944830.5114603-1824495-279107019138629/AnsiballZ_command.py
<127.0.0.1> EXEC /bin/sh -c 'chmod u+x /home/pieter/.ansible/tmp/ansible-tmp-1690944830.5114603-1824495-279107019138629/ /home/pieter/.ansible/tmp/ansible-tmp-1690944830.5114603-1824495-279107019138629/AnsiballZ_command.py && sleep 0'
<127.0.0.1> EXEC /bin/sh -c 'sudo -H -S -p "[sudo via ansible, key=gfciqiccesiblnfsfuarenctwdneivte] password:" -u root /bin/sh -c '"'"'echo BECOME-SUCCESS-gfciqiccesiblnfsfuarenctwdneivte ; /usr/bin/python3 /home/pieter/.ansible/tmp/ansible-tmp-1690944830.5114603-1824495-279107019138629/AnsiballZ_command.py'"'"' && sleep 0'
<127.0.0.1> EXEC /bin/sh -c 'rm -f -r /home/pieter/.ansible/tmp/ansible-tmp-1690944830.5114603-1824495-279107019138629/ > /dev/null 2>&1 && sleep 0'
failed: [localhost] (item={'id': '9002', 'name': 'debian-bookworm-template', 'tags': 'debian,bookworm,cloud-image', 'image': '/data/install/images/debian-12-genericcloud-amd64.qcow2'}) => {
    "ansible_loop_var": "item",
    "changed": true,
    "cmd": "qm unlock 9002 || true\nqm destroy 9002 || true\nqm create 9002  --name debian-bookworm-template  --tags debian,bookworm,cloud-image  --memory 4096  --cores 2  --net0 virtio,bridge=vmbr1  --scsihw virtio-scsi-pci  --ciuser pieter  --cipassword Password1  --searchdomain home.insanegenius.net  --sshkeys /data/install/images/ssh_keys  --ipconfig0 ip=dhcp  --ostype l26  --agent 1\nvirt-customize -a /data/install/images/debian-12-genericcloud-amd64.qcow2 --install qemu-guest-agent\nqm set 9002 --scsi0 vmdata:0,import-from=/data/install/images/debian-12-genericcloud-amd64.qcow2 --boot order=scsi0 --ide2 vmdata:cloudinit\nqm resize 9002 scsi0 8G\nqm template 9002\n",
    "delta": "0:00:11.894339",
    "end": "2023-08-01 19:54:02.562572",
    "invocation": {
        "module_args": {
            "_raw_params": "qm unlock 9002 || true\nqm destroy 9002 || true\nqm create 9002  --name debian-bookworm-template  --tags debian,bookworm,cloud-image  --memory 4096  --cores 2  --net0 virtio,bridge=vmbr1  --scsihw virtio-scsi-pci  --ciuser pieter  --cipassword Password1  --searchdomain home.insanegenius.net  --sshkeys /data/install/images/ssh_keys  --ipconfig0 ip=dhcp  --ostype l26  --agent 1\nvirt-customize -a /data/install/images/debian-12-genericcloud-amd64.qcow2 --install qemu-guest-agent\nqm set 9002 --scsi0 vmdata:0,import-from=/data/install/images/debian-12-genericcloud-amd64.qcow2 --boot order=scsi0 --ide2 vmdata:cloudinit\nqm resize 9002 scsi0 8G\nqm template 9002\n",
            "_uses_shell": true,
            "argv": null,
            "chdir": null,
            "creates": null,
            "executable": null,
            "removes": null,
            "stdin": null,
            "stdin_add_newline": true,
            "strip_empty_ends": true
        }
    },
    "item": {
        "id": "9002",
        "image": "/data/install/images/debian-12-genericcloud-amd64.qcow2",
        "name": "debian-bookworm-template",
        "tags": "debian,bookworm,cloud-image"
    },
    "msg": "non-zero return code",
    "rc": 25,
    "start": "2023-08-01 19:53:50.668233",
    "stderr": "failed to tcsetpgrp: Inappropriate ioctl for device\ngot no worker upid - start worker failed\nunable to create VM 9002 - VM 9002 already exists on node 'server-1'\nzfs error: cannot create 'ssdpool/vmdata/vm-9002-cloudinit': dataset already exists\nfailed to tcsetpgrp: Inappropriate ioctl for device\ngot no worker upid - start worker failed",
    "stderr_lines": [
        "failed to tcsetpgrp: Inappropriate ioctl for device",
        "got no worker upid - start worker failed",
        "unable to create VM 9002 - VM 9002 already exists on node 'server-1'",
        "zfs error: cannot create 'ssdpool/vmdata/vm-9002-cloudinit': dataset already exists",
        "failed to tcsetpgrp: Inappropriate ioctl for device",
        "got no worker upid - start worker failed"
    ],
    "stdout": "[   0.0] Examining the guest ...\n[   3.8] Setting a random seed\nvirt-customize: warning: random seed could not be set for this type of \nguest\n[   3.8] Installing packages: qemu-guest-agent\n[   6.0] Finishing off\nupdate VM 9002: -boot order=scsi0 -ide2 vmdata:cloudinit -scsi0 vmdata:0,import-from=/data/install/images/debian-12-genericcloud-amd64.qcow2",
    "stdout_lines": [
        "[   0.0] Examining the guest ...",
        "[   3.8] Setting a random seed",
        "virt-customize: warning: random seed could not be set for this type of ",
        "guest",
        "[   3.8] Installing packages: qemu-guest-agent",
        "[   6.0] Finishing off",
        "update VM 9002: -boot order=scsi0 -ide2 vmdata:cloudinit -scsi0 vmdata:0,import-from=/data/install/images/debian-12-genericcloud-amd64.qcow2"
    ]
}

PLAY RECAP *******************************************************************************************************************************************************************************************************************************************************************************************
localhost                  : ok=4    changed=0    unreachable=0    failed=1    skipped=0    rescued=0    ignored=0   

pieter@server-1:~/HomeAutomation$

When the task fails, I am left with incomplete VM images, or VM images that are in a "locked (create)" state.

I did try to use /bin/bash as shell, but it made no difference.

PieterV
  • 555
  • 1
  • 4
  • 18
  • You have some obvious error messages giving evidence your image already exists. Which steps did you take to debug the unlock/destroy phases in your script? As is, you're blindly ignoring any error. – Zeitounator Aug 02 '23 at 06:00
  • The problem is the 25 return code, even if I make sure the remnant locked images are cleaned up before the run (# Fix locked images: sudo qm destroy 9001 --destroy-unreferenced-disks 1 --skiplock 1), one of the "qm" commands fail with error 25. – PieterV Aug 03 '23 at 16:44

1 Answers1

0

With no solution in sight I upgraded to PVE 8.0.3, and no more error 25.

The upgrade did initially fail to install the kernel Error! The /var/lib/dkms/aufs/4.19+20190211/6.2.16-5-pve/x86_64/dkms.conf for module aufs includes a BUILD_EXCLUSIVE directive which does not match this kernel/arch/config., I had to dkms remove aufs/4.19+20190211, and after that all was well.

I still don't know root cause, but upgrading from PVE 7 to 8 resolved the issue.

PieterV
  • 555
  • 1
  • 4
  • 18