Setting up azure virtual machine scale set with LinuxDiagnostic extension

Question

I am trying to setup a Service fabric cluster and while doing so I am creating a azure virtual machine scale set with LinuxDiagnostic as one of the extension. Following is the code for the VM scale set:

resource "azurerm_virtual_machine_scale_set" "sf_scale_set" {
  name                = "sf-scale-set-${terraform.workspace}"
  location            = "${var.location}"
  resource_group_name = "${azurerm_resource_group.fusion.name}"

  # automatic rolling upgrade
  automatic_os_upgrade = true
  upgrade_policy_mode  = "Automatic"

  # required when using rolling upgrade policy
  health_probe_id = "${azurerm_lb_probe.sf_lb_probe.id}"

  sku {
    name     = "${var.sf_scale_set_vm_config["name"]}"
    tier     = "${var.sf_scale_set_vm_config["tier"]}"
    capacity = "${var.sf_scale_set_vm_config["capacity"]}"
  }

  storage_profile_image_reference {
    publisher = "Canonical"
    offer     = "UbuntuServer"
    sku       = "16.04"
    version   = "6.0.12"
  }

  storage_profile_os_disk {
    name              = ""
    caching           = "ReadWrite"
    create_option     = "FromImage"
    managed_disk_type = "Standard_LRS"
  }

  os_profile_secrets {
    source_vault_id = "${var.sf_vault_id}"

    vault_certificates {
      certificate_url = "${var.sf_vault_url}"
    }
  }

  storage_profile_data_disk {
    lun           = 0
    caching       = "ReadWrite"
    create_option = "Empty"
    disk_size_gb  = 40
  }

  os_profile {
    computer_name_prefix = "sf-vm-${terraform.workspace}"
    admin_username       = "hachadmin"
  }

  os_profile_linux_config {
    disable_password_authentication = true

    ssh_keys {
      path     = "/home/admin/.ssh/authorized_keys"
      key_data = "${file("sshkeys/admin.pub")}"
    }
  }

  network_profile {
    name    = "sf-vm-net-profile-${terraform.workspace}"
    primary = true

    ip_configuration {
      name                                   = "sf-ip-config-${terraform.workspace}"
      primary                                = true
      subnet_id                              = "${azurerm_subnet.sf_vnet_subnet.id}"
      load_balancer_backend_address_pool_ids = ["${azurerm_lb_backend_address_pool.sf_be_vm_set.id}"]
      load_balancer_inbound_nat_rules_ids    = ["${element(azurerm_lb_nat_pool.sf_nat_vm_set.*.id, count.index)}"]
    }
  }

  extension {
    name                 = "sf-scale-set-extension-${terraform.workspace}"
    publisher            = "Microsoft.Azure.ServiceFabric"
    type                 = "ServiceFabricLinuxNode"
    type_handler_version = "1.0"
    settings             = "{  \"certificate\": { \"thumbprint\": \"${var.cert_thumbprint}\", \"x509StoreName\": \"My\" } , \"clusterEndpoint\": \"${azurerm_service_fabric_cluster.sf_service.cluster_endpoint}\", \"nodeTypeRef\": \"${terraform.workspace}-sf-node-type\", \"durabilityLevel\": \"${var.sf_reliability}\",\"nicPrefixOverride\": \"${azurerm_subnet.sf_vnet_subnet.address_prefix}\",\"enableParallelJobs\": \"true\"}"
    protected_settings   = "{\"StorageAccountKey1\": \"${azurerm_storage_account.sf_storage.primary_access_key}\", \"StorageAccountKey2\": \"${azurerm_storage_account.sf_storage.secondary_access_key}\"}"
  }

  extension {
    name                       = "sf-scale-set-linux-diag-extension-${terraform.workspace}"                                                                                                                                                                                                                                        # This extension connects vms to the cluster.
    publisher                  = "Microsoft.OSTCExtensions"
    type                       = "LinuxDiagnostic"
    type_handler_version       = "2.3"
    auto_upgrade_minor_version = true
    protected_settings         = "{\"storageAccountName\": \"${azurerm_storage_account.sf_storage_app_diag.primary_access_key}\", \"StorageAccountKey1\": \"${azurerm_storage_account.sf_storage_app_diag.primary_access_key}\", \"StorageAccountKey2\": \"${azurerm_storage_account.sf_storage_app_diag.secondary_access_key}\"}"
    settings                   = "${data.template_file.settings.rendered}"
  }

  tags {
    Region      = "${var.location}"
    Createdby   = "${var.created_by_tag}"
    Team        = "${var.team_tag}"
    Environment = "${terraform.workspace}"
    ninetofive  = "${var.ninetofivetag}"
  }
}

data "template_file" "settings" {
  template = "${file("${path.module}/diagnostics/settings2.3.json.tpl")}"

  vars {
    xml_cfg           = "${base64encode(data.template_file.wadcfg.rendered)}"
    diag_storage_name = "${azurerm_storage_account.sf_storage_app_diag.name}"
  }
}

data "template_file" "wadcfg" {
  template   = "${file("${path.module}/diagnostics/wadcfg.xml.tpl")}"

  vars {
    virtual_machine_id = "${azurerm_virtual_machine_scale_set.sf_scale_set.id}"
  }
}

The end of Wadcfg file looks like follow:

<WadCfg>
<PerformanceCounters scheduledTransferPeriod="PT1M">
.....
......
    </PerformanceCounters>
    <Metrics resourceId="${virtual_machine_id}">
      <MetricAggregation scheduledTransferPeriod="PT1H"/>
      <MetricAggregation scheduledTransferPeriod="PT1M"/>
    </Metrics>
  </DiagnosticMonitorConfiguration>
</WadCfg>

Settings2.3.json.tpl file is

{
  "xmlCfg": "${xml_cfg}",
  "storageAccount": "${diag_storage_name}"
}

While trying to run the Terraform code I get the following error:

[+] Found tfvars file ./profiles/eu-sprint/eu-sprint.tfvars

Error: Cycle: data.template_file.wadcfg, data.template_file.settings, azurerm_virtual_machine_scale_set.sf_scale_set

I am assuming that Terraform is trying to render the template wadcfg.xml.tpl without the Azure VM scale set. Following are some of my question:

• How can I enforce Terraform to wait until the Azure VM scale set are created before trying to render the wadcfg.xml.tpl file
• As part of rendering my wadcfg.xml.tpl I am passing the VM id's, I know this will work if I am only creating one instance but will the code above also work for VM scale set without me explicitly looping through each VM's? If incase I would have to loop through each of them, what would be the recommenced approach?
• I saw there's a https://www.terraform.io/docs/providers/azurerm/r/virtual_machine_extension.html resource existing to install VM extension, will this also work for VM scale set? If not is there a better way I could organize my settings and protected_settings part so that they are reader friendly?

I would appreciate some help here.