3

I have a scale set with 5 extensions. 2 of them are for Service Profiler, to install .net 4.6.1, and for the service profiler agent itself. When I deploy the template with all 5 extensions, due (I think..) to the .net installation requiring a restart of the vm, the template will always result in a status of failed. However, it doesn't seem to have actually failed, as when the vm restarts, it resumes any extensions that haven't finished (again, I think.. Unfortunately, my project isn't in the position to be able to test this infrastructure with an application atm).

So, I have attempted to move the 2 service profiler extensions to a linked template, so the status of the scale set will go to succeeded, and actions after this won't be impacted. (As well as other resources in the template being dependent on the scale set, I assume application deployment from VSTS release manager won't happen if the infrastructure deployment 'failed'.)

So my linked template simply has the scale set resource, but with only the extensions defined:

"resources": [
{
  "apiVersion": "2016-03-30",
  "type": "Microsoft.Compute/virtualMachineScaleSets",
  "name": "[variables('vmNodeType0Name')]",
  "location": "[resourceGroup().location]",
  "properties": {
    "virtualMachineProfile": {
      "extensionProfile": {
        "extensions": [
          {
            "properties": {
              "publisher": "Microsoft.Compute",
              "type": "CustomScriptExtension",
              "typeHandlerVersion": "1.7",
              "autoUpgradeMinorVersion": false,
              "settings": {
                "fileUris": [ "https://serviceprofiler.azurewebsites.net/content/downloads/InstallNetFx46.ps1" ],
                "commandToExecute": "powershell.exe -ExecutionPolicy Unrestricted -File InstallNetFx46.ps1"
              },
              "forceUpdateTag": "RerunExtension"
            },
            "name": "CustomScriptExtensionInstallNet46"
          },
          {
            "properties": {
              "publisher": "Microsoft.VisualStudio.ServiceProfiler",
              "type": "ServiceProfilerAgent",
              "typeHandlerVersion": "0.1",
              "autoUpgradeMinorVersion": true,
              "settings": {
                "config": {
                  "ServiceName": "<nameChanged>",
                  "CircularEtlBufferMB": 200,
                  "MonitorSamplingRate": 1.0,
                  "ProfileSamplingRate": 0.05,
                  "AgentLogFilter": "Warning",
                  "ProvideUsageTelemetryData": true,
                  "EtwMetrics": [
                    {
                      "ProviderName": "Microsoft-ServiceFabric-Actors",
                      "ProviderKeywords": 2,
                      "ProviderLevel": "Verbose",
                      "Event": "ActorMethod/Start",
                      "EventStop": "ActorMethod/Stop",
                      "Name": "methodName"
                    },
                    {
                      "ProviderName": "Microsoft-ServiceFabric-Actors",
                      "ProviderKeywords": 4,
                      "ProviderLevel": "Verbose",
                      "Event": "ActorSaveState/Start",
                      "EventStop": "ActorSaveState/Stop",
                      "Name": "actorType"
                    },
                    {
                      "ProviderName": "<nameChanged>",
                      "ProviderKeywords": 0,
                      "ProviderLevel": "Informational",
                      "Event": "Request/Start",
                      "EventStop": "Request/Stop",
                      "Name": "url"
                    }
                  ],
                  "Tags": [
                    {
                      "Type": "Performance",
                      "Settings": {
                        "SampleIntervalInSeconds": "5",
                        "SamplesToConsider": "6",
                        "Triggers": [
                          {
                            "Name": "High CPU",
                            "Description": "High CPU usage",
                            "PerfCounter": "Processor Information\\% Processor Time\\_Total",
                            "Operator": ">",
                            "Metric": "70"
                          },
                          {
                            "Name": "Busy Disk",
                            "Description": "High disk usage",
                            "PerfCounter": "PhysicalDisk\\% Disk Time\\_Total",
                            "Operator": ">",
                            "Metric": "10"
                          },
                          {
                            "Name": "Memory Pressure",
                            "Description": "High memory usage",
                            "PerfCounter": "Memory\\Available MBytes",
                            "Operator": "<",
                            "Metric": "400"
                          },
                          {
                            "Name": "High GC",
                            "Description": "High GC time",
                            "PerfCounter": ".NET CLR Memory\\% Time in GC\\_Global_",
                            "Operator": ">",
                            "Metric": "10"
                          }
                        ]
                      }
                    },
                    {
                      "Type": "Version",
                      "Settings": {
                        "Source": {
                          "Type": "ServiceFabric"
                        }
                      }
                    }
                  ]
                }
              },
              "protectedSettings": {
                "storageAccountName": "[variables('applicationDiagnosticsStorageAccountName')]",
                "storageAccountKey": "[listKeys(resourceId('Microsoft.Storage/storageAccounts', variables('applicationDiagnosticsStorageAccountName')),'2015-05-01-preview').key1]",
                "storageAccountEndPoint": "https://core.windows.net/"
              }
            },
            "name": "ServiceProfilerAgent"
          }
        ]
      }
    }
  }

However, I get this error message:

"message": "{\r\n \"error\": {\r\n \"code\": \"OperationNotAllowed\",\r\n \"message\": \"VM Scale Set extensions of handler 'Microsoft.Azure.ServiceFabric.ServiceFabricNode' can be deleted only at the time of VM Scale Set deletion.\"\r\n }\r\n}" },

Seems kind of strange considering I'm not trying to delete anything. And ofc there is literally nothing, as far as I can tell, in the documentation about extensions in linked templates, or anywhere else...

Is this even supported? Or should I just leave it in 1 template? Any help would be great!

Edward Rixon
  • 1,199
  • 1
  • 17
  • 31
  • Extensions on VMSS are not child resources so I don't think you can split them into another template.... On your first attempt, is the only hiccup you're having with the .net install requiring a reboot (which is not handled appropriately)? – bmoore-msft Jun 03 '16 at 16:00
  • hi bmoore! ah ok, I thought it was possible to add extensions after the fact, but not for vmss then. Yes, it is just that extension reporting failure. Apart from that, the vmss onboards to automation for dsc, and on to the cluster fine. I am actually about to try using a custom image with .net already installed to see if that works better. That shouldn't cause any issues with msft updates right? It should still be PaaS, as the service fabric node extension is there at creation. – Edward Rixon Jun 03 '16 at 16:22
  • (I thought that was you ;) Custom image will work, I'm not sure how updates are handled, I would guess VMSS, you need to make sure you have the update agent installed. Are you using DSC to install the framework? I may have an inelegant workaround for DSC and the reboot problem. – bmoore-msft Jun 03 '16 at 21:54
  • atm .net is installed with an extension, taken from the service profiler website, while dsc is used for other things. Would dsc be able to setup the service profiler, as well as install .net? Also, custom image won't work for us unfortunately, as it limits the number of storage accounts for the ss vhds to 1. – Edward Rixon Jun 07 '16 at 13:04
  • for anyone who reads.. I have gone back some commits, to a version of my template with all extensions in it, and there is no conflict now. So that is weird. Maybe msft have changed something, maybe its inconsistent... but for now, it works, so im leaving it. – Edward Rixon Jun 08 '16 at 15:28
  • msft have indeed updated the dsc extension version to 2.18 over the weekend. one of the bug fixes: "Fix for the intermittent issue where extension stops processing configuration after a reboot. This was causing the DSC Extension to remain in ‘transitioning’ state." While this doesn't describe my case exactly, I suspect this has fixed my issue – Edward Rixon Jun 08 '16 at 15:49
  • So you got dotnet working via DSC? (if so, you have to share the solution :)) – bmoore-msft Jun 09 '16 at 18:07
  • lol nope, i just used my old template that has all 5 extensions in the SS. will post as answer. – Edward Rixon Jun 10 '16 at 10:13
  • or so I thought. After changing precisely nothing in the template, the last deployment failed. how can it be so inconsistent?! ffs.. – Edward Rixon Jun 10 '16 at 11:18
  • Failed? - you got an error or the extensions didn't run? Related to this maybe? http://stackoverflow.com/questions/37723567/how-to-force-vm-created-with-an-arm-template-customscriptextension-to-redownlo – bmoore-msft Jun 10 '16 at 16:12
  • No it was an entirely new deployment in a new resource group. It produces one error, with the extension failing to complete (the custom extension that installs .net). But! it doesn't happen on every machine within the scale set. go figure... I have now put the .net installation within a dsc script, which has worked great. I may have issues next week with the service profiler agent, but for now, im done. tyvm for your continued help tho! – Edward Rixon Jun 10 '16 at 16:34
  • np - it shouldn't be this much of a task, though feel free to reach out directly if you get blocked... – bmoore-msft Jun 12 '16 at 15:45

2 Answers2

1

the error message about deleting extensions is because the extension list is applied as a whole, so if you don't have the two original extensions (from the main template) it will think you are deleting them. I.e. you'd need to include all 4 extensions in the list. The suggestion in the comments of using a custom image with .Net installed is a good one. Another option for getting information on to the machine is to use the customData property - though it probably won't help this specific case.

There are some requests to create a platform image with latest .Net installed (I'm wondering whether any of the marketplace images already have this), and I think we should do it.

sendmarsh
  • 1,046
  • 7
  • 11
  • It appears that your suggestion won't work for me I'm afraid. The service fabric node extension requires a reference to the cluster endpoint so it can onboard the scale set. I tried passing the endpoint in as a parameter, as an output, and I even put the cluster itself into the second template but get this error everytime...: "The resource 'https:/westeurope.servicefabric.azure.com/runtime/clusters/' is not defined in the template." – Edward Rixon Jun 07 '16 at 14:56
  • Hi Guy. I was hoping you could confirm to me that .Net 461 is now present on the server 2012 r2 image for azure vm. It appears it is, but there has been no announcement, and a colleague of mine said it wasn't coming til server 2016... – Edward Rixon Jul 25 '16 at 16:35
1

Here is my full scale set resource, with all extensions in the single template. It originally didn't work, but it does now...

{
  "apiVersion": "[variables('vmssApiVersion')]",
  "type": "Microsoft.Compute/virtualMachineScaleSets",
  "name": "[parameters('vmNodeType0Name')]",
  "location": "[variables('computeLocation')]",
  "tags": {
    "resourceType": "node",
    "environmentName": "[parameters('prefix')]",
    "displayName": "[parameters('vmNodeType0DisplayName')]"
  },
  "dependsOn": [
    "storageLoop",
    "[variables('lbID0')]",
    "[concat('Microsoft.Network/virtualNetworks/', variables('virtualNetworkName'))]",
    "[concat('Microsoft.Storage/storageAccounts/', variables('supportLogStorageAccountName'))]",
    "[concat('Microsoft.Storage/storageAccounts/', variables('applicationDiagnosticsStorageAccountName'))]",
    "[concat('Microsoft.Automation/automationAccounts/', parameters('automationAccountName'))]",
    "[concat('Microsoft.Automation/automationAccounts/', parameters('automationAccountName'),'/Modules/',variables('dscModules').xNetworking.ModuleName)]",
    "[concat('Microsoft.Automation/automationAccounts/', parameters('automationAccountName'),'/Configurations/', parameters('configurationName'))]"
  ],
  "properties": {
    "overprovision": "[parameters('overProvision')]",
    "upgradePolicy": {
      "mode": "Automatic"
    },
    "virtualMachineProfile": {
      "extensionProfile": {
        "extensions": [
          {
            "properties": {
              "publisher": "Microsoft.Compute",
              "type": "CustomScriptExtension",
              "typeHandlerVersion": "1.7",
              "autoUpgradeMinorVersion": false,
              "settings": {
                "fileUris": [ "https://serviceprofiler.azurewebsites.net/content/downloads/InstallNetFx46.ps1" ],
                "commandToExecute": "powershell.exe -ExecutionPolicy Unrestricted -File InstallNetFx46.ps1"
              },
              "forceUpdateTag": "RerunExtension"
            },
            "name": "CustomScriptExtensionInstallNet46"
          },
          {
            "properties": {
              "publisher": "Microsoft.VisualStudio.ServiceProfiler",
              "type": "ServiceProfilerAgent",
              "typeHandlerVersion": "0.1",
              "autoUpgradeMinorVersion": true,
              "settings": {
                "config": {
                  "ServiceName": "<nameChanged>",
                  "CircularEtlBufferMB": 200,
                  "MonitorSamplingRate": 1.0,
                  "ProfileSamplingRate": 0.05,
                  "AgentLogFilter": "Warning",
                  "ProvideUsageTelemetryData": true,
                  "EtwMetrics": [
                    {
                      "ProviderName": "Microsoft-ServiceFabric-Actors",
                      "ProviderKeywords": 2,
                      "ProviderLevel": "Verbose",
                      "Event": "ActorMethod/Start",
                      "EventStop": "ActorMethod/Stop",
                      "Name": "methodName"
                    },
                    {
                      "ProviderName": "Microsoft-ServiceFabric-Actors",
                      "ProviderKeywords": 4,
                      "ProviderLevel": "Verbose",
                      "Event": "ActorSaveState/Start",
                      "EventStop": "ActorSaveState/Stop",
                      "Name": "actorType"
                    },
                    {
                      "ProviderName": "<nameChanged>",
                      "ProviderKeywords": 0,
                      "ProviderLevel": "Informational",
                      "Event": "Request/Start",
                      "EventStop": "Request/Stop",
                      "Name": "url"
                    }
                  ],
                  "Tags": [
                    {
                      "Type": "Performance",
                      "Settings": {
                        "SampleIntervalInSeconds": "5",
                        "SamplesToConsider": "6",
                        "Triggers": [
                          {
                            "Name": "High CPU",
                            "Description": "High CPU usage",
                            "PerfCounter": "Processor Information\\% Processor Time\\_Total",
                            "Operator": ">",
                            "Metric": "70"
                          },
                          {
                            "Name": "Busy Disk",
                            "Description": "High disk usage",
                            "PerfCounter": "PhysicalDisk\\% Disk Time\\_Total",
                            "Operator": ">",
                            "Metric": "10"
                          },
                          {
                            "Name": "Memory Pressure",
                            "Description": "High memory usage",
                            "PerfCounter": "Memory\\Available MBytes",
                            "Operator": "<",
                            "Metric": "400"
                          },
                          {
                            "Name": "High GC",
                            "Description": "High GC time",
                            "PerfCounter": ".NET CLR Memory\\% Time in GC\\_Global_",
                            "Operator": ">",
                            "Metric": "10"
                          }
                        ]
                      }
                    },
                    {
                      "Type": "Version",
                      "Settings": {
                        "Source": {
                          "Type": "ServiceFabric"
                        }
                      }
                    }
                  ]
                }
              },
              "protectedSettings": {
                "storageAccountName": "[variables('applicationDiagnosticsStorageAccountName')]",
                "storageAccountKey": "[listKeys(resourceId('Microsoft.Storage/storageAccounts', variables('applicationDiagnosticsStorageAccountName')),'2015-05-01-preview').key1]",
                "storageAccountEndPoint": "https://core.windows.net/"
              }
            },
            "name": "ServiceProfilerAgent"
          },
          {
            "name": "[concat(parameters('vmNodeType0Name'),'_ServiceFabricNode')]",
            "properties": {
              "type": "ServiceFabricNode",
              "autoUpgradeMinorVersion": false,
              "protectedSettings": {
                "StorageAccountKey1": "[listKeys(resourceId('Microsoft.Storage/storageAccounts', variables('supportLogStorageAccountName')),'2015-05-01-preview').key1]",
                "StorageAccountKey2": "[listKeys(resourceId('Microsoft.Storage/storageAccounts', variables('supportLogStorageAccountName')),'2015-05-01-preview').key2]"
              },
              "publisher": "Microsoft.Azure.ServiceFabric",
              "settings": {
                "clusterEndpoint": "[reference(parameters('clusterName')).clusterEndpoint]",
                "nodeTypeRef": "[parameters('vmNodeType0Name')]",
                "dataPath": "D:\\\\SvcFab",
                "durabilityLevel": "Bronze",
                "certificate": {
                  "thumbprint": "[parameters('certificateThumbprint')]",
                  "x509StoreName": "[parameters('certificateStoreValue')]"
                }
              },
              "typeHandlerVersion": "1.0"
            }
          },
          {
            "name": "[concat('VMDiagnosticsVmExt','_vmNodeType0Name')]",
            "properties": {
              "type": "IaaSDiagnostics",
              "autoUpgradeMinorVersion": true,
              "protectedSettings": {
                "storageAccountName": "[variables('applicationDiagnosticsStorageAccountName')]",
                "storageAccountKey": "[listkeys(variables('accountid'), '2015-05-01-preview').key1]",
                "storageAccountEndPoint": "https://core.windows.net/"
              },
              "publisher": "Microsoft.Azure.Diagnostics",
              "settings": {
                "WadCfg": {
                  "DiagnosticMonitorConfiguration": {
                    "overallQuotaInMB": "50000",
                    "EtwProviders": {
                      "EtwEventSourceProviderConfiguration": [
                        {
                          "provider": "Microsoft-ServiceFabric-Actors",
                          "scheduledTransferKeywordFilter": "1",
                          "scheduledTransferPeriod": "PT5M",
                          "DefaultEvents": {
                            "eventDestination": "ServiceFabricReliableActorEventTable"
                          }
                        },
                        {
                          "provider": "Microsoft-ServiceFabric-Services",
                          "scheduledTransferPeriod": "PT5M",
                          "DefaultEvents": {
                            "eventDestination": "ServiceFabricReliableServiceEventTable"
                          }
                        }
                      ],
                      "EtwManifestProviderConfiguration": [
                        {
                          "provider": "cbd93bc2-71e5-4566-b3a7-595d8eeca6e8",
                          "scheduledTransferLogLevelFilter": "Information",
                          "scheduledTransferKeywordFilter": "4611686018427387904",
                          "scheduledTransferPeriod": "PT5M",
                          "DefaultEvents": {
                            "eventDestination": "ServiceFabricSystemEventTable"
                          }
                        }
                      ]
                    }
                  }
                },
                "StorageAccount": "[variables('applicationDiagnosticsStorageAccountName')]"
              },
              "typeHandlerVersion": "1.5"
            }
          },
          {
            "name": "Microsoft.Powershell.DSC",
            "properties": {
              "publisher": "Microsoft.Powershell",
              "type": "DSC",
              "typeHandlerVersion": "2.17",
              "autoUpgradeMinorVersion": true,
              "protectedSettings": {
                "Items": {
                  "registrationKeyPrivate": "[parameters('registrationKey')]"
                }
              },
              "settings": {
                "WmfVersion": "latest",
                "ModulesUrl": "https://raw.github.com/Azure/azure-quickstart-templates/master/201-vmss-automation-dsc/UpdateLCMforAAPull.zip",
                "SasToken": "[parameters('_artifactsLocationSasToken')]",
                "ConfigurationFunction": "UpdateLCMforAAPull.ps1\\ConfigureLCMforAAPull",
                "Properties": [
                  {
                    "Name": "RegistrationKey",
                    "Value": {
                      "UserName": "[parameters('adminUserName')]",
                      "Password": "PrivateSettingsRef:registrationKeyPrivate"
                    },
                    "TypeName": "System.Management.Automation.PSCredential"
                  },
                  {
                    "Name": "RegistrationUrl",
                    "Value": "[parameters('registrationUrl')]",
                    "TypeName": "System.String"
                  },
                  {
                    "Name": "NodeConfigurationName",
                    "Value": "[parameters('nodeConfigurationName')]",
                    "TypeName": "System.String"
                  },
                  {
                    "Name": "ConfigurationMode",
                    "Value": "[parameters('configurationMode')]",
                    "TypeName": "System.String"
                  },
                  {
                    "Name": "ConfigurationModeFrequencyMins",
                    "Value": "[parameters('configurationModeFrequencyMins')]",
                    "TypeName": "System.Int32"
                  },
                  {
                    "Name": "RefreshFrequencyMins",
                    "Value": "[parameters('refreshFrequencyMins')]",
                    "TypeName": "System.Int32"
                  },
                  {
                    "Name": "RebootNodeIfNeeded",
                    "Value": "[parameters('rebootNodeIfNeeded')]",
                    "TypeName": "System.Boolean"
                  },
                  {
                    "Name": "ActionAfterReboot",
                    "Value": "[parameters('actionAfterReboot')]",
                    "TypeName": "System.String"
                  },
                  {
                    "Name": "AllowModuleOverwrite",
                    "Value": "[parameters('allowModuleOverwrite')]",
                    "TypeName": "System.Boolean"
                  },
                  {
                    "Name": "Timestamp",
                    "Value": "[parameters('timestamp')]",
                    "TypeName": "System.String"
                  }
                ]
              }
            }
          }
        ]
      },
      "networkProfile": {
        "networkInterfaceConfigurations": [
          {
            "name": "[concat(variables('nicName'), '-0')]",
            "properties": {
              "ipConfigurations": [
                {
                  "name": "[concat(variables('nicName'),'-',0)]",
                  "properties": {
                    "loadBalancerBackendAddressPools": [
                      {
                        "id": "[variables('lbPoolID0')]"
                      }
                    ],
                    "loadBalancerInboundNatPools": [
                      {
                        "id": "[variables('lbNatPoolID0')]"
                      }
                    ],
                    "subnet": {
                      "id": "[variables('subnet0Ref')]"
                    }
                  }
                }
              ],
              "primary": true
            }
          }
        ]
      },
      "osProfile": {
        "adminPassword": "[parameters('adminPassword')]",
        "adminUsername": "[parameters('adminUsername')]",
        "computernamePrefix": "[parameters('vmNodeType0Name')]",
        "secrets": [
          {
            "sourceVault": {
              "id": "[parameters('sourceVaultValue')]"
            },
            "vaultCertificates": [
              {
                "certificateStore": "[parameters('certificateStoreValue')]",
                "certificateUrl": "[parameters('certificateUrlValue')]"
              }
            ]
          }
        ]
      },
      "storageProfile": {
        "imageReference": {
          "publisher": "[parameters('vmImagePublisher')]",
          "offer": "[parameters('vmImageOffer')]",
          "sku": "[parameters('vmImageSku')]",
          "version": "[parameters('vmImageVersion')]"
        },
        "osDisk": {
          "vhdContainers": [
            "[concat('https://', variables('uniqueStringArray')[0], '.blob.core.windows.net/', parameters('vmStorageAccountContainerNameType0'))]",
            "[concat('https://', variables('uniqueStringArray')[1], '.blob.core.windows.net/', parameters('vmStorageAccountContainerNameType0'))]",
            "[concat('https://', variables('uniqueStringArray')[2], '.blob.core.windows.net/', parameters('vmStorageAccountContainerNameType0'))]",
            "[concat('https://', variables('uniqueStringArray')[3], '.blob.core.windows.net/', parameters('vmStorageAccountContainerNameType0'))]",
            "[concat('https://', variables('uniqueStringArray')[4], '.blob.core.windows.net/', parameters('vmStorageAccountContainerNameType0'))]"
          ],
          "name": "vmssosdisk",
          "caching": "ReadOnly",
          "createOption": "FromImage"
        }
      }
    }
  },
  "sku": {
    "name": "[parameters('vmNodeType0Size')]",
    "capacity": "5",
    "tier": "Standard"
  }
}
Edward Rixon
  • 1,199
  • 1
  • 17
  • 31