1

When I create a linked service in Azure Data Factory (ADF) for Databricks with terraform (using azurerm_data_factory_linked_service_azure_databricks) the linked service shows up only in live mode.

How can I make the linked service available in GIT mode where all the other ADF pipeline configurations are stored?

terraform {
  required_providers {
    azurerm = {
      source  = "hashicorp/azurerm"
      version = "=2.97.0"
    }
    databricks = {
      source = "databrickslabs/databricks"
    }
  }
}

provider "azurerm" {
  features {}
}

provider "databricks" {
  host          = azurerm_databricks_workspace.this.workspace_url
}

data "azurerm_client_config" "this" {
}

resource "azurerm_data_factory" "this" {
  name                = "myadf-9182371362"
  resource_group_name = "testrg"
  location            = "East US"

  identity {
    type = "SystemAssigned"
  }       

  vsts_configuration {
      account_name    = "mydevopsorg"
      branch_name     = "main"
      project_name    = "adftest"
      repository_name = "adftest"
      root_folder     = "/adf/"
      tenant_id       = data.azurerm_client_config.this.tenant_id
  }

}

resource "azurerm_databricks_workspace" "this" {
  name                 = "mydbworkspace"
  resource_group_name = "testrg"
  location            = "East US"
  sku                  = "standard"

}

data "databricks_node_type" "smallest" {
  local_disk = true

  depends_on = [
    azurerm_databricks_workspace.this
  ]
}

data "databricks_spark_version" "latest_lts" {
  long_term_support = true

  depends_on = [
    azurerm_databricks_workspace.this
  ]
}

resource "databricks_cluster" "this" {
  cluster_name = "Single Node"
  spark_version = data.databricks_spark_version.latest_lts.id
  node_type_id  = data.databricks_node_type.smallest.id
  autotermination_minutes = 20

  spark_conf = {
    "spark.databricks.cluster.profile" : "singleNode"
    "spark.master" : "local[*]"
  }

  depends_on = [
    azurerm_databricks_workspace.this
  ]

  custom_tags = {
    "ResourceClass" = "SingleNode"
  }
}

data "azurerm_resource_group" "this" {
  name = "testrg"
}

resource "azurerm_role_assignment" "example" {
  scope                = data.azurerm_resource_group.this.id 
  role_definition_name = "Contributor"  
  principal_id         = azurerm_data_factory.this.identity[0].principal_id
}

resource "azurerm_data_factory_linked_service_azure_databricks" "msi_linked" {
  name                = "ADBLinkedServiceViaMSI"
  data_factory_id     = azurerm_data_factory.this.id
  resource_group_name = "testrg"
  description         = "ADB Linked Service via MSI"
  adb_domain          = "https://${azurerm_databricks_workspace.this.workspace_url}"

  existing_cluster_id = databricks_cluster.this.id
  msi_work_space_resource_id = azurerm_databricks_workspace.this.id
}

result in git mode Git mode

result in live mode live mode

techtech
  • 31
  • 5
  • Is this the entire Terraform code? – Marko E Mar 14 '22 at 09:24
  • Marko, I updated the code, now it's a complete example – techtech Mar 14 '22 at 11:55
  • Ok, so there are no issues with this? – Marko E Mar 14 '22 at 15:02
  • Terraform creates the resources but the created linked service (Databricks connection) is in the live mode of data factory. The ADF pipeline configurations are stored and git and ADF is connected to Git. Now I have the linked service in live mode and the pipelines in git mode. But I need both in the same mode to run the pipeline using the linked service. – techtech Mar 14 '22 at 15:55

0 Answers0