Merge pull request #75 from ryhud/master

Adding AML 301 and updates to 201 and 202

quickstart/101-machine-learning/main.tf

@@ -1,5 +1,5 @@
 terraform {
-  required_version = ">=0.15.0"
+  required_version = ">=1.0"
 
   required_providers {
     azurerm = {

quickstart/201-machine-learning-moderately-secure/bastion.tf

@@ -0,0 +1,125 @@
+resource "azurerm_public_ip" "azure_bastion" {
+  name                = "pip-azure-bastion"
+  location            = azurerm_resource_group.default.location
+  resource_group_name = azurerm_resource_group.default.name
+  allocation_method   = "Static"
+  sku                 = "Standard"
+}
+
+resource "azurerm_network_security_group" "bastion_nsg" {
+  name                = "nsg-bastion"
+  location            = azurerm_resource_group.default.location
+  resource_group_name = azurerm_resource_group.default.name
+
+  security_rule {
+    name                       = "AllowHTTPSInbound"
+    priority                   = 100
+    direction                  = "Inbound"
+    access                     = "Allow"
+    protocol                   = "Tcp"
+    source_port_range          = "*"
+    destination_port_range     = "443"
+    source_address_prefix      = "Internet"
+    destination_address_prefix = "*"
+  }
+  security_rule {
+    name                       = "AllowGatewayManagerInbound"
+    priority                   = 200
+    direction                  = "Inbound"
+    access                     = "Allow"
+    protocol                   = "Tcp"
+    source_port_range          = "*"
+    destination_port_range     = "443"
+    source_address_prefix      = "GatewayManager"
+    destination_address_prefix = "*"
+  }
+  security_rule {
+    name                       = "AllowAzureLBInbound"
+    priority                   = 300
+    direction                  = "Inbound"
+    access                     = "Allow"
+    protocol                   = "Tcp"
+    source_port_range          = "*"
+    destination_port_range     = "443"
+    source_address_prefix      = "AzureLoadBalancer"
+    destination_address_prefix = "*"
+  }
+  security_rule {
+    name                       = "AllowBastionHostCommunication"
+    priority                   = 400
+    direction                  = "Inbound"
+    access                     = "Allow"
+    protocol                   = "*"
+    source_port_range          = "*"
+    destination_port_ranges    = ["5701", "8080"]
+    source_address_prefix      = "VirtualNetwork"
+    destination_address_prefix = "VirtualNetwork"
+  }
+  security_rule {
+    name                       = "AllowRdpSshOutbound"
+    priority                   = 100
+    direction                  = "Outbound"
+    access                     = "Allow"
+    protocol                   = "Tcp"
+    source_port_range          = "*"
+    destination_port_ranges    = ["22", "3389"]
+    source_address_prefix      = "*"
+    destination_address_prefix = "VirtualNetwork"
+  }
+  security_rule {
+    name                       = "AllowBastionHostCommunicationOutbound"
+    priority                   = 110
+    direction                  = "Outbound"
+    access                     = "Allow"
+    protocol                   = "Tcp"
+    source_port_range          = "*"
+    destination_port_ranges    = ["5701", "8080"]
+    source_address_prefix      = "VirtualNetwork"
+    destination_address_prefix = "VirtualNetwork"
+  }
+  security_rule {
+    name                       = "AllowAzureCloudOutbound"
+    priority                   = 120
+    direction                  = "Outbound"
+    access                     = "Allow"
+    protocol                   = "Tcp"
+    source_port_range          = "*"
+    destination_port_ranges    = ["443"]
+    source_address_prefix      = "*"
+    destination_address_prefix = "AzureCloud"
+  }
+  security_rule {
+    name                       = "AllowGetSessionInformation"
+    priority                   = 130
+    direction                  = "Outbound"
+    access                     = "Allow"
+    protocol                   = "Tcp"
+    source_port_range          = "*"
+    destination_port_ranges    = ["80"]
+    source_address_prefix      = "*"
+    destination_address_prefix = "Internet"
+  }
+
+}
+
+resource "azurerm_subnet_network_security_group_association" "bastion_nsg_assoc" {
+  subnet_id                 = azurerm_subnet.azure_bastion.id
+  network_security_group_id = azurerm_network_security_group.bastion_nsg.id
+  depends_on = [
+    azurerm_bastion_host.azure_bastion_instance
+  ]
+}
+
+
+resource "azurerm_bastion_host" "azure_bastion_instance" {
+  name                = "bas-${var.name}-${var.environment}"
+  location            = azurerm_resource_group.default.location
+  resource_group_name = azurerm_resource_group.default.name
+
+  ip_configuration {
+    name                 = "configuration"
+    subnet_id            = azurerm_subnet.azure_bastion.id
+    public_ip_address_id = azurerm_public_ip.azure_bastion.id
+  }
+}
+

quickstart/201-machine-learning-moderately-secure/dsvm.tf

@@ -0,0 +1,48 @@
+resource "azurerm_network_interface" "dsvm" {
+  name                = "nic-${var.dsvm_name}"
+  location            = azurerm_resource_group.default.location
+  resource_group_name = azurerm_resource_group.default.name
+
+  ip_configuration {
+    name                          = "configuration"
+    subnet_id                     = azurerm_subnet.snet-dsvm.id
+    private_ip_address_allocation = "Dynamic"
+  }
+}
+
+resource "azurerm_windows_virtual_machine" "dsvm" {
+  name                = var.dsvm_name
+  location            = azurerm_resource_group.default.location
+  resource_group_name = azurerm_resource_group.default.name
+  network_interface_ids = [
+    azurerm_network_interface.dsvm.id
+  ]
+  size = "Standard_DS3_v2"
+
+  source_image_reference {
+    publisher = "microsoft-dsvm"
+    offer     = "dsvm-win-2019"
+    sku       = "server-2019"
+    version   = "latest"
+  }
+
+  os_disk {
+    name                 = "osdisk-${var.dsvm_name}"
+    caching              = "ReadWrite"
+    storage_account_type = "Premium_LRS"
+  }
+
+  identity {
+    type = "SystemAssigned"
+  }
+  computer_name  = var.dsvm_name
+  admin_username = var.dsvm_admin_username
+  admin_password = var.dsvm_host_password
+
+  provision_vm_agent = true
+
+  timeouts {
+    create = "60m"
+    delete = "2h"
+  }
+}

quickstart/201-machine-learning-moderately-secure/main.tf

@@ -1,5 +1,5 @@
 terraform {
-  required_version = ">=0.15.0"
+  required_version = ">=1.0"
 
   required_providers {
     azurerm = {

quickstart/201-machine-learning-moderately-secure/network.tf

@@ -30,6 +30,21 @@ resource "azurerm_subnet" "snet-workspace" {
   enforce_private_link_endpoint_network_policies = true
 }
 
+resource "azurerm_subnet" "snet-dsvm" {
+  name                                           = "snet-dsvm"
+  resource_group_name                            = azurerm_resource_group.default.name
+  virtual_network_name                           = azurerm_virtual_network.default.name
+  address_prefixes                               = var.dsvm_subnet_address_space
+  enforce_private_link_endpoint_network_policies = true
+}
+
+resource "azurerm_subnet" "azure_bastion" {
+  name                 = "AzureBastionSubnet"
+  resource_group_name  = azurerm_resource_group.default.name
+  virtual_network_name = azurerm_virtual_network.default.name
+  address_prefixes     = var.bastion_subnet_address_space
+}
+
 # Private DNS Zones
 resource "azurerm_private_dns_zone" "dnsvault" {
   name                = "privatelink.vaultcore.azure.net"

quickstart/201-machine-learning-moderately-secure/readme.md

@@ -6,13 +6,15 @@ and its associated resources including Azure Key Vault, Azure Storage, Azure App
 In addition to these core services, this configuration specifies any networking components that are required to set up Azure Machine Learning
 for private network connectivity using [Azure Private Link](https://docs.microsoft.com/en-us/azure/private-link/). 
 
-This configuration describes the minimal set of resources you require to get started with Azure Machine Learning in a network-isolated set-up. This configuration creates new network components. If you want to reuse existing network components, see [202 example](../201-machine-learning-moderately-secure/readme.md).
+This configuration describes the minimal set of resources you require to get started with Azure Machine Learning in a network-isolated set-up. This configuration creates new network components. Use Azure Bastion to securely connect to the Windows Data Science Virtual Machine. If you want to reuse existing network components, see [202 example](../202-machine-learning-moderately-secure-existing-VNet/readme.md).
 
 ## Resources
 
 | Terraform Resource Type | Description |
 | - | - |
 | `azurerm_resource_group` | The resource group all resources get deployed into |
+| `azurerm_bastion_host` | An Azure Bastion Instance to securely RDP/SSH into Virtual Machines deployed into the Virtual Network |
+| `azurerm_windows_virtual_machine` | A Windows Data Science Virtual Machine used for connecting to the Azure Machine Learning workspace |
 | `azurerm_application_insights` | An Azure Application Insights instance associated to the Azure Machine Learning workspace |
 | `azurerm_key_vault` | An Azure Key Vault instance associated to the Azure Machine Learning workspace |
 | `azurerm_storage_account` | An Azure Storage instance associated to the Azure Machine Learning workspace |
@@ -39,6 +41,9 @@ This configuration describes the minimal set of resources you require to get sta
 | aks_subnet_address_space | Address space of the aks subnet | ["10.0.2.0/23"] |
 | ml_subnet_address_space | Address space of the ML workspace subnet | ["10.0.0.0/24"] |
 | image_build_compute_name | Name of the compute cluster to be created and configured for building docker images (Azure ML Environments) | image-builder |
+| dsvm_name | Name of the Windows Data Science VM resource | vmdsvm01 |
+| dsvm_admin_username | Admin username of the Windows Data Science VM | azureadmin |
+| dsvm_host_password | Password for the admin username of the Data Science VM | - |
 
 
 ## Usage

quickstart/201-machine-learning-moderately-secure/variables.tf

@@ -38,9 +38,38 @@ variable "ml_subnet_address_space" {
   description = "Address space of the ML workspace subnet"
   default     = ["10.0.0.0/24"]
 }
+variable "dsvm_subnet_address_space" {
+  type        = list(string)
+  description = "Address space of the DSVM subnet"
+  default     = ["10.0.4.0/24"]
+}
+
+variable "bastion_subnet_address_space" {
+  type        = list(string)
+  description = "Address space of the bastion subnet"
+  default     = ["10.0.5.0/24"]
+}
 
 variable "image_build_compute_name" {
   type        = string
   description = "Name of the compute cluster to be created and set to build docker images"
   default     = "image-builder"
 }
+
+# DSVM Variables
+variable "dsvm_name" {
+  type        = string
+  description = "Name of the Data Science VM"
+  default     = "vmdsvm01"
+}
+variable "dsvm_admin_username" {
+  type        = string
+  description = "Admin username of the Data Science VM"
+  default     = "azureadmin"
+}
+
+variable "dsvm_host_password" {
+  type        = string
+  description = "Password for the admin username of the Data Science VM"
+  sensitive   = true
+}

quickstart/201-machine-learning-moderately-secure/workspace.tf

@@ -63,6 +63,13 @@ resource "azurerm_machine_learning_workspace" "default" {
   # Args of use when using an Azure Private Link configuration
   public_network_access_enabled = false
   image_build_compute_name      = var.image_build_compute_name
+  depends_on = [
+    azurerm_private_endpoint.kv_ple,
+    azurerm_private_endpoint.st_ple_blob,
+    azurerm_private_endpoint.storage_ple_file,
+    azurerm_private_endpoint.cr_ple,
+    azurerm_subnet.snet-training
+  ]
 
 }
 

quickstart/202-machine-learning-moderately-secure-existing-VNet/main.tf

@@ -1,5 +1,5 @@
 terraform {
-  required_version = ">=0.15.0"
+  required_version = ">=1.0"
 
   required_providers {
     azurerm = {

quickstart/301-machine-learning-hub-spoke-secure/azure-firewall.tf

@@ -0,0 +1,490 @@
+# Generate random string for unique firewall diagnostic name
+resource "random_string" "fw_diag_prefix" {
+  length  = 8
+  upper   = false
+  special = false
+  number  = false
+}
+resource "azurerm_ip_group" "ip_group_hub" {
+  name                = "hub-ipgroup"
+  location            = azurerm_resource_group.hub_rg.location
+  resource_group_name = azurerm_resource_group.hub_rg.name
+  cidrs               = var.vnet_hub_address_space
+}
+
+resource "azurerm_ip_group" "ip_group_spoke" {
+  name                = "mlw-spoke-ipgroup"
+  location            = azurerm_resource_group.hub_rg.location
+  resource_group_name = azurerm_resource_group.hub_rg.name
+  cidrs               = var.vnet_address_space
+}
+
+resource "azurerm_ip_group" "ip_group_dsvm_subnet" {
+  name                = "dsvm-subnet-ipgroup"
+  location            = azurerm_resource_group.hub_rg.location
+  resource_group_name = azurerm_resource_group.hub_rg.name
+  cidrs               = var.jumphost_subnet_address_space
+}
+
+resource "azurerm_public_ip" "azure_firewall" {
+  name                = "pip-azfw"
+  location            = azurerm_resource_group.default.location
+  resource_group_name = azurerm_resource_group.hub_rg.name
+  allocation_method   = "Static"
+  sku                 = "Standard"
+}
+
+resource "azurerm_firewall_policy" "base_policy" {
+  name                = "afwp-base-01"
+  resource_group_name = azurerm_resource_group.hub_rg.name
+  location            = azurerm_resource_group.default.location
+  dns {
+    proxy_enabled = true
+  }
+
+}
+resource "azurerm_firewall" "azure_firewall_instance" {
+  name                = "afw-${var.name}-${var.environment}"
+  location            = azurerm_resource_group.default.location
+  resource_group_name = azurerm_resource_group.hub_rg.name
+  firewall_policy_id  = azurerm_firewall_policy.base_policy.id
+
+  ip_configuration {
+    name                 = "configuration"
+    subnet_id            = azurerm_subnet.azure_firewall.id
+    public_ip_address_id = azurerm_public_ip.azure_firewall.id
+  }
+
+  timeouts {
+    create = "60m"
+    delete = "2h"
+  }
+  depends_on = [
+    azurerm_public_ip.azure_firewall,
+    azurerm_subnet.azure_firewall,
+    azurerm_firewall_policy_rule_collection_group.azure_firewall_rules_collection
+  ]
+}
+
+resource "azurerm_monitor_diagnostic_setting" "azure_firewall_instance" {
+  name                       = "diagnostics-${var.name}-${var.environment}-${random_string.fw_diag_prefix.result}"
+  target_resource_id         = azurerm_firewall.azure_firewall_instance.id
+  log_analytics_workspace_id = azurerm_log_analytics_workspace.default.id
+
+  log {
+    category = "AzureFirewallApplicationRule"
+    enabled  = true
+
+    retention_policy {
+      enabled = false
+    }
+  }
+  log {
+    category = "AzureFirewallNetworkRule"
+    enabled  = true
+
+    retention_policy {
+      enabled = false
+    }
+  }
+  log {
+    category = "AzureFirewallDnsProxy"
+    enabled  = true
+
+    retention_policy {
+      enabled = false
+    }
+  }
+
+
+  metric {
+    category = "AllMetrics"
+
+    retention_policy {
+      enabled = false
+    }
+  }
+
+}
+
+resource "azurerm_firewall_policy_rule_collection_group" "azure_firewall_rules_collection" {
+  name               = "afwp-base-rule-collection-group"
+  firewall_policy_id = azurerm_firewall_policy.base_policy.id
+  priority           = 100
+
+  application_rule_collection {
+    name     = "afwp-base-app-rule-collection"
+    priority = 200
+    action   = "Allow"
+
+    rule {
+      name = "dsvm-to-internet"
+      protocols {
+        type = "Https"
+        port = 443
+      }
+      protocols {
+        type = "Http"
+        port = 80
+      }
+      source_ip_groups  = [azurerm_ip_group.ip_group_dsvm_subnet.id]
+      destination_fqdns = ["*"]
+    }
+
+    rule {
+      name = "aks-service-tag"
+      protocols {
+        type = "Https"
+        port = 443
+      }
+      source_ip_groups      = [azurerm_ip_group.ip_group_spoke.id]
+      destination_fqdn_tags = ["AzureKubernetesService"]
+    }
+
+    rule {
+      name = "ubuntu-libraries"
+      protocols {
+        type = "Https"
+        port = 443
+      }
+      source_ip_groups  = [azurerm_ip_group.ip_group_spoke.id]
+      destination_fqdns = ["api.snapcraft.io", "motd.ubuntu.com", ]
+    }
+
+    rule {
+      name = "microsoft-crls"
+      protocols {
+        type = "Http"
+        port = 80
+      }
+      source_ip_groups = [azurerm_ip_group.ip_group_spoke.id]
+      destination_fqdns = ["crl.microsoft.com",
+        "mscrl.microsoft.com",
+        "crl3.digicert.com",
+      "ocsp.digicert.com"]
+    }
+
+    rule {
+      name = "github-rules"
+      protocols {
+        type = "Https"
+        port = 443
+      }
+      source_ip_groups  = [azurerm_ip_group.ip_group_spoke.id]
+      destination_fqdns = ["github.com"]
+    }
+
+    rule {
+      name = "raw.githubusercontent.com"
+      protocols {
+        type = "Https"
+        port = 443
+      }
+      protocols {
+        type = "Http"
+        port = 80
+      }
+      source_ip_groups  = [azurerm_ip_group.ip_group_spoke.id]
+      destination_fqdns = ["raw.githubusercontent.com"]
+    }
+
+    rule {
+      name = "microsoft-metrics-rules"
+      protocols {
+        type = "Https"
+        port = 443
+      }
+      source_ip_groups  = [azurerm_ip_group.ip_group_spoke.id]
+      destination_fqdns = ["*.prod.microsoftmetrics.com"]
+    }
+
+    rule {
+      name = "aks-acs-rules"
+      protocols {
+        type = "Https"
+        port = 443
+      }
+      source_ip_groups = [azurerm_ip_group.ip_group_spoke.id]
+      destination_fqdns = ["acs-mirror.azureedge.net",
+        "*.docker.io",
+        "production.cloudflare.docker.com",
+      "*.azurecr.io"]
+    }
+
+    rule {
+      name = "microsoft-login-rules"
+      protocols {
+        type = "Https"
+        port = 443
+      }
+      source_ip_groups  = [azurerm_ip_group.ip_group_spoke.id]
+      destination_fqdns = ["login.microsoftonline.com"]
+    }
+
+    rule {
+      name = "graph.windows.net"
+      protocols {
+        type = "Http"
+        port = 80
+      }
+      protocols {
+        type = "Https"
+        port = 443
+      }
+      source_ip_groups  = [azurerm_ip_group.ip_group_spoke.id]
+      destination_fqdns = ["graph.windows.net"]
+    }
+
+    rule {
+      name = "anaconda.com"
+      protocols {
+        type = "Http"
+        port = 80
+      }
+      protocols {
+        type = "Https"
+        port = 443
+      }
+      source_ip_groups  = [azurerm_ip_group.ip_group_spoke.id]
+      destination_fqdns = ["anaconda.com", "*.anaconda.com"]
+    }
+
+    rule {
+      name = "anaconda.org"
+      protocols {
+        type = "Http"
+        port = 80
+      }
+      protocols {
+        type = "Https"
+        port = 443
+      }
+      source_ip_groups  = [azurerm_ip_group.ip_group_spoke.id]
+      destination_fqdns = ["*.anaconda.org"]
+    }
+
+    rule {
+      name = "pypi.org"
+      protocols {
+        type = "Http"
+        port = 80
+      }
+      protocols {
+        type = "Https"
+        port = 443
+      }
+      source_ip_groups  = [azurerm_ip_group.ip_group_spoke.id]
+      destination_fqdns = ["pypi.org"]
+    }
+
+    rule {
+      name = "cloud.r-project.org"
+      protocols {
+        type = "Http"
+        port = 80
+      }
+      protocols {
+        type = "Https"
+        port = 443
+      }
+      source_ip_groups  = [azurerm_ip_group.ip_group_spoke.id]
+      destination_fqdns = ["cloud.r-project.org"]
+    }
+
+    rule {
+      name = "pytorch.org"
+      protocols {
+        type = "Http"
+        port = 80
+      }
+      protocols {
+        type = "Https"
+        port = 443
+      }
+      source_ip_groups  = [azurerm_ip_group.ip_group_spoke.id]
+      destination_fqdns = ["*pytorch.org"]
+    }
+
+    rule {
+      name = "tensorflow.org"
+      protocols {
+        type = "Http"
+        port = 80
+      }
+      protocols {
+        type = "Https"
+        port = 443
+      }
+      source_ip_groups  = [azurerm_ip_group.ip_group_spoke.id]
+      destination_fqdns = ["*.tensorflow.org"]
+    }
+
+    rule {
+      name = "update.code.visualstudio.com"
+      protocols {
+        type = "Http"
+        port = 80
+      }
+      protocols {
+        type = "Https"
+        port = 443
+      }
+      source_ip_groups  = [azurerm_ip_group.ip_group_spoke.id]
+      destination_fqdns = ["update.code.visualstudio.com", "*.vo.msecnd.net"]
+    }
+
+    rule {
+      name = "dc.applicationinsights.azure.com"
+      protocols {
+        type = "Http"
+        port = 80
+      }
+      protocols {
+        type = "Https"
+        port = 443
+      }
+      source_ip_groups  = [azurerm_ip_group.ip_group_spoke.id]
+      destination_fqdns = ["dc.applicationinsights.azure.com"]
+    }
+
+    rule {
+      name = "dc.applicationinsights.microsoft.com"
+      protocols {
+        type = "Http"
+        port = 80
+      }
+      protocols {
+        type = "Https"
+        port = 443
+      }
+      source_ip_groups  = [azurerm_ip_group.ip_group_spoke.id]
+      destination_fqdns = ["dc.applicationinsights.microsoft.com"]
+    }
+
+    rule {
+      name = "dc.services.visualstudio.com"
+      protocols {
+        type = "Http"
+        port = 80
+      }
+      protocols {
+        type = "Https"
+        port = 443
+      }
+      source_ip_groups  = [azurerm_ip_group.ip_group_spoke.id]
+      destination_fqdns = ["dc.services.visualstudio.com"]
+    }
+
+    rule {
+      name = "azureml-instances"
+      protocols {
+        type = "Http"
+        port = 80
+      }
+      protocols {
+        type = "Https"
+        port = 443
+      }
+      source_ip_groups  = [azurerm_ip_group.ip_group_spoke.id]
+      destination_fqdns = ["*.instances.azureml.net", "*.instances.azureml.ms"]
+    }
+  }
+
+  network_rule_collection {
+    name     = "afwp-base-network-rule-collection"
+    priority = 100
+    action   = "Allow"
+
+    rule {
+      name                  = "hub-to-spoke-rule"
+      protocols             = ["Any"]
+      source_ip_groups      = [azurerm_ip_group.ip_group_spoke.id, azurerm_ip_group.ip_group_hub.id]
+      destination_ip_groups = [azurerm_ip_group.ip_group_hub.id, azurerm_ip_group.ip_group_spoke.id]
+      destination_ports     = ["*"]
+    }
+
+    rule {
+      name                  = "aks-global-network-rule"
+      protocols             = ["TCP"]
+      source_ip_groups      = [azurerm_ip_group.ip_group_spoke.id]
+      destination_addresses = ["AzureCloud"]
+      destination_ports     = ["443", "9000"]
+    }
+
+    rule {
+      name                  = "aks-ntp-network-rule"
+      protocols             = ["UDP"]
+      source_ip_groups      = [azurerm_ip_group.ip_group_spoke.id]
+      destination_addresses = ["*"]
+      destination_ports     = ["123"]
+    }
+
+    rule {
+      name                  = "Azure-Active-Directory"
+      protocols             = ["TCP"]
+      source_ip_groups      = [azurerm_ip_group.ip_group_spoke.id]
+      destination_addresses = ["AzureActiveDirectory"]
+      destination_ports     = ["*"]
+    }
+
+    rule {
+      name                  = "Azure-Machine-Learning"
+      protocols             = ["TCP"]
+      source_ip_groups      = [azurerm_ip_group.ip_group_spoke.id]
+      destination_addresses = ["AzureMachineLearning"]
+      destination_ports     = ["443"]
+    }
+
+    rule {
+      name                  = "Azure-Resource-Manager"
+      protocols             = ["TCP"]
+      source_ip_groups      = [azurerm_ip_group.ip_group_spoke.id]
+      destination_addresses = ["AzureResourceManager"]
+      destination_ports     = ["443"]
+    }
+
+    rule {
+      name                  = "Azure-Storage"
+      protocols             = ["TCP"]
+      source_ip_groups      = [azurerm_ip_group.ip_group_spoke.id]
+      destination_addresses = ["Storage"]
+      destination_ports     = ["443"]
+    }
+
+    rule {
+      name                  = "Azure-Front-Door-Frontend"
+      protocols             = ["TCP"]
+      source_ip_groups      = [azurerm_ip_group.ip_group_spoke.id]
+      destination_addresses = ["AzureFrontDoor.Frontend", "AzureFrontDoor.FirstParty"]
+      destination_ports     = ["443"]
+    }
+
+    rule {
+      name                  = "Azure-Container-Registry"
+      protocols             = ["TCP"]
+      source_ip_groups      = [azurerm_ip_group.ip_group_spoke.id]
+      destination_addresses = ["AzureContainerRegistry"]
+      destination_ports     = ["443"]
+    }
+
+    rule {
+      name                  = "Azure-Key-Vault"
+      protocols             = ["TCP"]
+      source_ip_groups      = [azurerm_ip_group.ip_group_spoke.id]
+      destination_addresses = ["AzureKeyVault"]
+      destination_ports     = ["443"]
+    }
+
+    rule {
+      name                  = "Microsoft-Container-Registry"
+      protocols             = ["TCP"]
+      source_ip_groups      = [azurerm_ip_group.ip_group_spoke.id]
+      destination_addresses = ["MicrosoftContainerRegistry"]
+      destination_ports     = ["443"]
+    }
+  }
+  depends_on = [
+    azurerm_ip_group.ip_group_hub,
+    azurerm_ip_group.ip_group_spoke
+  ]
+}

quickstart/301-machine-learning-hub-spoke-secure/bastion.tf

@@ -0,0 +1,126 @@
+resource "azurerm_public_ip" "azure_bastion" {
+  name                = "pip-azure-bastion"
+  location            = azurerm_resource_group.default.location
+  resource_group_name = azurerm_resource_group.hub_rg.name
+  allocation_method   = "Static"
+  sku                 = "Standard"
+}
+
+resource "azurerm_network_security_group" "bastion_nsg" {
+  name                = "nsg-bastion"
+  location            = azurerm_resource_group.default.location
+  resource_group_name = azurerm_resource_group.hub_rg.name
+
+  security_rule {
+    name                       = "AllowHTTPSInbound"
+    priority                   = 100
+    direction                  = "Inbound"
+    access                     = "Allow"
+    protocol                   = "Tcp"
+    source_port_range          = "*"
+    destination_port_range     = "443"
+    source_address_prefix      = "Internet"
+    destination_address_prefix = "*"
+  }
+  security_rule {
+    name                       = "AllowGatewayManagerInbound"
+    priority                   = 200
+    direction                  = "Inbound"
+    access                     = "Allow"
+    protocol                   = "Tcp"
+    source_port_range          = "*"
+    destination_port_range     = "443"
+    source_address_prefix      = "GatewayManager"
+    destination_address_prefix = "*"
+  }
+  security_rule {
+    name                       = "AllowAzureLBInbound"
+    priority                   = 300
+    direction                  = "Inbound"
+    access                     = "Allow"
+    protocol                   = "Tcp"
+    source_port_range          = "*"
+    destination_port_range     = "443"
+    source_address_prefix      = "AzureLoadBalancer"
+    destination_address_prefix = "*"
+  }
+  security_rule {
+    name                       = "AllowBastionHostCommunication"
+    priority                   = 400
+    direction                  = "Inbound"
+    access                     = "Allow"
+    protocol                   = "*"
+    source_port_range          = "*"
+    destination_port_ranges    = ["5701", "8080"]
+    source_address_prefix      = "VirtualNetwork"
+    destination_address_prefix = "VirtualNetwork"
+  }
+  security_rule {
+    name                       = "AllowRdpSshOutbound"
+    priority                   = 100
+    direction                  = "Outbound"
+    access                     = "Allow"
+    protocol                   = "Tcp"
+    source_port_range          = "*"
+    destination_port_ranges    = ["22", "3389"]
+    source_address_prefix      = "*"
+    destination_address_prefix = "VirtualNetwork"
+  }
+  security_rule {
+    name                       = "AllowBastionHostCommunicationOutbound"
+    priority                   = 110
+    direction                  = "Outbound"
+    access                     = "Allow"
+    protocol                   = "Tcp"
+    source_port_range          = "*"
+    destination_port_ranges    = ["5701", "8080"]
+    source_address_prefix      = "VirtualNetwork"
+    destination_address_prefix = "VirtualNetwork"
+  }
+  security_rule {
+    name                       = "AllowAzureCloudOutbound"
+    priority                   = 120
+    direction                  = "Outbound"
+    access                     = "Allow"
+    protocol                   = "Tcp"
+    source_port_range          = "*"
+    destination_port_ranges    = ["443"]
+    source_address_prefix      = "*"
+    destination_address_prefix = "AzureCloud"
+  }
+  security_rule {
+    name                       = "AllowGetSessionInformation"
+    priority                   = 130
+    direction                  = "Outbound"
+    access                     = "Allow"
+    protocol                   = "Tcp"
+    source_port_range          = "*"
+    destination_port_ranges    = ["80"]
+    source_address_prefix      = "*"
+    destination_address_prefix = "Internet"
+  }
+
+}
+
+resource "azurerm_subnet_network_security_group_association" "bastion_nsg_assoc" {
+  subnet_id                 = azurerm_subnet.azure_bastion.id
+  network_security_group_id = azurerm_network_security_group.bastion_nsg.id
+  depends_on = [
+    azurerm_bastion_host.azure_bastion_instance,
+    azurerm_subnet_network_security_group_association.jumphost_nsg_assoc
+  ]
+}
+
+
+resource "azurerm_bastion_host" "azure_bastion_instance" {
+  name                = "bas-${var.name}-${var.environment}"
+  location            = azurerm_resource_group.default.location
+  resource_group_name = azurerm_resource_group.hub_rg.name
+
+  ip_configuration {
+    name                 = "configuration"
+    subnet_id            = azurerm_subnet.azure_bastion.id
+    public_ip_address_id = azurerm_public_ip.azure_bastion.id
+  }
+}
+

quickstart/301-machine-learning-hub-spoke-secure/compute.tf

@@ -0,0 +1,44 @@
+# Generate random string for unique compute instance name
+resource "random_string" "ci_prefix" {
+  length  = 8
+  upper   = false
+  special = false
+  number  = false
+}
+
+# Compute instance
+resource "azurerm_machine_learning_compute_instance" "compute_instance" {
+  name                          = "${random_string.ci_prefix.result}instance"
+  location                      = azurerm_resource_group.default.location
+  machine_learning_workspace_id = azurerm_machine_learning_workspace.default.id
+  virtual_machine_size          = "STANDARD_DS2_V2"
+  subnet_resource_id            = azurerm_subnet.snet-training.id
+
+  depends_on = [
+    azurerm_private_endpoint.mlw_ple
+  ]
+}
+
+# Compute cluster
+resource "azurerm_machine_learning_compute_cluster" "compute" {
+  name                          = "cpu-cluster"
+  location                      = azurerm_resource_group.default.location
+  machine_learning_workspace_id = azurerm_machine_learning_workspace.default.id
+  vm_priority                   = "Dedicated"
+  vm_size                       = "STANDARD_DS2_V2"
+  subnet_resource_id            = azurerm_subnet.snet-training.id
+
+  identity {
+    type = "SystemAssigned"
+  }
+
+  scale_settings {
+    min_node_count                       = 0
+    max_node_count                       = 3
+    scale_down_nodes_after_idle_duration = "PT15M" # 15 minutes
+  }
+  depends_on = [
+    azurerm_private_endpoint.mlw_ple
+  ]
+
+}

quickstart/301-machine-learning-hub-spoke-secure/dsvm.tf

@@ -0,0 +1,48 @@
+resource "azurerm_network_interface" "dsvm" {
+  name                = "nic-${var.dsvm_name}"
+  location            = azurerm_resource_group.default.location
+  resource_group_name = azurerm_resource_group.default.name
+
+  ip_configuration {
+    name                          = "configuration"
+    subnet_id                     = azurerm_subnet.snet-jumphost.id
+    private_ip_address_allocation = "Dynamic"
+  }
+}
+
+resource "azurerm_windows_virtual_machine" "dsvm" {
+  name                = var.dsvm_name
+  location            = azurerm_resource_group.default.location
+  resource_group_name = azurerm_resource_group.default.name
+  network_interface_ids = [
+    azurerm_network_interface.dsvm.id
+  ]
+  size = "Standard_DS3_v2"
+
+  source_image_reference {
+    publisher = "microsoft-dsvm"
+    offer     = "dsvm-win-2019"
+    sku       = "server-2019"
+    version   = "latest"
+  }
+
+  os_disk {
+    name                 = "osdisk-${var.dsvm_name}"
+    caching              = "ReadWrite"
+    storage_account_type = "Premium_LRS"
+  }
+
+  identity {
+    type = "SystemAssigned"
+  }
+  computer_name  = var.dsvm_name
+  admin_username = var.dsvm_admin_username
+  admin_password = var.dsvm_host_password
+
+  provision_vm_agent = true
+
+  timeouts {
+    create = "60m"
+    delete = "2h"
+  }
+}

quickstart/301-machine-learning-hub-spoke-secure/main.tf

@@ -0,0 +1,28 @@
+terraform {
+  required_version = ">=1.0"
+
+  required_providers {
+    azurerm = {
+      source  = "hashicorp/azurerm"
+      version = "=2.79.1"
+    }
+  }
+}
+
+provider "azurerm" {
+  features {}
+}
+
+data "azurerm_client_config" "current" {}
+
+resource "azurerm_resource_group" "default" {
+  name     = "rg-${var.name}-${var.environment}"
+  location = var.location
+}
+
+#Hub Resource Group
+resource "azurerm_resource_group" "hub_rg" {
+  name     = "rg-hub-${var.name}-${var.environment}"
+  location = var.location
+
+}

quickstart/301-machine-learning-hub-spoke-secure/monitor.tf

@@ -0,0 +1,7 @@
+resource "azurerm_log_analytics_workspace" "default" {
+  name                = "log-${var.name}-${var.environment}"
+  location            = azurerm_resource_group.default.location
+  resource_group_name = azurerm_resource_group.hub_rg.name
+  sku                 = "PerGB2018"
+  retention_in_days   = 30
+}

quickstart/301-machine-learning-hub-spoke-secure/network-hub.tf

@@ -0,0 +1,176 @@
+#Hub Virtual Network
+
+resource "azurerm_virtual_network" "hub" {
+  name                = "vnet-hub-${var.name}-${var.environment}"
+  address_space       = var.vnet_hub_address_space
+  location            = azurerm_resource_group.default.location
+  resource_group_name = azurerm_resource_group.hub_rg.name
+}
+
+resource "azurerm_subnet" "snet-jumphost" {
+  name                 = "snet-jumphost"
+  resource_group_name  = azurerm_resource_group.hub_rg.name
+  virtual_network_name = azurerm_virtual_network.hub.name
+  address_prefixes     = var.jumphost_subnet_address_space
+
+}
+
+
+resource "azurerm_subnet" "azure_bastion" {
+  name                 = "AzureBastionSubnet"
+  resource_group_name  = azurerm_resource_group.hub_rg.name
+  virtual_network_name = azurerm_virtual_network.hub.name
+  address_prefixes     = var.bastion_subnet_address_space
+
+}
+resource "azurerm_subnet" "azure_firewall" {
+  name                 = "AzureFirewallSubnet"
+  resource_group_name  = azurerm_resource_group.hub_rg.name
+  virtual_network_name = azurerm_virtual_network.hub.name
+  address_prefixes     = var.firewall_subnet_address_space
+
+}
+
+#Vnet Peering
+
+resource "azurerm_virtual_network_peering" "direction1" {
+  name                         = "${azurerm_resource_group.hub_rg.name}-to-${azurerm_resource_group.default.name}"
+  resource_group_name          = azurerm_resource_group.hub_rg.name
+  virtual_network_name         = azurerm_virtual_network.hub.name
+  remote_virtual_network_id    = azurerm_virtual_network.default.id
+  allow_virtual_network_access = true
+  allow_forwarded_traffic      = false
+  allow_gateway_transit        = false
+  use_remote_gateways          = false
+  depends_on = [
+    azurerm_virtual_network.hub,
+    azurerm_virtual_network.default
+  ]
+
+}
+
+resource "azurerm_virtual_network_peering" "direction2" {
+  name                         = "${azurerm_resource_group.default.name}-to-${azurerm_resource_group.hub_rg.name}"
+  resource_group_name          = azurerm_resource_group.default.name
+  virtual_network_name         = azurerm_virtual_network.default.name
+  remote_virtual_network_id    = azurerm_virtual_network.hub.id
+  allow_virtual_network_access = true
+  allow_forwarded_traffic      = false
+  allow_gateway_transit        = false
+  use_remote_gateways          = false
+  depends_on = [
+    azurerm_virtual_network.hub,
+    azurerm_virtual_network.default
+  ]
+
+}
+
+# Private DNS Zones
+resource "azurerm_private_dns_zone" "dnsvault" {
+  name                = "privatelink.vaultcore.azure.net"
+  resource_group_name = azurerm_resource_group.hub_rg.name
+}
+
+resource "azurerm_private_dns_zone_virtual_network_link" "vnetlinkvault" {
+  name                  = "dnsvaultlink"
+  resource_group_name   = azurerm_resource_group.hub_rg.name
+  private_dns_zone_name = azurerm_private_dns_zone.dnsvault.name
+  virtual_network_id    = azurerm_virtual_network.hub.id
+}
+
+resource "azurerm_private_dns_zone" "dnsstorageblob" {
+  name                = "privatelink.blob.core.windows.net"
+  resource_group_name = azurerm_resource_group.hub_rg.name
+}
+
+resource "azurerm_private_dns_zone_virtual_network_link" "vnetlinkblob" {
+  name                  = "dnsblobstoragelink"
+  resource_group_name   = azurerm_resource_group.hub_rg.name
+  private_dns_zone_name = azurerm_private_dns_zone.dnsstorageblob.name
+  virtual_network_id    = azurerm_virtual_network.hub.id
+}
+
+resource "azurerm_private_dns_zone" "dnsstoragefile" {
+  name                = "privatelink.file.core.windows.net"
+  resource_group_name = azurerm_resource_group.hub_rg.name
+}
+
+resource "azurerm_private_dns_zone_virtual_network_link" "vnetlinkfile" {
+  name                  = "dnsfilestoragelink"
+  resource_group_name   = azurerm_resource_group.hub_rg.name
+  private_dns_zone_name = azurerm_private_dns_zone.dnsstoragefile.name
+  virtual_network_id    = azurerm_virtual_network.hub.id
+}
+
+resource "azurerm_private_dns_zone" "dnscontainerregistry" {
+  name                = "privatelink.azurecr.io"
+  resource_group_name = azurerm_resource_group.hub_rg.name
+}
+
+resource "azurerm_private_dns_zone_virtual_network_link" "vnetlinkcr" {
+  name                  = "dnscrlink"
+  resource_group_name   = azurerm_resource_group.hub_rg.name
+  private_dns_zone_name = azurerm_private_dns_zone.dnscontainerregistry.name
+  virtual_network_id    = azurerm_virtual_network.hub.id
+}
+
+resource "azurerm_private_dns_zone" "dnsazureml" {
+  name                = "privatelink.api.azureml.ms"
+  resource_group_name = azurerm_resource_group.hub_rg.name
+}
+
+resource "azurerm_private_dns_zone_virtual_network_link" "vnetlinkml" {
+  name                  = "dnsazuremllink"
+  resource_group_name   = azurerm_resource_group.hub_rg.name
+  private_dns_zone_name = azurerm_private_dns_zone.dnsazureml.name
+  virtual_network_id    = azurerm_virtual_network.hub.id
+}
+
+resource "azurerm_private_dns_zone" "dnsnotebooks" {
+  name                = "privatelink.notebooks.azure.net"
+  resource_group_name = azurerm_resource_group.hub_rg.name
+}
+
+resource "azurerm_private_dns_zone_virtual_network_link" "vnetlinknbs" {
+  name                  = "dnsnotebookslink"
+  resource_group_name   = azurerm_resource_group.hub_rg.name
+  private_dns_zone_name = azurerm_private_dns_zone.dnsnotebooks.name
+  virtual_network_id    = azurerm_virtual_network.hub.id
+}
+
+# NSG for jump_host Subnet
+
+resource "azurerm_network_security_group" "jump_host" {
+  name                = "nsg-jumphost-subnet"
+  location            = azurerm_resource_group.hub_rg.location
+  resource_group_name = azurerm_resource_group.hub_rg.name
+}
+
+resource "azurerm_subnet_network_security_group_association" "jumphost_nsg_assoc" {
+  subnet_id                 = azurerm_subnet.snet-jumphost.id
+  network_security_group_id = azurerm_network_security_group.jump_host.id
+  depends_on = [
+    azurerm_network_interface.dsvm
+  ]
+}
+
+# Route Table for Jump host subnet
+resource "azurerm_route_table" "jumphost_rt" {
+  name                = "rt-jumphost"
+  location            = azurerm_resource_group.default.location
+  resource_group_name = azurerm_resource_group.default.name
+}
+
+resource "azurerm_route" "jumphost-fw-route" {
+  name                   = "udr-Default"
+  resource_group_name    = azurerm_resource_group.default.name
+  route_table_name       = azurerm_route_table.jumphost_rt.name
+  address_prefix         = "0.0.0.0/0"
+  next_hop_type          = "VirtualAppliance"
+  next_hop_in_ip_address = azurerm_firewall.azure_firewall_instance.ip_configuration[0].private_ip_address
+}
+
+resource "azurerm_subnet_route_table_association" "rt-jumphost-link" {
+  subnet_id      = azurerm_subnet.snet-jumphost.id
+  route_table_id = azurerm_route_table.jumphost_rt.id
+}

quickstart/301-machine-learning-hub-spoke-secure/network-spoke.tf

@@ -0,0 +1,144 @@
+# Virtual Network
+resource "azurerm_virtual_network" "default" {
+  name                = "vnet-${var.name}-${var.environment}"
+  address_space       = var.vnet_address_space
+  location            = azurerm_resource_group.default.location
+  resource_group_name = azurerm_resource_group.default.name
+  dns_servers         = [azurerm_firewall.azure_firewall_instance.ip_configuration[0].private_ip_address]
+  depends_on = [
+    azurerm_virtual_network.hub,
+    azurerm_firewall.azure_firewall_instance
+  ]
+}
+
+resource "azurerm_subnet" "snet-training" {
+  name                                           = "snet-training"
+  resource_group_name                            = azurerm_resource_group.default.name
+  virtual_network_name                           = azurerm_virtual_network.default.name
+  address_prefixes                               = var.training_subnet_address_space
+  enforce_private_link_endpoint_network_policies = true
+}
+
+resource "azurerm_subnet" "snet-aks" {
+  name                                           = "snet-aks"
+  resource_group_name                            = azurerm_resource_group.default.name
+  virtual_network_name                           = azurerm_virtual_network.default.name
+  address_prefixes                               = var.aks_subnet_address_space
+  enforce_private_link_endpoint_network_policies = true
+}
+
+resource "azurerm_subnet" "snet-workspace" {
+  name                                           = "snet-workspace"
+  resource_group_name                            = azurerm_resource_group.default.name
+  virtual_network_name                           = azurerm_virtual_network.default.name
+  address_prefixes                               = var.ml_subnet_address_space
+  enforce_private_link_endpoint_network_policies = true
+}
+
+# Network Security Groups
+
+resource "azurerm_network_security_group" "nsg-training" {
+  name                = "nsg-training"
+  location            = azurerm_resource_group.default.location
+  resource_group_name = azurerm_resource_group.default.name
+
+  security_rule {
+    name                       = "BatchNodeManagement"
+    priority                   = 100
+    direction                  = "Inbound"
+    access                     = "Allow"
+    protocol                   = "Tcp"
+    source_port_range          = "*"
+    destination_port_range     = "29876-29877"
+    source_address_prefix      = "BatchNodeManagement"
+    destination_address_prefix = "*"
+  }
+
+  security_rule {
+    name                       = "AzureMachineLearning"
+    priority                   = 110
+    direction                  = "Inbound"
+    access                     = "Allow"
+    protocol                   = "Tcp"
+    source_port_range          = "*"
+    destination_port_range     = "44224"
+    source_address_prefix      = "AzureMachineLearning"
+    destination_address_prefix = "*"
+  }
+}
+
+resource "azurerm_subnet_network_security_group_association" "nsg-training-link" {
+  subnet_id                 = azurerm_subnet.snet-training.id
+  network_security_group_id = azurerm_network_security_group.nsg-training.id
+}
+
+resource "azurerm_network_security_group" "nsg-aks" {
+  name                = "nsg-aks"
+  location            = azurerm_resource_group.default.location
+  resource_group_name = azurerm_resource_group.default.name
+}
+
+resource "azurerm_subnet_network_security_group_association" "nsg-aks-link" {
+  subnet_id                 = azurerm_subnet.snet-aks.id
+  network_security_group_id = azurerm_network_security_group.nsg-aks.id
+}
+
+# User Defined Routes
+
+# UDR for compute instance and compute clusters
+resource "azurerm_route_table" "rt-training" {
+  name                = "rt-training"
+  location            = azurerm_resource_group.default.location
+  resource_group_name = azurerm_resource_group.default.name
+}
+
+resource "azurerm_route" "training-Internet-Route" {
+  name                   = "udr-Default"
+  resource_group_name    = azurerm_resource_group.default.name
+  route_table_name       = azurerm_route_table.rt-training.name
+  address_prefix         = "0.0.0.0/0"
+  next_hop_type          = "VirtualAppliance"
+  next_hop_in_ip_address = azurerm_firewall.azure_firewall_instance.ip_configuration[0].private_ip_address
+}
+
+resource "azurerm_route" "training-AzureMLRoute" {
+  name                = "udr-AzureML"
+  resource_group_name = azurerm_resource_group.default.name
+  route_table_name    = azurerm_route_table.rt-training.name
+  address_prefix      = "AzureMachineLearning"
+  next_hop_type       = "Internet"
+}
+
+resource "azurerm_route" "training-BatchRoute" {
+  name                = "udr-Batch"
+  resource_group_name = azurerm_resource_group.default.name
+  route_table_name    = azurerm_route_table.rt-training.name
+  address_prefix      = "BatchNodeManagement"
+  next_hop_type       = "Internet"
+}
+
+resource "azurerm_subnet_route_table_association" "rt-training-link" {
+  subnet_id      = azurerm_subnet.snet-training.id
+  route_table_id = azurerm_route_table.rt-training.id
+}
+
+# Inferencing (AKS) Route
+resource "azurerm_route_table" "rt-aks" {
+  name                = "rt-aks"
+  location            = azurerm_resource_group.default.location
+  resource_group_name = azurerm_resource_group.default.name
+}
+
+resource "azurerm_route" "aks-default-Route" {
+  name                   = "udr-Default"
+  resource_group_name    = azurerm_resource_group.default.name
+  route_table_name       = azurerm_route_table.rt-aks.name
+  address_prefix         = "0.0.0.0/0"
+  next_hop_type          = "VirtualAppliance"
+  next_hop_in_ip_address = azurerm_firewall.azure_firewall_instance.ip_configuration[0].private_ip_address
+}
+
+resource "azurerm_subnet_route_table_association" "rt-aks-link" {
+  subnet_id      = azurerm_subnet.snet-aks.id
+  route_table_id = azurerm_route_table.rt-aks.id
+}

quickstart/301-machine-learning-hub-spoke-secure/readme.md

@@ -0,0 +1,73 @@
+# Azure Machine Learning workspace (Secure Hub and Spoke with Firewall)
+
+This deployment configuration specifies an [Azure Machine Learning workspace](https://docs.microsoft.com/en-us/azure/machine-learning/concept-workspace), 
+and its associated resources including Azure Key Vault, Azure Storage, Azure Application Insights and Azure Container Registry.
+
+In addition to these core services, this configuration specifies any networking components that are required to set up Azure Machine Learning
+for private network connectivity using [Azure Private Link](https://docs.microsoft.com/en-us/azure/private-link/). 
+
+This configuration describes the full set of resources you require to get started with Azure Machine Learning in a network-isolated set-up. This configuration creates new network components. Use Azure Bastion to securely connect to the Windows Data Science Virtual Machine If you want to reuse existing network components, see [202 example](../202-machine-learning-moderately-secure-existing-VNet/readme.md).
+
+## Resources
+
+| Terraform Resource Type | Description |
+| - | - |
+| `azurerm_resource_group` | The resource group all resources get deployed into |
+| `azurerm_bastion_host` | An Azure Bastion Instance to securely RDP/SSH into Virtual Machines deployed into the Virtual Network |
+| `azurerm_windows_virtual_machine` | A Windows Data Science Virtual Machine used for connecting to the Azure Machine Learning workspace |
+| `azurerm_application_insights` | An Azure Application Insights instance associated to the Azure Machine Learning workspace |
+| `azurerm_log_analytics_workspace` | A Log analytics workspace used for Azure Firewall logs and to also host the Workspace-based Application Insights |
+| `azurerm_key_vault` | An Azure Key Vault instance associated to the Azure Machine Learning workspace |
+| `azurerm_storage_account` | An Azure Storage instance associated to the Azure Machine Learning workspace |
+| `azurerm_container_registry` | An Azure Container Registry instance associated to the Azure Machine Learning workspace |
+| `azurerm_machine_learning_workspace` | An Azure Machine Learning workspace instance |
+| `azurerm_virtual_network` | An Azure Machine Learning workspace instance |
+| `azurerm_subnet` | An Azure Machine Learning workspace instance |
+| `azurerm_private_dns_zone` | Private DNS Zones for FQDNs required for Azure Machine Learning and associated resources |
+| `azurerm_private_dns_zone_virtual_network_link` | Virtual network links of the Private DNS Zones to the virtual network resource |
+| `azurerm_private_endpoint` | Private Endpoints for the Azure Machine Learning workspace and associated resources |
+| `azurerm_machine_learning_compute_instance` | An Azure Machine Learning compute instance a single-node managed compute |
+| `azurerm_machine_learning_compute_cluster` | An Azure Machine Learning compute cluster as multi-node shared and managed compute |
+| `azurerm_network_security_group` | Network security group with required inbound and outbound rules for Azure Machine Learning |
+| `azurerm_firewall` | An Azure firewall instance used for egress traffic on the Virtual Network |
+| `azurerm_public_ip` | A public IP resource used for the Azure Firewall |
+
+## Variables
+
+| Name | Description | Default |
+|-|-|-|
+| name | Name of the deployment | - |
+| environment | The deployment environment name (used for pre- and postfixing resource names) | dev  |
+| location | The Azure region used for deployments | East US |
+| vnet_address_space | Address space of the virtual network | ["10.0.0.0/16"] |
+| training_subnet_address_space | Address space of the training subnet | ["10.0.1.0/24"] |
+| aks_subnet_address_space | Address space of the aks subnet | ["10.0.2.0/23"] |
+| ml_subnet_address_space | Address space of the ML workspace subnet | ["10.0.0.0/24"] |
+| vnet_hub_address_space | Address space of the Hub virtual network | ["10.1.0.0/16"] |
+| jumphost_subnet_address_space | Address space of the Jumphost subnet | ["10.1.2.0/24"] |
+| bastion_subnet_address_space | Address space of the bastion subnet | ["10.1.3.0/24"] |
+| firewall_subnet_address_space | Address space of the Az Fiewall subnet | ["10.1.4.0/24"] |
+| image_build_compute_name | Name of the compute cluster to be created and configured for building docker images (Azure ML Environments) | image-builder |
+| dsvm_name | Name of the Windows Data Science VM resource | vmdsvm01 |
+| dsvm_admin_username | Admin username of the Windows Data Science VM | azureadmin |
+| dsvm_host_password | Password for the admin username of the Data Science VM | - |
+
+## Usage
+
+```bash
+terraform init
+
+terraform plan \
+          -var name=azureml567 \
+          -var environment=dev \
+          # -var <for a full list of variables and default values, see 'Variables'> \
+          -out demo.tfplan
+
+terraform apply "demo.tfplan"
+```
+
+## Learn more
+
+- If you are new to Azure Machine Learning, see [Azure Machine Learning service](https://azure.microsoft.com/services/machine-learning-service/) and [Azure Machine Learning documentation](https://docs.microsoft.com/azure/machine-learning/).
+- To learn more about security configurations in Azure Machine Learning, see [Enterprise security and governance for Azure Machine Learning](https://docs.microsoft.com/en-us/azure/machine-learning/concept-enterprise-security).
+- For all configurations of Azure Machine Learning in Terraform, see [Terraform Hashicorp AzureRM provider documentation](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/machine_learning_workspace).

quickstart/301-machine-learning-hub-spoke-secure/variables.tf

@@ -0,0 +1,93 @@
+variable "name" {
+  type        = string
+  description = "Name of the deployment"
+}
+
+variable "environment" {
+  type        = string
+  description = "Name of the environment"
+  default     = "dev"
+}
+
+variable "location" {
+  type        = string
+  description = "Location of the resources"
+  default     = "East US"
+}
+
+#Spoke Virtual Network
+
+variable "vnet_address_space" {
+  type        = list(string)
+  description = "Address space of the spoke virtual network"
+  default     = ["10.0.0.0/16"]
+}
+
+variable "training_subnet_address_space" {
+  type        = list(string)
+  description = "Address space of the training subnet"
+  default     = ["10.0.1.0/24"]
+}
+
+variable "aks_subnet_address_space" {
+  type        = list(string)
+  description = "Address space of the aks subnet"
+  default     = ["10.0.2.0/23"]
+}
+
+variable "ml_subnet_address_space" {
+  type        = list(string)
+  description = "Address space of the ML workspace subnet"
+  default     = ["10.0.0.0/24"]
+}
+
+#Hub Virtual Network
+variable "vnet_hub_address_space" {
+  type        = list(string)
+  description = "Address space of the Hub virtual network"
+  default     = ["10.1.0.0/16"]
+}
+
+variable "jumphost_subnet_address_space" {
+  type        = list(string)
+  description = "Address space of the Jumphost subnet"
+  default     = ["10.1.2.0/24"]
+}
+
+variable "bastion_subnet_address_space" {
+  type        = list(string)
+  description = "Address space of the bastion subnet"
+  default     = ["10.1.3.0/24"]
+}
+
+variable "firewall_subnet_address_space" {
+  type        = list(string)
+  description = "Address space of the Az Fiewall subnet"
+  default     = ["10.1.4.0/24"]
+}
+
+#Image Build Compute
+variable "image_build_compute_name" {
+  type        = string
+  description = "Name of the compute cluster to be created and set to build docker images"
+  default     = "image-builder"
+}
+
+# DSVM
+variable "dsvm_name" {
+  type        = string
+  description = "Name of the Data Science VM"
+  default     = "vmdsvm01"
+}
+
+variable "dsvm_admin_username" {
+  type        = string
+  description = "Admin username of the Data Science VM"
+  default     = "azureadmin"
+}
+
+variable "dsvm_host_password" {
+  type        = string
+  description = "Password for the admin username of the Data Science VM"
+  sensitive   = true
+}

quickstart/301-machine-learning-hub-spoke-secure/workspace.tf

@@ -0,0 +1,196 @@
+# Dependent resources for Azure Machine Learning
+resource "azurerm_application_insights" "default" {
+  name                = "appi-${var.name}-${var.environment}"
+  location            = azurerm_resource_group.default.location
+  resource_group_name = azurerm_resource_group.default.name
+  workspace_id        = azurerm_log_analytics_workspace.default.id
+  application_type    = "web"
+}
+
+resource "azurerm_key_vault" "default" {
+  name                     = "kv-${var.name}-${var.environment}"
+  location                 = azurerm_resource_group.default.location
+  resource_group_name      = azurerm_resource_group.default.name
+  tenant_id                = data.azurerm_client_config.current.tenant_id
+  sku_name                 = "premium"
+  purge_protection_enabled = true
+
+  network_acls {
+    default_action = "Deny"
+    bypass         = "AzureServices"
+  }
+}
+
+resource "azurerm_storage_account" "default" {
+  name                     = "st${var.name}${var.environment}"
+  location                 = azurerm_resource_group.default.location
+  resource_group_name      = azurerm_resource_group.default.name
+  account_tier             = "Standard"
+  account_replication_type = "GRS"
+
+  network_rules {
+    default_action = "Deny"
+    bypass         = ["AzureServices"]
+  }
+}
+
+resource "azurerm_container_registry" "default" {
+  name                = "cr${var.name}${var.environment}"
+  location            = azurerm_resource_group.default.location
+  resource_group_name = azurerm_resource_group.default.name
+  sku                 = "Premium"
+  admin_enabled       = true
+
+  network_rule_set {
+    default_action = "Deny"
+  }
+  public_network_access_enabled = false
+}
+
+# Machine Learning workspace
+resource "azurerm_machine_learning_workspace" "default" {
+  name                    = "mlw-${var.name}-${var.environment}"
+  location                = azurerm_resource_group.default.location
+  resource_group_name     = azurerm_resource_group.default.name
+  application_insights_id = azurerm_application_insights.default.id
+  key_vault_id            = azurerm_key_vault.default.id
+  storage_account_id      = azurerm_storage_account.default.id
+  container_registry_id   = azurerm_container_registry.default.id
+
+  identity {
+    type = "SystemAssigned"
+  }
+
+  # Args of use when using an Azure Private Link configuration
+  public_network_access_enabled = false
+  image_build_compute_name      = var.image_build_compute_name
+  depends_on = [
+    azurerm_firewall.azure_firewall_instance,
+    azurerm_private_endpoint.kv_ple,
+    azurerm_private_endpoint.st_ple_blob,
+    azurerm_private_endpoint.storage_ple_file,
+    azurerm_private_endpoint.cr_ple,
+    azurerm_subnet.snet-training
+  ]
+
+}
+
+# Private endpoints
+resource "azurerm_private_endpoint" "kv_ple" {
+  name                = "ple-${var.name}-${var.environment}-kv"
+  location            = azurerm_resource_group.default.location
+  resource_group_name = azurerm_resource_group.default.name
+  subnet_id           = azurerm_subnet.snet-workspace.id
+
+  private_dns_zone_group {
+    name                 = "private-dns-zone-group"
+    private_dns_zone_ids = [azurerm_private_dns_zone.dnsvault.id]
+  }
+
+  private_service_connection {
+    name                           = "psc-${var.name}-kv"
+    private_connection_resource_id = azurerm_key_vault.default.id
+    subresource_names              = ["vault"]
+    is_manual_connection           = false
+  }
+}
+
+resource "azurerm_private_endpoint" "st_ple_blob" {
+  name                = "ple-${var.name}-${var.environment}-st-blob"
+  location            = azurerm_resource_group.default.location
+  resource_group_name = azurerm_resource_group.default.name
+  subnet_id           = azurerm_subnet.snet-workspace.id
+
+  private_dns_zone_group {
+    name                 = "private-dns-zone-group"
+    private_dns_zone_ids = [azurerm_private_dns_zone.dnsstorageblob.id]
+  }
+
+  private_service_connection {
+    name                           = "psc-${var.name}-st"
+    private_connection_resource_id = azurerm_storage_account.default.id
+    subresource_names              = ["blob"]
+    is_manual_connection           = false
+  }
+}
+
+resource "azurerm_private_endpoint" "storage_ple_file" {
+  name                = "ple-${var.name}-${var.environment}-st-file"
+  location            = azurerm_resource_group.default.location
+  resource_group_name = azurerm_resource_group.default.name
+  subnet_id           = azurerm_subnet.snet-workspace.id
+
+  private_dns_zone_group {
+    name                 = "private-dns-zone-group"
+    private_dns_zone_ids = [azurerm_private_dns_zone.dnsstoragefile.id]
+  }
+
+  private_service_connection {
+    name                           = "psc-${var.name}-st"
+    private_connection_resource_id = azurerm_storage_account.default.id
+    subresource_names              = ["file"]
+    is_manual_connection           = false
+  }
+}
+
+resource "azurerm_private_endpoint" "cr_ple" {
+  name                = "ple-${var.name}-${var.environment}-cr"
+  location            = azurerm_resource_group.default.location
+  resource_group_name = azurerm_resource_group.default.name
+  subnet_id           = azurerm_subnet.snet-workspace.id
+
+  private_dns_zone_group {
+    name                 = "private-dns-zone-group"
+    private_dns_zone_ids = [azurerm_private_dns_zone.dnscontainerregistry.id]
+  }
+
+  private_service_connection {
+    name                           = "psc-${var.name}-cr"
+    private_connection_resource_id = azurerm_container_registry.default.id
+    subresource_names              = ["registry"]
+    is_manual_connection           = false
+  }
+}
+
+resource "azurerm_private_endpoint" "mlw_ple" {
+  name                = "ple-${var.name}-${var.environment}-mlw"
+  location            = azurerm_resource_group.default.location
+  resource_group_name = azurerm_resource_group.default.name
+  subnet_id           = azurerm_subnet.snet-workspace.id
+
+  private_dns_zone_group {
+    name                 = "private-dns-zone-group"
+    private_dns_zone_ids = [azurerm_private_dns_zone.dnsazureml.id, azurerm_private_dns_zone.dnsnotebooks.id]
+  }
+
+  private_service_connection {
+    name                           = "psc-${var.name}-mlw"
+    private_connection_resource_id = azurerm_machine_learning_workspace.default.id
+    subresource_names              = ["amlworkspace"]
+    is_manual_connection           = false
+  }
+}
+
+# Compute cluster for image building required since the workspace is behind a vnet.
+# For more details, see https://docs.microsoft.com/en-us/azure/machine-learning/tutorial-create-secure-workspace#configure-image-builds.
+resource "azurerm_machine_learning_compute_cluster" "image-builder" {
+  name                          = var.image_build_compute_name
+  location                      = azurerm_resource_group.default.location
+  vm_priority                   = "LowPriority"
+  vm_size                       = "Standard_DS2_v2"
+  machine_learning_workspace_id = azurerm_machine_learning_workspace.default.id
+  subnet_resource_id            = azurerm_subnet.snet-training.id
+
+  scale_settings {
+    min_node_count                       = 0
+    max_node_count                       = 3
+    scale_down_nodes_after_idle_duration = "PT15M" # 15 minutes
+  }
+
+  identity {
+    type = "SystemAssigned"
+  }
+  depends_on = [
+    azurerm_private_endpoint.mlw_ple
+  ]
+}