From 32a9580ffb0b1745cbfc3d8304fd94b7bf8288cc Mon Sep 17 00:00:00 2001
From: ryhud <rhudson@microsoft.com>
Date: Wed, 6 Oct 2021 14:10:15 -0400
Subject: [PATCH 01/20] adding AML 301

---
 .../azure-firewall.tf                         |  80 ++++++++
 .../bastion.tf                                | 123 ++++++++++++
 .../compute.tf                                |  41 ++++
 .../dsvm.tf                                   |  52 +++++
 .../main.tf                                   |  28 +++
 .../monitor.tf                                |   7 +
 .../network-hub.tf                            | 144 ++++++++++++++
 .../network-spoke.tf                          | 137 +++++++++++++
 .../readme.md                                 |  62 ++++++
 .../variables.tf                              |  93 +++++++++
 .../workspace.tf                              | 185 ++++++++++++++++++
 11 files changed, 952 insertions(+)
 create mode 100644 quickstart/301-machine-learning-hub-spoke-secure/azure-firewall.tf
 create mode 100644 quickstart/301-machine-learning-hub-spoke-secure/bastion.tf
 create mode 100644 quickstart/301-machine-learning-hub-spoke-secure/compute.tf
 create mode 100644 quickstart/301-machine-learning-hub-spoke-secure/dsvm.tf
 create mode 100644 quickstart/301-machine-learning-hub-spoke-secure/main.tf
 create mode 100644 quickstart/301-machine-learning-hub-spoke-secure/monitor.tf
 create mode 100644 quickstart/301-machine-learning-hub-spoke-secure/network-hub.tf
 create mode 100644 quickstart/301-machine-learning-hub-spoke-secure/network-spoke.tf
 create mode 100644 quickstart/301-machine-learning-hub-spoke-secure/readme.md
 create mode 100644 quickstart/301-machine-learning-hub-spoke-secure/variables.tf
 create mode 100644 quickstart/301-machine-learning-hub-spoke-secure/workspace.tf

diff --git a/quickstart/301-machine-learning-hub-spoke-secure/azure-firewall.tf b/quickstart/301-machine-learning-hub-spoke-secure/azure-firewall.tf
new file mode 100644
index 00000000..23365dd0
--- /dev/null
+++ b/quickstart/301-machine-learning-hub-spoke-secure/azure-firewall.tf
@@ -0,0 +1,80 @@
+resource "azurerm_public_ip" "azure_firewall" {
+    name                        = "pip-azfw"
+    location                    = azurerm_resource_group.default.location
+    resource_group_name         = azurerm_resource_group.hub_rg.name
+    allocation_method           = "Static"
+    sku                         = "Standard"
+}
+
+resource "azurerm_firewall_policy" "base_policy" {
+  name                = "afwp-base-01"
+  resource_group_name = azurerm_resource_group.hub_rg.name
+  location            = azurerm_resource_group.default.location
+  dns {
+    proxy_enabled = true
+  }
+  depends_on = [
+    azurerm_virtual_network_peering.direction1,
+    azurerm_virtual_network_peering.direction2
+  ]
+}
+resource "azurerm_firewall" "azure_firewall_instance" { 
+    name                        = "afw-${var.name}-${var.environment}"
+    location                    = azurerm_resource_group.default.location
+    resource_group_name         = azurerm_resource_group.hub_rg.name
+    firewall_policy_id          = azurerm_firewall_policy.base_policy.id   
+
+    ip_configuration {
+        name                    = "configuration"
+        subnet_id               = azurerm_subnet.azure_firewall.id 
+        public_ip_address_id    = azurerm_public_ip.azure_firewall.id
+    }
+
+    timeouts {
+      create = "60m"
+      delete = "2h"
+  }
+  depends_on = [ azurerm_public_ip.azure_firewall,
+  azurerm_firewall_policy.base_policy]
+}
+
+resource "azurerm_monitor_diagnostic_setting" "azure_firewall_instance" {
+  name                        = "diagnostics"
+  target_resource_id          = azurerm_firewall.azure_firewall_instance.id
+  log_analytics_workspace_id  = azurerm_log_analytics_workspace.default.id
+
+  log {
+    category = "AzureFirewallApplicationRule"
+    enabled  = true
+
+    retention_policy {
+      enabled = false
+    }
+  }
+  log {
+    category = "AzureFirewallNetworkRule"
+    enabled  = true
+
+    retention_policy {
+      enabled = false
+    }
+  }
+  log {
+    category = "AzureFirewallDnsProxy"
+    enabled  = true
+
+    retention_policy {
+      enabled = false
+    }
+  }
+  
+
+  metric {
+    category = "AllMetrics"
+
+    retention_policy {
+      enabled = false
+    }
+  }
+
+}
\ No newline at end of file
diff --git a/quickstart/301-machine-learning-hub-spoke-secure/bastion.tf b/quickstart/301-machine-learning-hub-spoke-secure/bastion.tf
new file mode 100644
index 00000000..e83f90d8
--- /dev/null
+++ b/quickstart/301-machine-learning-hub-spoke-secure/bastion.tf
@@ -0,0 +1,123 @@
+resource "azurerm_public_ip" "azure_bastion" { 
+    name                        = "pip-azure-bastion"
+    location                    = azurerm_resource_group.default.location
+    resource_group_name         = azurerm_resource_group.hub_rg.name
+    allocation_method           = "Static"
+    sku                         = "Standard" 
+}
+
+resource "azurerm_network_security_group" "bastion_nsg" { 
+    name                        = "nsg-bastion"
+    location                    = azurerm_resource_group.default.location
+    resource_group_name         = azurerm_resource_group.hub_rg.name
+
+    security_rule {      
+      name                        = "AllowHTTPSInbound"
+      priority                    = 100
+      direction                   = "Inbound"
+      access                      = "Allow"
+      protocol                    = "Tcp"
+      source_port_range           = "*"
+      destination_port_range      = "443"
+      source_address_prefix       = "Internet"
+      destination_address_prefix  = "*"
+    }
+    security_rule {
+      name                        = "AllowGatewayManagerInbound"
+      priority                    = 200
+      direction                   = "Inbound"
+      access                      = "Allow"
+      protocol                    = "Tcp"
+      source_port_range           = "*"
+      destination_port_range      = "443"
+      source_address_prefix       = "GatewayManager"
+      destination_address_prefix  = "*"
+    }
+    security_rule {
+      name                        = "AllowAzureLBInbound"
+      priority                    = 300
+      direction                   = "Inbound"
+      access                      = "Allow"
+      protocol                    = "Tcp"
+      source_port_range           = "*"
+      destination_port_range      = "443"
+      source_address_prefix       = "AzureLoadBalancer"
+      destination_address_prefix  = "*"
+    }
+    security_rule {
+      name                        = "AllowBastionHostCommunication"
+      priority                    = 400
+      direction                   = "Inbound"
+      access                      = "Allow"
+      protocol                    = "*"
+      source_port_range           = "*"
+      destination_port_ranges      = ["5701","8080"]
+      source_address_prefix       = "VirtualNetwork"
+      destination_address_prefix  = "VirtualNetwork"
+    }
+    security_rule {
+      name                        = "AllowRdpSshOutbound"
+      priority                    = 100
+      direction                   = "Outbound"
+      access                      = "Allow"
+      protocol                    = "Tcp"
+      source_port_range           = "*"
+      destination_port_ranges      = ["22", "3389"]
+      source_address_prefix       = "*"
+      destination_address_prefix  = "VirtualNetwork"
+    }
+      security_rule {
+      name                        = "AllowBastionHostCommunicationOutbound"
+      priority                    = 110
+      direction                   = "Outbound"
+      access                      = "Allow"
+      protocol                    = "Tcp"
+      source_port_range           = "*"
+      destination_port_ranges      = ["5701", "8080"]
+      source_address_prefix       = "VirtualNetwork"
+      destination_address_prefix  = "VirtualNetwork"
+  }
+    security_rule {
+      name                        = "AllowAzureCloudOutbound"
+      priority                    = 120
+      direction                   = "Outbound"
+      access                      = "Allow"
+      protocol                    = "Tcp"
+      source_port_range           = "*"
+      destination_port_ranges      = ["443"]
+      source_address_prefix       = "*"
+      destination_address_prefix  = "AzureCloud"
+    }
+    security_rule {
+      name                        = "AllowGetSessionInformation"
+      priority                    = 130
+      direction                   = "Outbound"
+      access                      = "Allow"
+      protocol                    = "Tcp"
+      source_port_range           = "*"
+      destination_port_ranges      = ["80"]
+      source_address_prefix       = "*"
+      destination_address_prefix  = "Internet"
+}
+    
+}
+
+resource "azurerm_subnet_network_security_group_association" "bastion_nsg_assoc" {
+  subnet_id                 = azurerm_subnet.azure_bastion.id
+  network_security_group_id = azurerm_network_security_group.bastion_nsg.id
+  depends_on = [ azurerm_bastion_host.azure_bastion_instance ]
+}
+
+
+resource "azurerm_bastion_host" "azure_bastion_instance" {
+    name                        = "bas-${var.name}-${var.environment}"
+    location                    = azurerm_resource_group.default.location
+    resource_group_name         = azurerm_resource_group.hub_rg.name
+
+    ip_configuration { 
+        name                    = "configuration"
+        subnet_id               = azurerm_subnet.azure_bastion.id
+        public_ip_address_id    = azurerm_public_ip.azure_bastion.id 
+    }
+}
+
diff --git a/quickstart/301-machine-learning-hub-spoke-secure/compute.tf b/quickstart/301-machine-learning-hub-spoke-secure/compute.tf
new file mode 100644
index 00000000..b95541b8
--- /dev/null
+++ b/quickstart/301-machine-learning-hub-spoke-secure/compute.tf
@@ -0,0 +1,41 @@
+# Generate random string for unique compute instance name
+resource "random_string" "ci_prefix" {
+  length  = 8
+  upper   = false
+  special = false
+  number  = false
+}
+
+/* # Compute instance
+resource "azurerm_machine_learning_compute_instance" "compute_instance" {
+  name                          = "${random_string.ci_prefix.result}instance"
+  location                      = azurerm_resource_group.default.location
+  machine_learning_workspace_id = azurerm_machine_learning_workspace.default.id
+  virtual_machine_size          = "STANDARD_DS2_V2"
+  subnet_resource_id            = azurerm_subnet.snet-training.id
+
+  depends_on = [
+    azurerm_private_endpoint.mlw_ple
+  ]
+}
+*/
+# Compute cluster
+resource "azurerm_machine_learning_compute_cluster" "compute" {
+  name                          = "cpu-cluster"
+  location                      = azurerm_resource_group.default.location
+  machine_learning_workspace_id = azurerm_machine_learning_workspace.default.id
+  vm_priority                   = "Dedicated"
+  vm_size                       = "STANDARD_DS2_V2"
+  subnet_resource_id            = azurerm_subnet.snet-training.id
+
+  identity {
+    type = "SystemAssigned"
+  }
+
+  scale_settings {
+    min_node_count                       = 0
+    max_node_count                       = 3
+    scale_down_nodes_after_idle_duration = "PT15M" # 15 minutes
+  }
+
+}
\ No newline at end of file
diff --git a/quickstart/301-machine-learning-hub-spoke-secure/dsvm.tf b/quickstart/301-machine-learning-hub-spoke-secure/dsvm.tf
new file mode 100644
index 00000000..8e1f93a3
--- /dev/null
+++ b/quickstart/301-machine-learning-hub-spoke-secure/dsvm.tf
@@ -0,0 +1,52 @@
+resource "azurerm_network_interface" "dsvm" { 
+    name                              = "nic-${var.dsvm_name}"
+    location                          = azurerm_resource_group.default.location
+    resource_group_name               = azurerm_resource_group.default.name
+    
+    ip_configuration { 
+        name                          = "configuration"
+        subnet_id                     = azurerm_subnet.snet-jumphost.id 
+        private_ip_address_allocation = "Dynamic"        
+    }
+    /*depends_on = [
+      azurerm_route_table.jumphost_rt
+    ]
+    */
+}
+
+resource "azurerm_windows_virtual_machine" "dsvm" {
+  name                  = var.dsvm_name
+  location              = azurerm_resource_group.default.location
+  resource_group_name   = azurerm_resource_group.default.name
+  network_interface_ids = [
+        azurerm_network_interface.dsvm.id
+    ]
+  size               = "Standard_DS3_v2"
+ 
+  source_image_reference {
+    publisher = "microsoft-dsvm"
+    offer     = "dsvm-win-2019"
+    sku       = "server-2019"
+    version   = "latest"
+  }
+ 
+  os_disk {
+    name              = "osdisk-${var.dsvm_name}"
+    caching           = "ReadWrite"  
+    storage_account_type = "Premium_LRS"
+  }
+ 
+  identity {
+    type = "SystemAssigned"
+  }
+  computer_name      = var.dsvm_name
+  admin_username     = var.dsvm_admin_username
+  admin_password     = var.dsvm_host_password
+  
+  provision_vm_agent = true
+ 
+  timeouts {
+      create = "60m"
+      delete = "2h"
+  }
+}
diff --git a/quickstart/301-machine-learning-hub-spoke-secure/main.tf b/quickstart/301-machine-learning-hub-spoke-secure/main.tf
new file mode 100644
index 00000000..124361e0
--- /dev/null
+++ b/quickstart/301-machine-learning-hub-spoke-secure/main.tf
@@ -0,0 +1,28 @@
+terraform {
+  required_version = ">=0.15.0"
+
+  required_providers {
+    azurerm = {
+      source  = "hashicorp/azurerm"
+      version = "=2.79.1"
+    }
+  }
+}
+
+provider "azurerm" {
+  features {}
+}
+
+data "azurerm_client_config" "current" {}
+
+resource "azurerm_resource_group" "default" {
+  name     = "rg-${var.name}-${var.environment}"
+  location = var.location
+}
+
+#Hub Resource Group
+resource "azurerm_resource_group" "hub_rg" {
+  name     = "rg-hub-${var.name}-${var.environment}"
+  location = var.location
+  
+}
\ No newline at end of file
diff --git a/quickstart/301-machine-learning-hub-spoke-secure/monitor.tf b/quickstart/301-machine-learning-hub-spoke-secure/monitor.tf
new file mode 100644
index 00000000..bcb633eb
--- /dev/null
+++ b/quickstart/301-machine-learning-hub-spoke-secure/monitor.tf
@@ -0,0 +1,7 @@
+resource "azurerm_log_analytics_workspace" "default" {
+  name                = "log-${var.name}-${var.environment}"
+  location            = azurerm_resource_group.default.location
+  resource_group_name = azurerm_resource_group.hub_rg.name
+  sku                 = "PerGB2018"
+  retention_in_days   = 30
+}
\ No newline at end of file
diff --git a/quickstart/301-machine-learning-hub-spoke-secure/network-hub.tf b/quickstart/301-machine-learning-hub-spoke-secure/network-hub.tf
new file mode 100644
index 00000000..94c3d5b1
--- /dev/null
+++ b/quickstart/301-machine-learning-hub-spoke-secure/network-hub.tf
@@ -0,0 +1,144 @@
+#Hub Virtual Network
+
+resource "azurerm_virtual_network" "hub" {
+  name                = "vnet-hub-${var.name}-${var.environment}"
+  address_space       = var.vnet_hub_address_space
+  location            = azurerm_resource_group.default.location
+  resource_group_name = azurerm_resource_group.hub_rg.name
+}
+
+resource "azurerm_subnet" "snet-jumphost" {
+  name                          = "snet-jumphost"
+  resource_group_name           = azurerm_resource_group.hub_rg.name
+  virtual_network_name          = azurerm_virtual_network.hub.name
+  address_prefixes              = var.jumphost_subnet_address_space
+
+}
+
+
+resource "azurerm_subnet" "azure_bastion" {
+    name                        = "AzureBastionSubnet"
+    resource_group_name         = azurerm_resource_group.hub_rg.name
+    virtual_network_name        = azurerm_virtual_network.hub.name
+    address_prefixes            = var.bastion_subnet_address_space
+    
+} 
+resource "azurerm_subnet" "azure_firewall" {
+    name                        = "AzureFirewallSubnet"
+    resource_group_name         = azurerm_resource_group.hub_rg.name
+    virtual_network_name        = azurerm_virtual_network.hub.name
+    address_prefixes            = var.firewall_subnet_address_space
+    
+} 
+
+#Vnet Peering
+
+resource "azurerm_virtual_network_peering" "direction1" {
+  name                         = "${azurerm_resource_group.hub_rg.name}-to-${azurerm_resource_group.default.name}"
+  resource_group_name          = azurerm_resource_group.hub_rg.name
+  virtual_network_name         = azurerm_virtual_network.hub.name
+  remote_virtual_network_id    = azurerm_virtual_network.default.id
+  allow_virtual_network_access = true
+  allow_forwarded_traffic      = false
+  allow_gateway_transit        = false
+  use_remote_gateways          = false
+  
+}
+
+resource "azurerm_virtual_network_peering" "direction2" {
+  name                         = "${azurerm_resource_group.default.name}-to-${azurerm_resource_group.hub_rg.name}"
+  resource_group_name          = azurerm_resource_group.default.name
+  virtual_network_name         = azurerm_virtual_network.default.name
+  remote_virtual_network_id    = azurerm_virtual_network.hub.id
+  allow_virtual_network_access = true
+  allow_forwarded_traffic      = false
+  allow_gateway_transit        = false
+  use_remote_gateways          = false
+
+}
+
+# Private DNS Zones
+resource "azurerm_private_dns_zone" "dnsvault" {
+  name                = "privatelink.vaultcore.azure.net"
+  resource_group_name = azurerm_resource_group.hub_rg.name
+}
+
+resource "azurerm_private_dns_zone_virtual_network_link" "vnetlinkvault" {
+  name                  = "dnsvaultlink"
+  resource_group_name   = azurerm_resource_group.hub_rg.name
+  private_dns_zone_name = azurerm_private_dns_zone.dnsvault.name
+  virtual_network_id    = azurerm_virtual_network.hub.id
+}
+
+resource "azurerm_private_dns_zone" "dnsstorageblob" {
+  name                = "privatelink.blob.core.windows.net"
+  resource_group_name = azurerm_resource_group.hub_rg.name
+}
+
+resource "azurerm_private_dns_zone_virtual_network_link" "vnetlinkblob" {
+  name                  = "dnsblobstoragelink"
+  resource_group_name   = azurerm_resource_group.hub_rg.name
+  private_dns_zone_name = azurerm_private_dns_zone.dnsstorageblob.name
+  virtual_network_id    = azurerm_virtual_network.hub.id
+}
+
+resource "azurerm_private_dns_zone" "dnsstoragefile" {
+  name                = "privatelink.file.core.windows.net"
+  resource_group_name = azurerm_resource_group.hub_rg.name
+}
+
+resource "azurerm_private_dns_zone_virtual_network_link" "vnetlinkfile" {
+  name                  = "dnsfilestoragelink"
+  resource_group_name   = azurerm_resource_group.hub_rg.name
+  private_dns_zone_name = azurerm_private_dns_zone.dnsstoragefile.name
+  virtual_network_id    = azurerm_virtual_network.hub.id
+}
+
+resource "azurerm_private_dns_zone" "dnscontainerregistry" {
+  name                = "privatelink.azurecr.io"
+  resource_group_name = azurerm_resource_group.hub_rg.name
+}
+
+resource "azurerm_private_dns_zone_virtual_network_link" "vnetlinkcr" {
+  name                  = "dnscrlink"
+  resource_group_name   = azurerm_resource_group.hub_rg.name
+  private_dns_zone_name = azurerm_private_dns_zone.dnscontainerregistry.name
+  virtual_network_id    = azurerm_virtual_network.hub.id
+}
+
+resource "azurerm_private_dns_zone" "dnsazureml" {
+  name                = "privatelink.api.azureml.ms"
+  resource_group_name = azurerm_resource_group.hub_rg.name
+}
+
+resource "azurerm_private_dns_zone_virtual_network_link" "vnetlinkml" {
+  name                  = "dnsazuremllink"
+  resource_group_name   = azurerm_resource_group.hub_rg.name
+  private_dns_zone_name = azurerm_private_dns_zone.dnsazureml.name
+  virtual_network_id    = azurerm_virtual_network.hub.id
+}
+
+resource "azurerm_private_dns_zone" "dnsnotebooks" {
+  name                = "privatelink.notebooks.azure.net"
+  resource_group_name = azurerm_resource_group.hub_rg.name
+}
+
+resource "azurerm_private_dns_zone_virtual_network_link" "vnetlinknbs" {
+  name                  = "dnsnotebookslink"
+  resource_group_name   = azurerm_resource_group.hub_rg.name
+  private_dns_zone_name = azurerm_private_dns_zone.dnsnotebooks.name
+  virtual_network_id    = azurerm_virtual_network.hub.id
+}
+
+# NSG for jump_host Subnet
+
+resource "azurerm_network_security_group" "jump_host" { 
+    name                        = "nsg-jumphost-subnet"
+    location                    = azurerm_resource_group.hub_rg.location
+    resource_group_name         = azurerm_resource_group.hub_rg.name
+}
+
+resource "azurerm_subnet_network_security_group_association" "jumphost_nsg_assoc" {
+  subnet_id                 = azurerm_subnet.snet-jumphost.id 
+  network_security_group_id = azurerm_network_security_group.jump_host.id
+}
diff --git a/quickstart/301-machine-learning-hub-spoke-secure/network-spoke.tf b/quickstart/301-machine-learning-hub-spoke-secure/network-spoke.tf
new file mode 100644
index 00000000..0d5568d3
--- /dev/null
+++ b/quickstart/301-machine-learning-hub-spoke-secure/network-spoke.tf
@@ -0,0 +1,137 @@
+# Virtual Network
+resource "azurerm_virtual_network" "default" {
+  name                = "vnet-${var.name}-${var.environment}"
+  address_space       = var.vnet_address_space
+  location            = azurerm_resource_group.default.location
+  resource_group_name = azurerm_resource_group.default.name
+}
+
+resource "azurerm_subnet" "snet-training" {
+  name                                           = "snet-training"
+  resource_group_name                            = azurerm_resource_group.default.name
+  virtual_network_name                           = azurerm_virtual_network.default.name
+  address_prefixes                               = var.training_subnet_address_space
+  enforce_private_link_endpoint_network_policies = true
+}
+
+resource "azurerm_subnet" "snet-aks" {
+  name                                           = "snet-aks"
+  resource_group_name                            = azurerm_resource_group.default.name
+  virtual_network_name                           = azurerm_virtual_network.default.name
+  address_prefixes                               = var.aks_subnet_address_space
+  enforce_private_link_endpoint_network_policies = true
+}
+
+resource "azurerm_subnet" "snet-workspace" {
+  name                                           = "snet-workspace"
+  resource_group_name                            = azurerm_resource_group.default.name
+  virtual_network_name                           = azurerm_virtual_network.default.name
+  address_prefixes                               = var.ml_subnet_address_space
+  enforce_private_link_endpoint_network_policies = true
+}
+
+# Network Security Groups
+
+resource "azurerm_network_security_group" "nsg-training" {
+  name                = "nsg-training"
+  location            = azurerm_resource_group.default.location
+  resource_group_name = azurerm_resource_group.default.name
+
+  security_rule {
+    name                       = "BatchNodeManagement"
+    priority                   = 100
+    direction                  = "Inbound"
+    access                     = "Allow"
+    protocol                   = "Tcp"
+    source_port_range          = "*"
+    destination_port_range     = "29876-29877"
+    source_address_prefix      = "BatchNodeManagement"
+    destination_address_prefix = "*"
+  }
+
+  security_rule {
+    name                       = "AzureMachineLearning"
+    priority                   = 110
+    direction                  = "Inbound"
+    access                     = "Allow"
+    protocol                   = "Tcp"
+    source_port_range          = "*"
+    destination_port_range     = "44224"
+    source_address_prefix      = "AzureMachineLearning"
+    destination_address_prefix = "*"
+  }
+}
+
+resource "azurerm_subnet_network_security_group_association" "nsg-training-link" {
+  subnet_id                 = azurerm_subnet.snet-training.id
+  network_security_group_id = azurerm_network_security_group.nsg-training.id
+}
+
+resource "azurerm_network_security_group" "nsg-aks" {
+  name                = "nsg-aks"
+  location            = azurerm_resource_group.default.location
+  resource_group_name = azurerm_resource_group.default.name
+}
+
+resource "azurerm_subnet_network_security_group_association" "nsg-aks-link" {
+  subnet_id                 = azurerm_subnet.snet-aks.id
+  network_security_group_id = azurerm_network_security_group.nsg-aks.id
+}
+
+# User Defined Routes
+
+# UDR for compute instance and compute clusters
+resource "azurerm_route_table" "rt-training" {
+  name                = "rt-training"
+  location            = azurerm_resource_group.default.location
+  resource_group_name = azurerm_resource_group.default.name
+}
+
+resource "azurerm_route" "training-Internet-Route" {
+  name                = "Internet"
+  resource_group_name = azurerm_resource_group.default.name
+  route_table_name    = azurerm_route_table.rt-training.name
+  address_prefix      = "0.0.0.0/0"
+  next_hop_type       = "Internet"
+}
+
+resource "azurerm_route" "training-AzureMLRoute" {
+  name                = "AzureMLRoute"
+  resource_group_name = azurerm_resource_group.default.name
+  route_table_name    = azurerm_route_table.rt-training.name
+  address_prefix      = "AzureMachineLearning"
+  next_hop_type       = "Internet"
+}
+
+resource "azurerm_route" "training-BatchRoute" {
+  name                = "BatchRoute"
+  resource_group_name = azurerm_resource_group.default.name
+  route_table_name    = azurerm_route_table.rt-training.name
+  address_prefix      = "BatchNodeManagement"
+  next_hop_type       = "Internet"
+}
+
+resource "azurerm_subnet_route_table_association" "rt-training-link" {
+  subnet_id      = azurerm_subnet.snet-training.id
+  route_table_id = azurerm_route_table.rt-training.id
+}
+
+# Inferencing (AKS) Route
+resource "azurerm_route_table" "rt-aks" {
+  name                = "rt-aks"
+  location            = azurerm_resource_group.default.location
+  resource_group_name = azurerm_resource_group.default.name
+}
+
+resource "azurerm_route" "aks-Internet-Route" {
+  name                = "Internet"
+  resource_group_name = azurerm_resource_group.default.name
+  route_table_name    = azurerm_route_table.rt-aks.name
+  address_prefix      = "0.0.0.0/0"
+  next_hop_type       = "Internet"
+}
+
+resource "azurerm_subnet_route_table_association" "rt-aks-link" {
+  subnet_id      = azurerm_subnet.snet-aks.id
+  route_table_id = azurerm_route_table.rt-aks.id
+}
diff --git a/quickstart/301-machine-learning-hub-spoke-secure/readme.md b/quickstart/301-machine-learning-hub-spoke-secure/readme.md
new file mode 100644
index 00000000..688c8cf6
--- /dev/null
+++ b/quickstart/301-machine-learning-hub-spoke-secure/readme.md
@@ -0,0 +1,62 @@
+# Azure Machine Learning workspace (Secure Hub and Spoke with Firewall)
+
+This deployment configuration specifies an [Azure Machine Learning workspace](https://docs.microsoft.com/en-us/azure/machine-learning/concept-workspace), 
+and its associated resources including Azure Key Vault, Azure Storage, Azure Application Insights and Azure Container Registry.
+
+In addition to these core services, this configuration specifies any networking components that are required to set up Azure Machine Learning
+for private network connectivity using [Azure Private Link](https://docs.microsoft.com/en-us/azure/private-link/). 
+
+This configuration describes the minimal set of resources you require to get started with Azure Machine Learning in a network-isolated set-up. This configuration creates new network components. If you want to reuse existing network components, see [202 example](../201-machine-learning-moderately-secure/readme.md).
+
+## Resources
+
+| Terraform Resource Type | Description |
+| - | - |
+| `azurerm_resource_group` | The resource group all resources get deployed into |
+| `azurerm_application_insights` | An Azure Application Insights instance associated to the Azure Machine Learning workspace |
+| `azurerm_key_vault` | An Azure Key Vault instance associated to the Azure Machine Learning workspace |
+| `azurerm_storage_account` | An Azure Storage instance associated to the Azure Machine Learning workspace |
+| `azurerm_container_registry` | An Azure Container Registry instance associated to the Azure Machine Learning workspace |
+| `azurerm_machine_learning_workspace` | An Azure Machine Learning workspace instance |
+| `azurerm_virtual_network` | An Azure Machine Learning workspace instance |
+| `azurerm_subnet` | An Azure Machine Learning workspace instance |
+| `azurerm_private_dns_zone` | Private DNS Zones for FQDNs required for Azure Machine Learning and associated resources |
+| `azurerm_private_dns_zone_virtual_network_link` | Virtual network links of the Private DNS Zones to the virtual network resource |
+| `azurerm_private_endpoint` | Private Endpoints for the Azure Machine Learning workspace and associated resources |
+| `azurerm_machine_learning_compute_instance` | An Azure Machine Learning compute instance a single-node managed compute. |
+| `azurerm_machine_learning_compute_cluster` | An Azure Machine Learning compute cluster as multi-node shared and managed compute. |
+| `azurerm_network_security_group` | Network security group with required inbound and outbound rules for Azure Machine Learning. |
+
+## Variables
+
+| Name | Description | Default |
+|-|-|-|
+| name | Name of the deployment | - |
+| environment | The deployment environment name (used for pre- and postfixing resource names) | dev  |
+| location | The Azure region used for deployments | East US |
+| vnet_address_space | Address space of the virtual network | ["10.0.0.0/16"] |
+| training_subnet_address_space | Address space of the training subnet | ["10.0.1.0/24"] |
+| aks_subnet_address_space | Address space of the aks subnet | ["10.0.2.0/23"] |
+| ml_subnet_address_space | Address space of the ML workspace subnet | ["10.0.0.0/24"] |
+| image_build_compute_name | Name of the compute cluster to be created and configured for building docker images (Azure ML Environments) | image-builder |
+
+
+## Usage
+
+```bash
+terraform init
+
+terraform plan \
+          -var name=azureml567 \
+          -var environment=dev \
+          # -var <for a full list of variables and default values, see 'Variables'> \
+          -out demo.tfplan
+
+terraform apply "demo.tfplan"
+```
+
+## Learn more
+
+- If you are new to Azure Machine Learning, see [Azure Machine Learning service](https://azure.microsoft.com/services/machine-learning-service/) and [Azure Machine Learning documentation](https://docs.microsoft.com/azure/machine-learning/).
+- To learn more about security configurations in Azure Machine Learning, see [Enterprise security and governance for Azure Machine Learning](https://docs.microsoft.com/en-us/azure/machine-learning/concept-enterprise-security).
+- For all configurations of Azure Machine Learning in Terraform, see [Terraform Hashicorp AzureRM provider documentation](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/machine_learning_workspace).
diff --git a/quickstart/301-machine-learning-hub-spoke-secure/variables.tf b/quickstart/301-machine-learning-hub-spoke-secure/variables.tf
new file mode 100644
index 00000000..7d0ef6b1
--- /dev/null
+++ b/quickstart/301-machine-learning-hub-spoke-secure/variables.tf
@@ -0,0 +1,93 @@
+variable "name" {
+  type        = string
+  description = "Name of the deployment"
+}
+
+variable "environment" {
+  type        = string
+  description = "Name of the environment"
+  default     = "dev"
+}
+
+variable "location" {
+  type        = string
+  description = "Location of the resources"
+  default     = "East US"
+}
+
+#Spoke Virtual Network
+
+variable "vnet_address_space" {
+  type        = list(string)
+  description = "Address space of the spoke virtual network"
+  default     = ["10.0.0.0/16"]
+}
+
+variable "training_subnet_address_space" {
+  type        = list(string)
+  description = "Address space of the training subnet"
+  default     = ["10.0.1.0/24"]
+}
+
+variable "aks_subnet_address_space" {
+  type        = list(string)
+  description = "Address space of the aks subnet"
+  default     = ["10.0.2.0/23"]
+}
+
+variable "ml_subnet_address_space" {
+  type        = list(string)
+  description = "Address space of the ML workspace subnet"
+  default     = ["10.0.0.0/24"]
+}
+
+#Hub Virtual Network
+variable "vnet_hub_address_space" {
+  type        = list(string)
+  description = "Address space of the Hub virtual network"
+  default     = ["10.1.0.0/16"]
+}
+
+variable "jumphost_subnet_address_space" {
+  type        = list(string)
+  description = "Address space of the Jumphost subnet"
+  default     = ["10.1.2.0/24"]
+}
+
+variable "bastion_subnet_address_space" {
+  type        = list(string)
+  description = "Address space of the bastion subnet"
+  default     = ["10.1.3.0/24"]
+}
+
+variable "firewall_subnet_address_space" {
+  type        = list(string)
+  description = "Address space of the Az Fiewall subnet"
+  default     = ["10.1.4.0/24"]
+}
+
+#Image Build Compute
+variable "image_build_compute_name" {
+  type        = string
+  description = "Name of the compute cluster to be created and set to build docker images"
+  default     = "image-builder"
+}
+
+# DSVM
+variable "dsvm_name" {
+  type        = string
+  description = "Name of the Data Science VM"
+  default     = "vmdsvm01"
+}
+
+variable "dsvm_admin_username" {
+  type        = string
+  description = "Admin username of the Data Science VM"
+  default     = "azureadmin"
+}
+
+variable "dsvm_host_password" {
+  type        = string
+  description = "Password for the admin username of the Data Science VM"
+  
+}
\ No newline at end of file
diff --git a/quickstart/301-machine-learning-hub-spoke-secure/workspace.tf b/quickstart/301-machine-learning-hub-spoke-secure/workspace.tf
new file mode 100644
index 00000000..f9be016d
--- /dev/null
+++ b/quickstart/301-machine-learning-hub-spoke-secure/workspace.tf
@@ -0,0 +1,185 @@
+# Dependent resources for Azure Machine Learning
+resource "azurerm_application_insights" "default" {
+  name                = "appi-${var.name}-${var.environment}"
+  location            = azurerm_resource_group.default.location
+  resource_group_name = azurerm_resource_group.default.name
+  workspace_id        = azurerm_log_analytics_workspace.default.id
+  application_type    = "web"
+}
+
+resource "azurerm_key_vault" "default" {
+  name                     = "kv-${var.name}-${var.environment}"
+  location                 = azurerm_resource_group.default.location
+  resource_group_name      = azurerm_resource_group.default.name
+  tenant_id                = data.azurerm_client_config.current.tenant_id
+  sku_name                 = "premium"
+  purge_protection_enabled = true
+
+  network_acls {
+    default_action = "Deny"
+    bypass         = "AzureServices"
+  }
+}
+
+resource "azurerm_storage_account" "default" {
+  name                     = "st${var.name}${var.environment}"
+  location                 = azurerm_resource_group.default.location
+  resource_group_name      = azurerm_resource_group.default.name
+  account_tier             = "Standard"
+  account_replication_type = "GRS"
+
+  network_rules {
+    default_action = "Deny"
+    bypass         = ["AzureServices"]
+  }
+}
+
+resource "azurerm_container_registry" "default" {
+  name                = "cr${var.name}${var.environment}"
+  location            = azurerm_resource_group.default.location
+  resource_group_name = azurerm_resource_group.default.name
+  sku                 = "Premium"
+  admin_enabled       = true
+
+  network_rule_set {
+    default_action = "Deny"
+  }
+  public_network_access_enabled = false
+}
+
+# Machine Learning workspace
+resource "azurerm_machine_learning_workspace" "default" {
+  name                    = "mlw-${var.name}-${var.environment}"
+  location                = azurerm_resource_group.default.location
+  resource_group_name     = azurerm_resource_group.default.name
+  application_insights_id = azurerm_application_insights.default.id
+  key_vault_id            = azurerm_key_vault.default.id
+  storage_account_id      = azurerm_storage_account.default.id
+  container_registry_id   = azurerm_container_registry.default.id
+
+  identity {
+    type = "SystemAssigned"
+  }
+
+  # Args of use when using an Azure Private Link configuration
+  public_network_access_enabled = false
+  image_build_compute_name      = var.image_build_compute_name
+
+}
+
+# Private endpoints
+resource "azurerm_private_endpoint" "kv_ple" {
+  name                = "ple-${var.name}-${var.environment}-kv"
+  location            = azurerm_resource_group.default.location
+  resource_group_name = azurerm_resource_group.default.name
+  subnet_id           = azurerm_subnet.snet-workspace.id
+
+  private_dns_zone_group {
+    name                 = "private-dns-zone-group"
+    private_dns_zone_ids = [azurerm_private_dns_zone.dnsvault.id]
+  }
+
+  private_service_connection {
+    name                           = "psc-${var.name}-kv"
+    private_connection_resource_id = azurerm_key_vault.default.id
+    subresource_names              = ["vault"]
+    is_manual_connection           = false
+  }
+}
+
+resource "azurerm_private_endpoint" "st_ple_blob" {
+  name                = "ple-${var.name}-${var.environment}-st-blob"
+  location            = azurerm_resource_group.default.location
+  resource_group_name = azurerm_resource_group.default.name
+  subnet_id           = azurerm_subnet.snet-workspace.id
+
+  private_dns_zone_group {
+    name                 = "private-dns-zone-group"
+    private_dns_zone_ids = [azurerm_private_dns_zone.dnsstorageblob.id]
+  }
+
+  private_service_connection {
+    name                           = "psc-${var.name}-st"
+    private_connection_resource_id = azurerm_storage_account.default.id
+    subresource_names              = ["blob"]
+    is_manual_connection           = false
+  }
+}
+
+resource "azurerm_private_endpoint" "storage_ple_file" {
+  name                = "ple-${var.name}-${var.environment}-st-file"
+  location            = azurerm_resource_group.default.location
+  resource_group_name = azurerm_resource_group.default.name
+  subnet_id           = azurerm_subnet.snet-workspace.id
+
+  private_dns_zone_group {
+    name                 = "private-dns-zone-group"
+    private_dns_zone_ids = [azurerm_private_dns_zone.dnsstoragefile.id]
+  }
+
+  private_service_connection {
+    name                           = "psc-${var.name}-st"
+    private_connection_resource_id = azurerm_storage_account.default.id
+    subresource_names              = ["file"]
+    is_manual_connection           = false
+  }
+}
+
+resource "azurerm_private_endpoint" "cr_ple" {
+  name                = "ple-${var.name}-${var.environment}-cr"
+  location            = azurerm_resource_group.default.location
+  resource_group_name = azurerm_resource_group.default.name
+  subnet_id           = azurerm_subnet.snet-workspace.id
+
+  private_dns_zone_group {
+    name                 = "private-dns-zone-group"
+    private_dns_zone_ids = [azurerm_private_dns_zone.dnscontainerregistry.id]
+  }
+
+  private_service_connection {
+    name                           = "psc-${var.name}-cr"
+    private_connection_resource_id = azurerm_container_registry.default.id
+    subresource_names              = ["registry"]
+    is_manual_connection           = false
+  }
+}
+
+resource "azurerm_private_endpoint" "mlw_ple" {
+  name                = "ple-${var.name}-${var.environment}-mlw"
+  location            = azurerm_resource_group.default.location
+  resource_group_name = azurerm_resource_group.default.name
+  subnet_id           = azurerm_subnet.snet-workspace.id
+
+  private_dns_zone_group {
+    name                 = "private-dns-zone-group"
+    private_dns_zone_ids = [azurerm_private_dns_zone.dnsazureml.id, azurerm_private_dns_zone.dnsnotebooks.id]
+  }
+
+  private_service_connection {
+    name                           = "psc-${var.name}-mlw"
+    private_connection_resource_id = azurerm_machine_learning_workspace.default.id
+    subresource_names              = ["amlworkspace"]
+    is_manual_connection           = false
+  }
+}
+
+# Compute cluster for image building required since the workspace is behind a vnet.
+# For more details, see https://docs.microsoft.com/en-us/azure/machine-learning/tutorial-create-secure-workspace#configure-image-builds.
+resource "azurerm_machine_learning_compute_cluster" "image-builder" {
+  name                          = var.image_build_compute_name
+  location                      = azurerm_resource_group.default.location
+  vm_priority                   = "LowPriority"
+  vm_size                       = "Standard_DS2_v2"
+  machine_learning_workspace_id = azurerm_machine_learning_workspace.default.id
+  subnet_resource_id            = azurerm_subnet.snet-training.id
+
+  scale_settings {
+    min_node_count                       = 0
+    max_node_count                       = 3
+    scale_down_nodes_after_idle_duration = "PT15M" # 15 minutes
+  }
+
+  identity {
+    type = "SystemAssigned"
+  }
+}

From e3d2f4db37b998654508f835d300bb56e0eed7a0 Mon Sep 17 00:00:00 2001
From: Dylan Reed <71025787+djr1991@users.noreply.github.com>
Date: Thu, 7 Oct 2021 13:26:44 -0400
Subject: [PATCH 02/20] Add FW rules, spoke DNS and UDRs to FW

---
 .gitignore                                    |   4 +
 .../azure-firewall.tf                         | 390 +++++++++++++++++-
 .../bastion.tf                                |   5 +-
 .../compute.tf                                |   7 +-
 .../network-hub.tf                            |  36 +-
 .../network-spoke.tf                          |  21 +-
 .../workspace.tf                              |   3 +
 7 files changed, 447 insertions(+), 19 deletions(-)

diff --git a/.gitignore b/.gitignore
index e69de29b..ec9e5b1d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -0,0 +1,4 @@
+quickstart/301-machine-learning-hub-spoke-secure/*.terraform.lock.hcl
+quickstart/301-machine-learning-hub-spoke-secure/*.tfstate
+quickstart/301-machine-learning-hub-spoke-secure/.terraform/providers/registry.terraform.io/hashicorp/azurerm/2.79.1/windows_amd64/terraform-provider-azurerm_v2.79.1_x5.exe
+quickstart/301-machine-learning-hub-spoke-secure/.terraform/providers/registry.terraform.io/hashicorp/random/3.1.0/windows_amd64/terraform-provider-random_v3.1.0_x5.exe
diff --git a/quickstart/301-machine-learning-hub-spoke-secure/azure-firewall.tf b/quickstart/301-machine-learning-hub-spoke-secure/azure-firewall.tf
index 23365dd0..fddc61ee 100644
--- a/quickstart/301-machine-learning-hub-spoke-secure/azure-firewall.tf
+++ b/quickstart/301-machine-learning-hub-spoke-secure/azure-firewall.tf
@@ -1,3 +1,25 @@
+
+resource "azurerm_ip_group" "ip_group_hub" {
+  name                = "hub-ipgroup"
+  location            = azurerm_resource_group.hub_rg.location
+  resource_group_name = azurerm_resource_group.hub_rg.name
+  cidrs               = var.vnet_hub_address_space
+}
+
+resource "azurerm_ip_group" "ip_group_spoke" {
+  name                = "mlw-spoke-ipgroup"
+  location            = azurerm_resource_group.hub_rg.location
+  resource_group_name = azurerm_resource_group.hub_rg.name
+  cidrs               = var.vnet_address_space
+}
+
+resource "azurerm_ip_group" "ip_group_dsvm_subnet" {
+  name                = "dsvm-subnet-ipgroup"
+  location            = azurerm_resource_group.hub_rg.location
+  resource_group_name = azurerm_resource_group.hub_rg.name
+  cidrs               = var.jumphost_subnet_address_space
+}
+
 resource "azurerm_public_ip" "azure_firewall" {
     name                        = "pip-azfw"
     location                    = azurerm_resource_group.default.location
@@ -13,10 +35,7 @@ resource "azurerm_firewall_policy" "base_policy" {
   dns {
     proxy_enabled = true
   }
-  depends_on = [
-    azurerm_virtual_network_peering.direction1,
-    azurerm_virtual_network_peering.direction2
-  ]
+  
 }
 resource "azurerm_firewall" "azure_firewall_instance" { 
     name                        = "afw-${var.name}-${var.environment}"
@@ -34,12 +53,15 @@ resource "azurerm_firewall" "azure_firewall_instance" {
       create = "60m"
       delete = "2h"
   }
-  depends_on = [ azurerm_public_ip.azure_firewall,
-  azurerm_firewall_policy.base_policy]
+  depends_on = [ 
+    azurerm_public_ip.azure_firewall,
+    azurerm_subnet.azure_firewall,
+    azurerm_firewall_policy_rule_collection_group.azure_firewall_rules_collection
+    ]
 }
 
 resource "azurerm_monitor_diagnostic_setting" "azure_firewall_instance" {
-  name                        = "diagnostics"
+  name                        = "diagnostics-${var.name}-${var.environment}"
   target_resource_id          = azurerm_firewall.azure_firewall_instance.id
   log_analytics_workspace_id  = azurerm_log_analytics_workspace.default.id
 
@@ -77,4 +99,358 @@ resource "azurerm_monitor_diagnostic_setting" "azure_firewall_instance" {
     }
   }
 
+}
+
+resource "azurerm_firewall_policy_rule_collection_group" "azure_firewall_rules_collection" {
+  name               = "afwp-base-rule-collection-group"
+  firewall_policy_id = azurerm_firewall_policy.base_policy.id
+  priority           = 100
+
+application_rule_collection {
+    name     = "afwp-base-app-rule-collection"
+    priority = 200
+    action   = "Allow"
+
+    rule {
+      name = "dsvm-to-internet"
+      protocols {
+        type = "Https"
+        port = 443
+      }
+      protocols {
+        type = "Http"
+        port= 80
+      }
+      source_ip_groups = [azurerm_ip_group.ip_group_dsvm_subnet.id]      
+      destination_fqdns = ["*"]
+    }
+
+    rule {
+      name = "aks-service-tag"
+      protocols {
+        type = "Https"
+        port = 443
+      }
+      source_ip_groups = [azurerm_ip_group.ip_group_spoke.id]      
+      destination_fqdn_tags = ["AzureKubernetesService"]
+    }
+
+    rule {
+      name = "ubuntu-libraries"
+      protocols {
+        type = "Https"
+        port = 443
+      }
+      source_ip_groups = [azurerm_ip_group.ip_group_spoke.id]      
+      destination_fqdns = ["api.snapcraft.io","motd.ubuntu.com",]
+    }
+
+    rule {
+      name = "microsoft-crls"
+      protocols {
+        type = "Http"
+        port = 80
+      }
+      source_ip_groups = [azurerm_ip_group.ip_group_spoke.id]      
+      destination_fqdns = ["crl.microsoft.com",
+                          "mscrl.microsoft.com",  
+                          "crl3.digicert.com",
+                          "ocsp.digicert.com"]
+    }
+
+    rule {
+      name = "github-rules"
+      protocols {
+        type = "Https"
+        port = 443
+      }
+      source_ip_groups = [azurerm_ip_group.ip_group_spoke.id]      
+      destination_fqdns = ["github.com"]
+    }
+
+    rule {
+      name = "microsoft-metrics-rules"
+      protocols {
+        type = "Https"
+        port = 443
+      }
+      source_ip_groups = [azurerm_ip_group.ip_group_spoke.id]      
+      destination_fqdns = ["*.prod.microsoftmetrics.com"]
+    }
+
+    rule {
+      name = "aks-acs-rules"
+      protocols {
+        type = "Https"
+        port = 443
+      }
+      source_ip_groups = [azurerm_ip_group.ip_group_spoke.id]      
+      destination_fqdns = ["acs-mirror.azureedge.net",
+                           "*.docker.io",
+                           "production.cloudflare.docker.com",
+                          "*.azurecr.io"]
+    }
+
+    rule {
+      name = "microsoft-login-rules"
+      protocols {
+        type = "Https"
+        port = 443
+      }
+      source_ip_groups = [azurerm_ip_group.ip_group_spoke.id]      
+      destination_fqdns = ["login.microsoftonline.com"]
+    }
+
+        rule {
+      name = "graph.windows.net"
+      protocols {
+        type = "Http"
+        port = 80
+      }
+      protocols {
+        type = "Https"
+        port = 443
+      }
+      source_ip_groups = [azurerm_ip_group.ip_group_spoke.id]      
+      destination_fqdns = ["graph.windows.net"]
+    }
+
+    rule {
+      name = "anaconda.com"
+      protocols {
+        type = "Http"
+        port = 80
+      }
+      protocols {
+        type = "Https"
+        port = 443
+      }
+      source_ip_groups = [azurerm_ip_group.ip_group_spoke.id]      
+      destination_fqdns = ["anaconda.com", "*.anaconda.com"]
+    }
+
+    rule {
+      name = "anaconda.org"
+      protocols {
+        type = "Http"
+        port = 80
+      }
+      protocols {
+        type = "Https"
+        port = 443
+      }
+      source_ip_groups = [azurerm_ip_group.ip_group_spoke.id]      
+      destination_fqdns = ["*.anaconda.org"]
+    }
+    
+    rule {
+      name = "pypi.org"
+      protocols {
+        type = "Http"
+        port = 80
+      }
+      protocols {
+        type = "Https"
+        port = 443
+      }
+      source_ip_groups = [azurerm_ip_group.ip_group_spoke.id]      
+      destination_fqdns = ["pypi.org"]
+    }
+
+    rule {
+      name = "cloud.r-project.org"
+      protocols {
+        type = "Http"
+        port = 80
+      }
+      protocols {
+        type = "Https"
+        port = 443
+      }
+      source_ip_groups = [azurerm_ip_group.ip_group_spoke.id]      
+      destination_fqdns = ["cloud.r-project.org"]
+    }
+
+    rule {
+      name = "pytorch.org"
+      protocols {
+        type = "Http"
+        port = 80
+      }
+      protocols {
+        type = "Https"
+        port = 443
+      }
+      source_ip_groups = [azurerm_ip_group.ip_group_spoke.id]      
+      destination_fqdns = ["*pytorch.org"]
+    }
+
+    rule {
+      name = "tensorflow.org"
+      protocols {
+        type = "Http"
+        port = 80
+      }
+      protocols {
+        type = "Https"
+        port = 443
+      }
+      source_ip_groups = [azurerm_ip_group.ip_group_spoke.id]      
+      destination_fqdns = ["*.tensorflow.org"]
+    }
+
+    rule {
+      name = "update.code.visualstudio.com"
+      protocols {
+        type = "Http"
+        port = 80
+      }
+      protocols {
+        type = "Https"
+        port = 443
+      }
+      source_ip_groups = [azurerm_ip_group.ip_group_spoke.id]      
+      destination_fqdns = ["update.code.visualstudio.com", "*.vo.msecnd.net"]
+    }
+
+    rule {
+      name = "dc.applicationinsights.azure.com"
+      protocols {
+        type = "Http"
+        port = 80
+      }
+      protocols {
+        type = "Https"
+        port = 443
+      }
+      source_ip_groups = [azurerm_ip_group.ip_group_spoke.id]      
+      destination_fqdns = ["dc.applicationinsights.azure.com"]
+    }
+
+    rule {
+      name = "dc.applicationinsights.microsoft.com"
+      protocols {
+        type = "Http"
+        port = 80
+      }
+      protocols {
+        type = "Https"
+        port = 443
+      }
+      source_ip_groups = [azurerm_ip_group.ip_group_spoke.id]      
+      destination_fqdns = ["dc.applicationinsights.microsoft.com"]
+    }
+
+    rule {
+      name = "dc.services.visualstudio.com"
+      protocols {
+        type = "Http"
+        port = 80
+      }
+      protocols {
+        type = "Https"
+        port = 443
+      }
+      source_ip_groups = [azurerm_ip_group.ip_group_spoke.id]      
+      destination_fqdns = ["dc.services.visualstudio.com"]
+    }
+  }
+
+ network_rule_collection {
+    name     = "afwp-base-network-rule-collection"
+    priority = 100
+    action   = "Allow"
+
+    rule {
+      name                  = "hub-to-spoke-rule"
+      protocols             = ["Any"]
+      source_ip_groups      = [azurerm_ip_group.ip_group_spoke.id]   
+      destination_ip_groups = [azurerm_ip_group.ip_group_hub.id]
+      destination_ports     = ["*"]
+    }
+
+     rule {
+      name                  = "aks-global-network-rule"
+      protocols             = ["TCP"]
+      source_ip_groups      = [azurerm_ip_group.ip_group_spoke.id]      
+      destination_addresses = ["AzureCloud"]
+      destination_ports     = ["443", "9000"]
+    }
+
+    rule {
+      name                  = "aks-ntp-network-rule"
+      protocols             = ["UDP"]
+      source_ip_groups      = [azurerm_ip_group.ip_group_spoke.id]      
+      destination_addresses = ["*"]
+      destination_ports     = ["123"]
+    }
+
+    rule {
+      name                  = "Azure-Active-Directory"
+      protocols             = ["TCP"]
+      source_ip_groups      = [azurerm_ip_group.ip_group_spoke.id]      
+      destination_addresses = ["AzureActiveDirectory"]
+      destination_ports     = ["*"]
+    }
+
+    rule {
+      name                  = "Azure-Machine-Learning"
+      protocols             = ["TCP"]
+      source_ip_groups      = [azurerm_ip_group.ip_group_spoke.id]      
+      destination_addresses = ["AzureMachineLearning"]
+      destination_ports     = ["443"]
+    }
+
+    rule {
+      name                  = "Azure-Resource-Manager"
+      protocols             = ["TCP"]
+      source_ip_groups      = [azurerm_ip_group.ip_group_spoke.id]      
+      destination_addresses = ["AzureResourceManager"]
+      destination_ports     = ["443"]
+    }
+
+    rule {
+      name                  = "Azure-Storage"
+      protocols             = ["TCP"]
+      source_ip_groups      = [azurerm_ip_group.ip_group_spoke.id]      
+      destination_addresses = ["Storage"]
+      destination_ports     = ["443"]
+    }
+
+    rule {
+      name                  = "Azure-Front-Door-Frontend"
+      protocols             = ["TCP"]
+      source_ip_groups      = [azurerm_ip_group.ip_group_spoke.id]      
+      destination_addresses = ["AzureFrontDoor.Frontend"]
+      destination_ports     = ["443"]
+    }
+
+    rule {
+      name                  = "Azure-Container-Registry"
+      protocols             = ["TCP"]
+      source_ip_groups      = [azurerm_ip_group.ip_group_spoke.id]      
+      destination_addresses = ["AzureContainerRegistry"]
+      destination_ports     = ["443"]
+    }
+
+    rule {
+      name                  = "Azure-Key-Vault"
+      protocols             = ["TCP"]
+      source_ip_groups      = [azurerm_ip_group.ip_group_spoke.id]      
+      destination_addresses = ["AzureKeyVault"]
+      destination_ports     = ["443"]
+    }
+
+    rule {
+      name                  = "Microsoft-Container-Registry"
+      protocols             = ["TCP"]
+      source_ip_groups      = [azurerm_ip_group.ip_group_spoke.id]      
+      destination_addresses = ["MicrosoftContainerRegistry"]
+      destination_ports     = ["443"]
+    }
+ }
+ depends_on = [
+   azurerm_ip_group.ip_group_hub,
+   azurerm_ip_group.ip_group_spoke
+ ]
 }
\ No newline at end of file
diff --git a/quickstart/301-machine-learning-hub-spoke-secure/bastion.tf b/quickstart/301-machine-learning-hub-spoke-secure/bastion.tf
index e83f90d8..89c81847 100644
--- a/quickstart/301-machine-learning-hub-spoke-secure/bastion.tf
+++ b/quickstart/301-machine-learning-hub-spoke-secure/bastion.tf
@@ -105,7 +105,10 @@ resource "azurerm_network_security_group" "bastion_nsg" {
 resource "azurerm_subnet_network_security_group_association" "bastion_nsg_assoc" {
   subnet_id                 = azurerm_subnet.azure_bastion.id
   network_security_group_id = azurerm_network_security_group.bastion_nsg.id
-  depends_on = [ azurerm_bastion_host.azure_bastion_instance ]
+  depends_on = [ 
+    azurerm_bastion_host.azure_bastion_instance,
+    azurerm_subnet_network_security_group_association.jumphost_nsg_assoc
+    ]
 }
 
 
diff --git a/quickstart/301-machine-learning-hub-spoke-secure/compute.tf b/quickstart/301-machine-learning-hub-spoke-secure/compute.tf
index b95541b8..520031a7 100644
--- a/quickstart/301-machine-learning-hub-spoke-secure/compute.tf
+++ b/quickstart/301-machine-learning-hub-spoke-secure/compute.tf
@@ -6,7 +6,7 @@ resource "random_string" "ci_prefix" {
   number  = false
 }
 
-/* # Compute instance
+# Compute instance
 resource "azurerm_machine_learning_compute_instance" "compute_instance" {
   name                          = "${random_string.ci_prefix.result}instance"
   location                      = azurerm_resource_group.default.location
@@ -18,7 +18,7 @@ resource "azurerm_machine_learning_compute_instance" "compute_instance" {
     azurerm_private_endpoint.mlw_ple
   ]
 }
-*/
+
 # Compute cluster
 resource "azurerm_machine_learning_compute_cluster" "compute" {
   name                          = "cpu-cluster"
@@ -37,5 +37,8 @@ resource "azurerm_machine_learning_compute_cluster" "compute" {
     max_node_count                       = 3
     scale_down_nodes_after_idle_duration = "PT15M" # 15 minutes
   }
+  depends_on = [
+    azurerm_private_endpoint.mlw_ple
+  ]
 
 }
\ No newline at end of file
diff --git a/quickstart/301-machine-learning-hub-spoke-secure/network-hub.tf b/quickstart/301-machine-learning-hub-spoke-secure/network-hub.tf
index 94c3d5b1..cf0bb055 100644
--- a/quickstart/301-machine-learning-hub-spoke-secure/network-hub.tf
+++ b/quickstart/301-machine-learning-hub-spoke-secure/network-hub.tf
@@ -42,7 +42,11 @@ resource "azurerm_virtual_network_peering" "direction1" {
   allow_forwarded_traffic      = false
   allow_gateway_transit        = false
   use_remote_gateways          = false
-  
+  depends_on = [
+    azurerm_virtual_network.hub,
+    azurerm_virtual_network.default
+  ]
+
 }
 
 resource "azurerm_virtual_network_peering" "direction2" {
@@ -54,7 +58,11 @@ resource "azurerm_virtual_network_peering" "direction2" {
   allow_forwarded_traffic      = false
   allow_gateway_transit        = false
   use_remote_gateways          = false
-
+  depends_on = [
+    azurerm_virtual_network.hub,
+    azurerm_virtual_network.default
+  ]
+  
 }
 
 # Private DNS Zones
@@ -141,4 +149,28 @@ resource "azurerm_network_security_group" "jump_host" {
 resource "azurerm_subnet_network_security_group_association" "jumphost_nsg_assoc" {
   subnet_id                 = azurerm_subnet.snet-jumphost.id 
   network_security_group_id = azurerm_network_security_group.jump_host.id
+  depends_on = [
+    azurerm_network_interface.dsvm
+  ]
 }
+
+# Route Table for Jump host subnet
+resource "azurerm_route_table" "jumphost_rt" {
+  name                = "rt-jumphost"
+  location            = azurerm_resource_group.default.location
+  resource_group_name = azurerm_resource_group.default.name
+}
+
+resource "azurerm_route" "jumphost-fw-route" {
+  name                = "udr-Default"
+  resource_group_name = azurerm_resource_group.default.name
+  route_table_name    = azurerm_route_table.jumphost_rt.name
+  address_prefix      = "0.0.0.0/0"
+  next_hop_type       = "VirtualAppliance"
+  next_hop_in_ip_address = azurerm_firewall.azure_firewall_instance.ip_configuration[0].private_ip_address
+}
+
+resource "azurerm_subnet_route_table_association" "rt-jumphost-link" {
+  subnet_id      = azurerm_subnet.snet-jumphost.id
+  route_table_id = azurerm_route_table.jumphost_rt.id
+}
\ No newline at end of file
diff --git a/quickstart/301-machine-learning-hub-spoke-secure/network-spoke.tf b/quickstart/301-machine-learning-hub-spoke-secure/network-spoke.tf
index 0d5568d3..75a98046 100644
--- a/quickstart/301-machine-learning-hub-spoke-secure/network-spoke.tf
+++ b/quickstart/301-machine-learning-hub-spoke-secure/network-spoke.tf
@@ -4,6 +4,11 @@ resource "azurerm_virtual_network" "default" {
   address_space       = var.vnet_address_space
   location            = azurerm_resource_group.default.location
   resource_group_name = azurerm_resource_group.default.name
+  dns_servers         = [azurerm_firewall.azure_firewall_instance.ip_configuration[0].private_ip_address]
+  depends_on = [
+    azurerm_virtual_network.hub,
+    azurerm_firewall.azure_firewall_instance
+  ]
 }
 
 resource "azurerm_subnet" "snet-training" {
@@ -88,15 +93,16 @@ resource "azurerm_route_table" "rt-training" {
 }
 
 resource "azurerm_route" "training-Internet-Route" {
-  name                = "Internet"
+  name                = "udr-Default"
   resource_group_name = azurerm_resource_group.default.name
   route_table_name    = azurerm_route_table.rt-training.name
   address_prefix      = "0.0.0.0/0"
-  next_hop_type       = "Internet"
+  next_hop_type       = "VirtualAppliance"
+  next_hop_in_ip_address = azurerm_firewall.azure_firewall_instance.ip_configuration[0].private_ip_address
 }
 
 resource "azurerm_route" "training-AzureMLRoute" {
-  name                = "AzureMLRoute"
+  name                = "udr-AzureML"
   resource_group_name = azurerm_resource_group.default.name
   route_table_name    = azurerm_route_table.rt-training.name
   address_prefix      = "AzureMachineLearning"
@@ -104,7 +110,7 @@ resource "azurerm_route" "training-AzureMLRoute" {
 }
 
 resource "azurerm_route" "training-BatchRoute" {
-  name                = "BatchRoute"
+  name                = "udr-Batch"
   resource_group_name = azurerm_resource_group.default.name
   route_table_name    = azurerm_route_table.rt-training.name
   address_prefix      = "BatchNodeManagement"
@@ -123,12 +129,13 @@ resource "azurerm_route_table" "rt-aks" {
   resource_group_name = azurerm_resource_group.default.name
 }
 
-resource "azurerm_route" "aks-Internet-Route" {
-  name                = "Internet"
+resource "azurerm_route" "aks-default-Route" {
+  name                = "udr-Default"
   resource_group_name = azurerm_resource_group.default.name
   route_table_name    = azurerm_route_table.rt-aks.name
   address_prefix      = "0.0.0.0/0"
-  next_hop_type       = "Internet"
+  next_hop_type       = "VirtualAppliance"
+  next_hop_in_ip_address = azurerm_firewall.azure_firewall_instance.ip_configuration[0].private_ip_address
 }
 
 resource "azurerm_subnet_route_table_association" "rt-aks-link" {
diff --git a/quickstart/301-machine-learning-hub-spoke-secure/workspace.tf b/quickstart/301-machine-learning-hub-spoke-secure/workspace.tf
index f9be016d..b749f10a 100644
--- a/quickstart/301-machine-learning-hub-spoke-secure/workspace.tf
+++ b/quickstart/301-machine-learning-hub-spoke-secure/workspace.tf
@@ -182,4 +182,7 @@ resource "azurerm_machine_learning_compute_cluster" "image-builder" {
   identity {
     type = "SystemAssigned"
   }
+  depends_on = [
+    azurerm_private_endpoint.mlw_ple
+  ]
 }

From 7cfe24f0ffc53b6fbcb5c37a44f0aaeb0ec0617a Mon Sep 17 00:00:00 2001
From: Dylan Reed <71025787+djr1991@users.noreply.github.com>
Date: Thu, 7 Oct 2021 15:55:47 -0400
Subject: [PATCH 03/20] add missing fw rules for ml

---
 .gitignore                                    |  2 ++
 .../azure-firewall.tf                         | 20 ++++++++++++++++---
 2 files changed, 19 insertions(+), 3 deletions(-)

diff --git a/.gitignore b/.gitignore
index ec9e5b1d..0d6b95a3 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,3 +2,5 @@ quickstart/301-machine-learning-hub-spoke-secure/*.terraform.lock.hcl
 quickstart/301-machine-learning-hub-spoke-secure/*.tfstate
 quickstart/301-machine-learning-hub-spoke-secure/.terraform/providers/registry.terraform.io/hashicorp/azurerm/2.79.1/windows_amd64/terraform-provider-azurerm_v2.79.1_x5.exe
 quickstart/301-machine-learning-hub-spoke-secure/.terraform/providers/registry.terraform.io/hashicorp/random/3.1.0/windows_amd64/terraform-provider-random_v3.1.0_x5.exe
+quickstart/301-machine-learning-hub-spoke-secure/.terraform.tfstate.lock.info
+quickstart/301-machine-learning-hub-spoke-secure/terraform.tfstate.*
diff --git a/quickstart/301-machine-learning-hub-spoke-secure/azure-firewall.tf b/quickstart/301-machine-learning-hub-spoke-secure/azure-firewall.tf
index fddc61ee..c5ab0b41 100644
--- a/quickstart/301-machine-learning-hub-spoke-secure/azure-firewall.tf
+++ b/quickstart/301-machine-learning-hub-spoke-secure/azure-firewall.tf
@@ -354,6 +354,20 @@ application_rule_collection {
       source_ip_groups = [azurerm_ip_group.ip_group_spoke.id]      
       destination_fqdns = ["dc.services.visualstudio.com"]
     }
+
+    rule {
+      name = "azureml-instances"
+      protocols {
+        type = "Http"
+        port = 80
+      }
+      protocols {
+        type = "Https"
+        port = 443
+      }
+      source_ip_groups = [azurerm_ip_group.ip_group_spoke.id]      
+      destination_fqdns = ["*.instances.azureml.net", "*.instances.azureml.ms"]
+    }
   }
 
  network_rule_collection {
@@ -364,8 +378,8 @@ application_rule_collection {
     rule {
       name                  = "hub-to-spoke-rule"
       protocols             = ["Any"]
-      source_ip_groups      = [azurerm_ip_group.ip_group_spoke.id]   
-      destination_ip_groups = [azurerm_ip_group.ip_group_hub.id]
+      source_ip_groups      = [azurerm_ip_group.ip_group_spoke.id,azurerm_ip_group.ip_group_hub.id]   
+      destination_ip_groups = [azurerm_ip_group.ip_group_hub.id,azurerm_ip_group.ip_group_spoke.id]
       destination_ports     = ["*"]
     }
 
@@ -421,7 +435,7 @@ application_rule_collection {
       name                  = "Azure-Front-Door-Frontend"
       protocols             = ["TCP"]
       source_ip_groups      = [azurerm_ip_group.ip_group_spoke.id]      
-      destination_addresses = ["AzureFrontDoor.Frontend"]
+      destination_addresses = ["AzureFrontDoor.Frontend","AzureFrontDoor.FirstParty"]
       destination_ports     = ["443"]
     }
 

From 54355dcdc32a3f9078c7e9af3d3ca676a6e73e6d Mon Sep 17 00:00:00 2001
From: David Apolinar <david.apolinar@hotmail.com>
Date: Thu, 14 Oct 2021 17:41:42 -0400
Subject: [PATCH 04/20] added dependency for Azure Firewall

---
 quickstart/301-machine-learning-hub-spoke-secure/workspace.tf | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/quickstart/301-machine-learning-hub-spoke-secure/workspace.tf b/quickstart/301-machine-learning-hub-spoke-secure/workspace.tf
index b749f10a..460c38cd 100644
--- a/quickstart/301-machine-learning-hub-spoke-secure/workspace.tf
+++ b/quickstart/301-machine-learning-hub-spoke-secure/workspace.tf
@@ -64,6 +64,9 @@ resource "azurerm_machine_learning_workspace" "default" {
   # Args of use when using an Azure Private Link configuration
   public_network_access_enabled = false
   image_build_compute_name      = var.image_build_compute_name
+  depends_on = [
+    azurerm_firewall.azure_firewall_instance
+  ]
 
 }
 

From 2b9b074b9cc7c6b4bfd8e68fd18bebb0db825abb Mon Sep 17 00:00:00 2001
From: ryhud <rhudson@microsoft.com>
Date: Mon, 25 Oct 2021 10:45:29 -0400
Subject: [PATCH 05/20] adding Azure Bastion and DSVM

---
 .../bastion.tf                                | 125 ++++++++++++++++++
 .../dsvm.tf                                   |  52 ++++++++
 .../network.tf                                |  15 +++
 .../variables.tf                              |  29 ++++
 4 files changed, 221 insertions(+)
 create mode 100644 quickstart/201-machine-learning-moderately-secure/bastion.tf
 create mode 100644 quickstart/201-machine-learning-moderately-secure/dsvm.tf

diff --git a/quickstart/201-machine-learning-moderately-secure/bastion.tf b/quickstart/201-machine-learning-moderately-secure/bastion.tf
new file mode 100644
index 00000000..d8cae3d8
--- /dev/null
+++ b/quickstart/201-machine-learning-moderately-secure/bastion.tf
@@ -0,0 +1,125 @@
+resource "azurerm_public_ip" "azure_bastion" { 
+    name                        = "pip-azure-bastion"
+    location                    = azurerm_resource_group.default.location
+    resource_group_name         = azurerm_resource_group.default.name
+    allocation_method           = "Static"
+    sku                         = "Standard" 
+}
+
+resource "azurerm_network_security_group" "bastion_nsg" { 
+    name                        = "nsg-bastion"
+    location                    = azurerm_resource_group.default.location
+    resource_group_name         = azurerm_resource_group.default.name
+
+    security_rule {      
+      name                        = "AllowHTTPSInbound"
+      priority                    = 100
+      direction                   = "Inbound"
+      access                      = "Allow"
+      protocol                    = "Tcp"
+      source_port_range           = "*"
+      destination_port_range      = "443"
+      source_address_prefix       = "Internet"
+      destination_address_prefix  = "*"
+    }
+    security_rule {
+      name                        = "AllowGatewayManagerInbound"
+      priority                    = 200
+      direction                   = "Inbound"
+      access                      = "Allow"
+      protocol                    = "Tcp"
+      source_port_range           = "*"
+      destination_port_range      = "443"
+      source_address_prefix       = "GatewayManager"
+      destination_address_prefix  = "*"
+    }
+    security_rule {
+      name                        = "AllowAzureLBInbound"
+      priority                    = 300
+      direction                   = "Inbound"
+      access                      = "Allow"
+      protocol                    = "Tcp"
+      source_port_range           = "*"
+      destination_port_range      = "443"
+      source_address_prefix       = "AzureLoadBalancer"
+      destination_address_prefix  = "*"
+    }
+    security_rule {
+      name                        = "AllowBastionHostCommunication"
+      priority                    = 400
+      direction                   = "Inbound"
+      access                      = "Allow"
+      protocol                    = "*"
+      source_port_range           = "*"
+      destination_port_ranges      = ["5701","8080"]
+      source_address_prefix       = "VirtualNetwork"
+      destination_address_prefix  = "VirtualNetwork"
+    }
+    security_rule {
+      name                        = "AllowRdpSshOutbound"
+      priority                    = 100
+      direction                   = "Outbound"
+      access                      = "Allow"
+      protocol                    = "Tcp"
+      source_port_range           = "*"
+      destination_port_ranges      = ["22", "3389"]
+      source_address_prefix       = "*"
+      destination_address_prefix  = "VirtualNetwork"
+    }
+      security_rule {
+      name                        = "AllowBastionHostCommunicationOutbound"
+      priority                    = 110
+      direction                   = "Outbound"
+      access                      = "Allow"
+      protocol                    = "Tcp"
+      source_port_range           = "*"
+      destination_port_ranges      = ["5701", "8080"]
+      source_address_prefix       = "VirtualNetwork"
+      destination_address_prefix  = "VirtualNetwork"
+  }
+    security_rule {
+      name                        = "AllowAzureCloudOutbound"
+      priority                    = 120
+      direction                   = "Outbound"
+      access                      = "Allow"
+      protocol                    = "Tcp"
+      source_port_range           = "*"
+      destination_port_ranges      = ["443"]
+      source_address_prefix       = "*"
+      destination_address_prefix  = "AzureCloud"
+    }
+    security_rule {
+      name                        = "AllowGetSessionInformation"
+      priority                    = 130
+      direction                   = "Outbound"
+      access                      = "Allow"
+      protocol                    = "Tcp"
+      source_port_range           = "*"
+      destination_port_ranges      = ["80"]
+      source_address_prefix       = "*"
+      destination_address_prefix  = "Internet"
+}
+    
+}
+
+resource "azurerm_subnet_network_security_group_association" "bastion_nsg_assoc" {
+  subnet_id                 = azurerm_subnet.azure_bastion.id
+  network_security_group_id = azurerm_network_security_group.bastion_nsg.id
+  depends_on = [ 
+    azurerm_bastion_host.azure_bastion_instance    
+    ]
+}
+
+
+resource "azurerm_bastion_host" "azure_bastion_instance" {
+    name                        = "bas-${var.name}-${var.environment}"
+    location                    = azurerm_resource_group.default.location
+    resource_group_name         = azurerm_resource_group.default.name
+
+    ip_configuration { 
+        name                    = "configuration"
+        subnet_id               = azurerm_subnet.azure_bastion.id
+        public_ip_address_id    = azurerm_public_ip.azure_bastion.id 
+    }
+}
+
diff --git a/quickstart/201-machine-learning-moderately-secure/dsvm.tf b/quickstart/201-machine-learning-moderately-secure/dsvm.tf
new file mode 100644
index 00000000..0ad12aa8
--- /dev/null
+++ b/quickstart/201-machine-learning-moderately-secure/dsvm.tf
@@ -0,0 +1,52 @@
+resource "azurerm_network_interface" "dsvm" { 
+    name                              = "nic-${var.dsvm_name}"
+    location                          = azurerm_resource_group.default.location
+    resource_group_name               = azurerm_resource_group.default.name
+    
+    ip_configuration { 
+        name                          = "configuration"
+        subnet_id                     = azurerm_subnet.snet-dsvm.id 
+        private_ip_address_allocation = "Dynamic"        
+    }
+    /*depends_on = [
+      azurerm_route_table.jumphost_rt
+    ]
+    */
+}
+
+resource "azurerm_windows_virtual_machine" "dsvm" {
+  name                  = var.dsvm_name
+  location              = azurerm_resource_group.default.location
+  resource_group_name   = azurerm_resource_group.default.name
+  network_interface_ids = [
+        azurerm_network_interface.dsvm.id
+    ]
+  size               = "Standard_DS3_v2"
+ 
+  source_image_reference {
+    publisher = "microsoft-dsvm"
+    offer     = "dsvm-win-2019"
+    sku       = "server-2019"
+    version   = "latest"
+  }
+ 
+  os_disk {
+    name              = "osdisk-${var.dsvm_name}"
+    caching           = "ReadWrite"  
+    storage_account_type = "Premium_LRS"
+  }
+ 
+  identity {
+    type = "SystemAssigned"
+  }
+  computer_name      = var.dsvm_name
+  admin_username     = var.dsvm_admin_username
+  admin_password     = var.dsvm_host_password
+  
+  provision_vm_agent = true
+ 
+  timeouts {
+      create = "60m"
+      delete = "2h"
+  }
+}
diff --git a/quickstart/201-machine-learning-moderately-secure/network.tf b/quickstart/201-machine-learning-moderately-secure/network.tf
index 0e56d74d..11bf0c28 100644
--- a/quickstart/201-machine-learning-moderately-secure/network.tf
+++ b/quickstart/201-machine-learning-moderately-secure/network.tf
@@ -30,6 +30,21 @@ resource "azurerm_subnet" "snet-workspace" {
   enforce_private_link_endpoint_network_policies = true
 }
 
+resource "azurerm_subnet" "snet-dsvm" {
+  name                                           = "snet-dsvm"
+  resource_group_name                            = azurerm_resource_group.default.name
+  virtual_network_name                           = azurerm_virtual_network.default.name
+  address_prefixes                               = var.dsvm_subnet_address_space
+  enforce_private_link_endpoint_network_policies = true
+}
+
+resource "azurerm_subnet" "azure_bastion" {
+    name                        = "AzureBastionSubnet"
+    resource_group_name         = azurerm_resource_group.default.name
+    virtual_network_name        = azurerm_virtual_network.default.name
+    address_prefixes            = var.bastion_subnet_address_space
+}
+
 # Private DNS Zones
 resource "azurerm_private_dns_zone" "dnsvault" {
   name                = "privatelink.vaultcore.azure.net"
diff --git a/quickstart/201-machine-learning-moderately-secure/variables.tf b/quickstart/201-machine-learning-moderately-secure/variables.tf
index 6a67c802..fb8299d2 100644
--- a/quickstart/201-machine-learning-moderately-secure/variables.tf
+++ b/quickstart/201-machine-learning-moderately-secure/variables.tf
@@ -38,9 +38,38 @@ variable "ml_subnet_address_space" {
   description = "Address space of the ML workspace subnet"
   default     = ["10.0.0.0/24"]
 }
+variable "dsvm_subnet_address_space" {
+  type        = list(string)
+  description = "Address space of the DSVM subnet"
+  default     = ["10.0.4.0/24"]
+}
+
+variable "bastion_subnet_address_space" {
+  type        = list(string)
+  description = "Address space of the bastion subnet"
+  default     = ["10.0.5.0/24"]
+}
 
 variable "image_build_compute_name" {
   type        = string
   description = "Name of the compute cluster to be created and set to build docker images"
   default     = "image-builder"
+}
+
+# DSVM Variables
+variable "dsvm_name" {
+  type        = string
+  description = "Name of the Data Science VM"
+  default     = "vmdsvm01"
+}
+variable "dsvm_admin_username" {
+  type        = string
+  description = "Admin username of the Data Science VM"
+  default     = "azureadmin"
+}
+
+variable "dsvm_host_password" {
+  type        = string
+  description = "Password for the admin username of the Data Science VM"
+  sensitive   = true
 }
\ No newline at end of file

From a1d13658a39efdc2cfd8bfb6eecdfb819cb17eb1 Mon Sep 17 00:00:00 2001
From: Dylan Reed <71025787+djr1991@users.noreply.github.com>
Date: Mon, 25 Oct 2021 19:10:15 -0400
Subject: [PATCH 06/20] random string for fw diagnostics + missing FW rule

---
 .../azure-firewall.tf                         | 24 +++++++++++++++++--
 1 file changed, 22 insertions(+), 2 deletions(-)

diff --git a/quickstart/301-machine-learning-hub-spoke-secure/azure-firewall.tf b/quickstart/301-machine-learning-hub-spoke-secure/azure-firewall.tf
index c5ab0b41..4c83703d 100644
--- a/quickstart/301-machine-learning-hub-spoke-secure/azure-firewall.tf
+++ b/quickstart/301-machine-learning-hub-spoke-secure/azure-firewall.tf
@@ -1,4 +1,10 @@
-
+# Generate random string for unique firewall diagnostic name
+resource "random_string" "fw_diag_prefix" {
+  length  = 8
+  upper   = false
+  special = false
+  number  = false
+}
 resource "azurerm_ip_group" "ip_group_hub" {
   name                = "hub-ipgroup"
   location            = azurerm_resource_group.hub_rg.location
@@ -61,7 +67,7 @@ resource "azurerm_firewall" "azure_firewall_instance" {
 }
 
 resource "azurerm_monitor_diagnostic_setting" "azure_firewall_instance" {
-  name                        = "diagnostics-${var.name}-${var.environment}"
+  name                        = "diagnostics-${var.name}-${var.environment}-${random_string.fw_diag_prefix.result}"
   target_resource_id          = azurerm_firewall.azure_firewall_instance.id
   log_analytics_workspace_id  = azurerm_log_analytics_workspace.default.id
 
@@ -168,6 +174,20 @@ application_rule_collection {
       destination_fqdns = ["github.com"]
     }
 
+    rule {
+      name = "raw.githubusercontent.com"
+      protocols {
+        type = "Https"
+        port = 443
+      }
+      protocols {
+        type = "Http"
+        port = 80
+      }
+      source_ip_groups = [azurerm_ip_group.ip_group_spoke.id]      
+      destination_fqdns = ["raw.githubusercontent.com"]
+    }
+
     rule {
       name = "microsoft-metrics-rules"
       protocols {

From 8e503110f91a7d570e720b09d9c2761b65ddb848 Mon Sep 17 00:00:00 2001
From: ryhud <rhudson@microsoft.com>
Date: Tue, 26 Oct 2021 10:11:56 -0400
Subject: [PATCH 07/20] Updating AML 201 readme

---
 .../201-machine-learning-moderately-secure/readme.md       | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/quickstart/201-machine-learning-moderately-secure/readme.md b/quickstart/201-machine-learning-moderately-secure/readme.md
index 1d206849..b4a9f376 100644
--- a/quickstart/201-machine-learning-moderately-secure/readme.md
+++ b/quickstart/201-machine-learning-moderately-secure/readme.md
@@ -6,13 +6,15 @@ and its associated resources including Azure Key Vault, Azure Storage, Azure App
 In addition to these core services, this configuration specifies any networking components that are required to set up Azure Machine Learning
 for private network connectivity using [Azure Private Link](https://docs.microsoft.com/en-us/azure/private-link/). 
 
-This configuration describes the minimal set of resources you require to get started with Azure Machine Learning in a network-isolated set-up. This configuration creates new network components. If you want to reuse existing network components, see [202 example](../201-machine-learning-moderately-secure/readme.md).
+This configuration describes the minimal set of resources you require to get started with Azure Machine Learning in a network-isolated set-up. This configuration creates new network components. Use Azure Bastion to securely connect to the Windows Data Science Virtual Machine. If you want to reuse existing network components, see [202 example](../201-machine-learning-moderately-secure/readme.md).
 
 ## Resources
 
 | Terraform Resource Type | Description |
 | - | - |
 | `azurerm_resource_group` | The resource group all resources get deployed into |
+| `azurerm_bastion_host` | An Azure Bastion Instance to securely RDP/SSH into Virtual Machines deployed into the Virtual Network |
+| `azurerm_windows_virtual_machine` | A Windows Data Science Virtual Machine used for connecting to the Azure Machine Learning workspace |
 | `azurerm_application_insights` | An Azure Application Insights instance associated to the Azure Machine Learning workspace |
 | `azurerm_key_vault` | An Azure Key Vault instance associated to the Azure Machine Learning workspace |
 | `azurerm_storage_account` | An Azure Storage instance associated to the Azure Machine Learning workspace |
@@ -39,6 +41,9 @@ This configuration describes the minimal set of resources you require to get sta
 | aks_subnet_address_space | Address space of the aks subnet | ["10.0.2.0/23"] |
 | ml_subnet_address_space | Address space of the ML workspace subnet | ["10.0.0.0/24"] |
 | image_build_compute_name | Name of the compute cluster to be created and configured for building docker images (Azure ML Environments) | image-builder |
+| dsvm_name | Name of the Windows Data Science VM resource | vmdsvm01 |
+| dsvm_admin_username | Admin username of the Windows Data Science VM | azureadmin |
+| dsvm_host_password | Password for the admin username of the Data Science VM | - |
 
 
 ## Usage

From e6cbe158b7c571adad124221c74e26d87d18ec16 Mon Sep 17 00:00:00 2001
From: ryhud <rhudson@microsoft.com>
Date: Thu, 28 Oct 2021 11:05:39 -0400
Subject: [PATCH 08/20] adding sensitive to DSVM password variable

---
 quickstart/301-machine-learning-hub-spoke-secure/variables.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/quickstart/301-machine-learning-hub-spoke-secure/variables.tf b/quickstart/301-machine-learning-hub-spoke-secure/variables.tf
index 7d0ef6b1..9618d98c 100644
--- a/quickstart/301-machine-learning-hub-spoke-secure/variables.tf
+++ b/quickstart/301-machine-learning-hub-spoke-secure/variables.tf
@@ -89,5 +89,5 @@ variable "dsvm_admin_username" {
 variable "dsvm_host_password" {
   type        = string
   description = "Password for the admin username of the Data Science VM"
-  
+  sensitive   = true
 }
\ No newline at end of file

From 0943dd056804c7760e7c126ded3c102f153076b5 Mon Sep 17 00:00:00 2001
From: ryhud <rhudson@microsoft.com>
Date: Thu, 28 Oct 2021 11:35:41 -0400
Subject: [PATCH 09/20] adding resource dependancy to fix destroy issue

---
 .../201-machine-learning-moderately-secure/workspace.tf    | 7 +++++++
 .../301-machine-learning-hub-spoke-secure/workspace.tf     | 7 ++++++-
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/quickstart/201-machine-learning-moderately-secure/workspace.tf b/quickstart/201-machine-learning-moderately-secure/workspace.tf
index 34b24334..dc0036cc 100644
--- a/quickstart/201-machine-learning-moderately-secure/workspace.tf
+++ b/quickstart/201-machine-learning-moderately-secure/workspace.tf
@@ -63,6 +63,13 @@ resource "azurerm_machine_learning_workspace" "default" {
   # Args of use when using an Azure Private Link configuration
   public_network_access_enabled = false
   image_build_compute_name      = var.image_build_compute_name
+  depends_on = [    
+    azurerm_private_endpoint.kv_ple,
+    azurerm_private_endpoint.st_ple_blob,
+    azurerm_private_endpoint.storage_ple_file,
+    azurerm_private_endpoint.cr_ple,
+    azurerm_subnet.snet-training
+  ]
 
 }
 
diff --git a/quickstart/301-machine-learning-hub-spoke-secure/workspace.tf b/quickstart/301-machine-learning-hub-spoke-secure/workspace.tf
index 460c38cd..ddf72ac1 100644
--- a/quickstart/301-machine-learning-hub-spoke-secure/workspace.tf
+++ b/quickstart/301-machine-learning-hub-spoke-secure/workspace.tf
@@ -65,7 +65,12 @@ resource "azurerm_machine_learning_workspace" "default" {
   public_network_access_enabled = false
   image_build_compute_name      = var.image_build_compute_name
   depends_on = [
-    azurerm_firewall.azure_firewall_instance
+    azurerm_firewall.azure_firewall_instance,
+    azurerm_private_endpoint.kv_ple,
+    azurerm_private_endpoint.st_ple_blob,
+    azurerm_private_endpoint.storage_ple_file,
+    azurerm_private_endpoint.cr_ple,
+    azurerm_subnet.snet-training
   ]
 
 }

From 3adc707d420ad5694514605440e040117f8c3c28 Mon Sep 17 00:00:00 2001
From: ryhud <rhudson@microsoft.com>
Date: Thu, 28 Oct 2021 11:51:42 -0400
Subject: [PATCH 10/20] removing comments

---
 quickstart/201-machine-learning-moderately-secure/dsvm.tf | 6 +-----
 quickstart/301-machine-learning-hub-spoke-secure/dsvm.tf  | 6 +-----
 2 files changed, 2 insertions(+), 10 deletions(-)

diff --git a/quickstart/201-machine-learning-moderately-secure/dsvm.tf b/quickstart/201-machine-learning-moderately-secure/dsvm.tf
index 0ad12aa8..5beed51f 100644
--- a/quickstart/201-machine-learning-moderately-secure/dsvm.tf
+++ b/quickstart/201-machine-learning-moderately-secure/dsvm.tf
@@ -7,11 +7,7 @@ resource "azurerm_network_interface" "dsvm" {
         name                          = "configuration"
         subnet_id                     = azurerm_subnet.snet-dsvm.id 
         private_ip_address_allocation = "Dynamic"        
-    }
-    /*depends_on = [
-      azurerm_route_table.jumphost_rt
-    ]
-    */
+    }    
 }
 
 resource "azurerm_windows_virtual_machine" "dsvm" {
diff --git a/quickstart/301-machine-learning-hub-spoke-secure/dsvm.tf b/quickstart/301-machine-learning-hub-spoke-secure/dsvm.tf
index 8e1f93a3..383fccd3 100644
--- a/quickstart/301-machine-learning-hub-spoke-secure/dsvm.tf
+++ b/quickstart/301-machine-learning-hub-spoke-secure/dsvm.tf
@@ -7,11 +7,7 @@ resource "azurerm_network_interface" "dsvm" {
         name                          = "configuration"
         subnet_id                     = azurerm_subnet.snet-jumphost.id 
         private_ip_address_allocation = "Dynamic"        
-    }
-    /*depends_on = [
-      azurerm_route_table.jumphost_rt
-    ]
-    */
+    }    
 }
 
 resource "azurerm_windows_virtual_machine" "dsvm" {

From 71b05cfc0eb9e5ccda5cbedbe09659a70c371d1f Mon Sep 17 00:00:00 2001
From: ryhud <rhudson@microsoft.com>
Date: Thu, 28 Oct 2021 11:52:12 -0400
Subject: [PATCH 11/20] updating readme

---
 .../301-machine-learning-hub-spoke-secure/readme.md  | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/quickstart/301-machine-learning-hub-spoke-secure/readme.md b/quickstart/301-machine-learning-hub-spoke-secure/readme.md
index 688c8cf6..a495a86c 100644
--- a/quickstart/301-machine-learning-hub-spoke-secure/readme.md
+++ b/quickstart/301-machine-learning-hub-spoke-secure/readme.md
@@ -13,6 +13,8 @@ This configuration describes the minimal set of resources you require to get sta
 | Terraform Resource Type | Description |
 | - | - |
 | `azurerm_resource_group` | The resource group all resources get deployed into |
+| `azurerm_bastion_host` | An Azure Bastion Instance to securely RDP/SSH into Virtual Machines deployed into the Virtual Network |
+| `azurerm_windows_virtual_machine` | A Windows Data Science Virtual Machine used for connecting to the Azure Machine Learning workspace |
 | `azurerm_application_insights` | An Azure Application Insights instance associated to the Azure Machine Learning workspace |
 | `azurerm_key_vault` | An Azure Key Vault instance associated to the Azure Machine Learning workspace |
 | `azurerm_storage_account` | An Azure Storage instance associated to the Azure Machine Learning workspace |
@@ -26,6 +28,8 @@ This configuration describes the minimal set of resources you require to get sta
 | `azurerm_machine_learning_compute_instance` | An Azure Machine Learning compute instance a single-node managed compute. |
 | `azurerm_machine_learning_compute_cluster` | An Azure Machine Learning compute cluster as multi-node shared and managed compute. |
 | `azurerm_network_security_group` | Network security group with required inbound and outbound rules for Azure Machine Learning. |
+| `azurerm_firewall` | An Azure firewall instance used for egress traffic on the Virtual Network. |
+| `azurerm_public_ip` | A public IP resource used for the Azure Firewall. |
 
 ## Variables
 
@@ -38,8 +42,14 @@ This configuration describes the minimal set of resources you require to get sta
 | training_subnet_address_space | Address space of the training subnet | ["10.0.1.0/24"] |
 | aks_subnet_address_space | Address space of the aks subnet | ["10.0.2.0/23"] |
 | ml_subnet_address_space | Address space of the ML workspace subnet | ["10.0.0.0/24"] |
+| vnet_hub_address_space | Address space of the Hub virtual network | ["10.1.0.0/16"] |
+| jumphost_subnet_address_space | Address space of the Jumphost subnet | ["10.1.2.0/24"] |
+| bastion_subnet_address_space | Address space of the bastion subnet | ["10.1.3.0/24"] |
+| firewall_subnet_address_space | Address space of the Az Fiewall subnet | ["10.1.4.0/24"] |
 | image_build_compute_name | Name of the compute cluster to be created and configured for building docker images (Azure ML Environments) | image-builder |
-
+| dsvm_name | Name of the Windows Data Science VM resource | vmdsvm01 |
+| dsvm_admin_username | Admin username of the Windows Data Science VM | azureadmin |
+| dsvm_host_password | Password for the admin username of the Data Science VM | - |
 
 ## Usage
 

From f851ebafdcdca3653bcfde9793f4daa7d35f0387 Mon Sep 17 00:00:00 2001
From: ryhud <rhudson@microsoft.com>
Date: Thu, 28 Oct 2021 12:17:22 -0400
Subject: [PATCH 12/20] Updating readme

---
 quickstart/201-machine-learning-moderately-secure/readme.md | 2 +-
 quickstart/301-machine-learning-hub-spoke-secure/readme.md  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/quickstart/201-machine-learning-moderately-secure/readme.md b/quickstart/201-machine-learning-moderately-secure/readme.md
index b4a9f376..a65ec816 100644
--- a/quickstart/201-machine-learning-moderately-secure/readme.md
+++ b/quickstart/201-machine-learning-moderately-secure/readme.md
@@ -6,7 +6,7 @@ and its associated resources including Azure Key Vault, Azure Storage, Azure App
 In addition to these core services, this configuration specifies any networking components that are required to set up Azure Machine Learning
 for private network connectivity using [Azure Private Link](https://docs.microsoft.com/en-us/azure/private-link/). 
 
-This configuration describes the minimal set of resources you require to get started with Azure Machine Learning in a network-isolated set-up. This configuration creates new network components. Use Azure Bastion to securely connect to the Windows Data Science Virtual Machine. If you want to reuse existing network components, see [202 example](../201-machine-learning-moderately-secure/readme.md).
+This configuration describes the minimal set of resources you require to get started with Azure Machine Learning in a network-isolated set-up. This configuration creates new network components. Use Azure Bastion to securely connect to the Windows Data Science Virtual Machine. If you want to reuse existing network components, see [202 example](../202-machine-learning-moderately-secure/readme.md).
 
 ## Resources
 
diff --git a/quickstart/301-machine-learning-hub-spoke-secure/readme.md b/quickstart/301-machine-learning-hub-spoke-secure/readme.md
index a495a86c..9544061d 100644
--- a/quickstart/301-machine-learning-hub-spoke-secure/readme.md
+++ b/quickstart/301-machine-learning-hub-spoke-secure/readme.md
@@ -6,7 +6,7 @@ and its associated resources including Azure Key Vault, Azure Storage, Azure App
 In addition to these core services, this configuration specifies any networking components that are required to set up Azure Machine Learning
 for private network connectivity using [Azure Private Link](https://docs.microsoft.com/en-us/azure/private-link/). 
 
-This configuration describes the minimal set of resources you require to get started with Azure Machine Learning in a network-isolated set-up. This configuration creates new network components. If you want to reuse existing network components, see [202 example](../201-machine-learning-moderately-secure/readme.md).
+This configuration describes the full set of resources you require to get started with Azure Machine Learning in a network-isolated set-up. This configuration creates new network components. Use Azure Bastion to securely connect to the Windows Data Science Virtual Machine If you want to reuse existing network components, see [202 example](../202-machine-learning-moderately-secure/readme.md).
 
 ## Resources
 

From fa746473ff9c0c34c623a7d3b53e710f7f9d6866 Mon Sep 17 00:00:00 2001
From: ryhud <rhudson@microsoft.com>
Date: Thu, 28 Oct 2021 12:19:00 -0400
Subject: [PATCH 13/20] updating readme

---
 quickstart/201-machine-learning-moderately-secure/readme.md | 2 +-
 quickstart/301-machine-learning-hub-spoke-secure/readme.md  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/quickstart/201-machine-learning-moderately-secure/readme.md b/quickstart/201-machine-learning-moderately-secure/readme.md
index a65ec816..d1394a2d 100644
--- a/quickstart/201-machine-learning-moderately-secure/readme.md
+++ b/quickstart/201-machine-learning-moderately-secure/readme.md
@@ -6,7 +6,7 @@ and its associated resources including Azure Key Vault, Azure Storage, Azure App
 In addition to these core services, this configuration specifies any networking components that are required to set up Azure Machine Learning
 for private network connectivity using [Azure Private Link](https://docs.microsoft.com/en-us/azure/private-link/). 
 
-This configuration describes the minimal set of resources you require to get started with Azure Machine Learning in a network-isolated set-up. This configuration creates new network components. Use Azure Bastion to securely connect to the Windows Data Science Virtual Machine. If you want to reuse existing network components, see [202 example](../202-machine-learning-moderately-secure/readme.md).
+This configuration describes the minimal set of resources you require to get started with Azure Machine Learning in a network-isolated set-up. This configuration creates new network components. Use Azure Bastion to securely connect to the Windows Data Science Virtual Machine. If you want to reuse existing network components, see [202 example](../202-machine-learning-moderately-secure-existing-VNet/readme.md).
 
 ## Resources
 
diff --git a/quickstart/301-machine-learning-hub-spoke-secure/readme.md b/quickstart/301-machine-learning-hub-spoke-secure/readme.md
index 9544061d..65b570f6 100644
--- a/quickstart/301-machine-learning-hub-spoke-secure/readme.md
+++ b/quickstart/301-machine-learning-hub-spoke-secure/readme.md
@@ -6,7 +6,7 @@ and its associated resources including Azure Key Vault, Azure Storage, Azure App
 In addition to these core services, this configuration specifies any networking components that are required to set up Azure Machine Learning
 for private network connectivity using [Azure Private Link](https://docs.microsoft.com/en-us/azure/private-link/). 
 
-This configuration describes the full set of resources you require to get started with Azure Machine Learning in a network-isolated set-up. This configuration creates new network components. Use Azure Bastion to securely connect to the Windows Data Science Virtual Machine If you want to reuse existing network components, see [202 example](../202-machine-learning-moderately-secure/readme.md).
+This configuration describes the full set of resources you require to get started with Azure Machine Learning in a network-isolated set-up. This configuration creates new network components. Use Azure Bastion to securely connect to the Windows Data Science Virtual Machine If you want to reuse existing network components, see [202 example](../202-machine-learning-moderately-secure-existing-VNet/readme.md).
 
 ## Resources
 

From 442a99009d7e98af53ce857712165fe451ee08ca Mon Sep 17 00:00:00 2001
From: ryhud <rhudson@microsoft.com>
Date: Fri, 29 Oct 2021 09:05:20 -0400
Subject: [PATCH 14/20] adding Log Analytics to the resources section

---
 .../301-machine-learning-hub-spoke-secure/readme.md   | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/quickstart/301-machine-learning-hub-spoke-secure/readme.md b/quickstart/301-machine-learning-hub-spoke-secure/readme.md
index 65b570f6..5370725e 100644
--- a/quickstart/301-machine-learning-hub-spoke-secure/readme.md
+++ b/quickstart/301-machine-learning-hub-spoke-secure/readme.md
@@ -16,6 +16,7 @@ This configuration describes the full set of resources you require to get starte
 | `azurerm_bastion_host` | An Azure Bastion Instance to securely RDP/SSH into Virtual Machines deployed into the Virtual Network |
 | `azurerm_windows_virtual_machine` | A Windows Data Science Virtual Machine used for connecting to the Azure Machine Learning workspace |
 | `azurerm_application_insights` | An Azure Application Insights instance associated to the Azure Machine Learning workspace |
+| `azurerm_log_analytics_workspace` | A Log analytics workspace used for Azure Firewall logs and to also host the Workspace-based Application Insights |
 | `azurerm_key_vault` | An Azure Key Vault instance associated to the Azure Machine Learning workspace |
 | `azurerm_storage_account` | An Azure Storage instance associated to the Azure Machine Learning workspace |
 | `azurerm_container_registry` | An Azure Container Registry instance associated to the Azure Machine Learning workspace |
@@ -25,11 +26,11 @@ This configuration describes the full set of resources you require to get starte
 | `azurerm_private_dns_zone` | Private DNS Zones for FQDNs required for Azure Machine Learning and associated resources |
 | `azurerm_private_dns_zone_virtual_network_link` | Virtual network links of the Private DNS Zones to the virtual network resource |
 | `azurerm_private_endpoint` | Private Endpoints for the Azure Machine Learning workspace and associated resources |
-| `azurerm_machine_learning_compute_instance` | An Azure Machine Learning compute instance a single-node managed compute. |
-| `azurerm_machine_learning_compute_cluster` | An Azure Machine Learning compute cluster as multi-node shared and managed compute. |
-| `azurerm_network_security_group` | Network security group with required inbound and outbound rules for Azure Machine Learning. |
-| `azurerm_firewall` | An Azure firewall instance used for egress traffic on the Virtual Network. |
-| `azurerm_public_ip` | A public IP resource used for the Azure Firewall. |
+| `azurerm_machine_learning_compute_instance` | An Azure Machine Learning compute instance a single-node managed compute |
+| `azurerm_machine_learning_compute_cluster` | An Azure Machine Learning compute cluster as multi-node shared and managed compute |
+| `azurerm_network_security_group` | Network security group with required inbound and outbound rules for Azure Machine Learning |
+| `azurerm_firewall` | An Azure firewall instance used for egress traffic on the Virtual Network |
+| `azurerm_public_ip` | A public IP resource used for the Azure Firewall |
 
 ## Variables
 

From 3de3a1242f7755dc9cb4b5572a5d153f80e374ec Mon Sep 17 00:00:00 2001
From: Ryan Hudson <rhudson@microsoft.com>
Date: Fri, 29 Oct 2021 09:14:21 -0400
Subject: [PATCH 15/20] Update .gitignore

---
 .gitignore | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/.gitignore b/.gitignore
index 0d6b95a3..8b137891 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1 @@
-quickstart/301-machine-learning-hub-spoke-secure/*.terraform.lock.hcl
-quickstart/301-machine-learning-hub-spoke-secure/*.tfstate
-quickstart/301-machine-learning-hub-spoke-secure/.terraform/providers/registry.terraform.io/hashicorp/azurerm/2.79.1/windows_amd64/terraform-provider-azurerm_v2.79.1_x5.exe
-quickstart/301-machine-learning-hub-spoke-secure/.terraform/providers/registry.terraform.io/hashicorp/random/3.1.0/windows_amd64/terraform-provider-random_v3.1.0_x5.exe
-quickstart/301-machine-learning-hub-spoke-secure/.terraform.tfstate.lock.info
-quickstart/301-machine-learning-hub-spoke-secure/terraform.tfstate.*
+

From b16eefa5ca69d261762f6db3fc3ae437dea428d0 Mon Sep 17 00:00:00 2001
From: ryhud <rhudson@microsoft.com>
Date: Fri, 12 Nov 2021 15:37:16 -0500
Subject: [PATCH 16/20] terraform fmt

---
 .../azure-firewall.tf                         | 156 ++++++-------
 .../bastion.tf                                | 216 +++++++++---------
 .../dsvm.tf                                   |  56 ++---
 .../main.tf                                   |   2 +-
 .../network-hub.tf                            |  54 ++---
 .../network-spoke.tf                          |  20 +-
 6 files changed, 252 insertions(+), 252 deletions(-)

diff --git a/quickstart/301-machine-learning-hub-spoke-secure/azure-firewall.tf b/quickstart/301-machine-learning-hub-spoke-secure/azure-firewall.tf
index 4c83703d..6293ab1c 100644
--- a/quickstart/301-machine-learning-hub-spoke-secure/azure-firewall.tf
+++ b/quickstart/301-machine-learning-hub-spoke-secure/azure-firewall.tf
@@ -27,11 +27,11 @@ resource "azurerm_ip_group" "ip_group_dsvm_subnet" {
 }
 
 resource "azurerm_public_ip" "azure_firewall" {
-    name                        = "pip-azfw"
-    location                    = azurerm_resource_group.default.location
-    resource_group_name         = azurerm_resource_group.hub_rg.name
-    allocation_method           = "Static"
-    sku                         = "Standard"
+  name                = "pip-azfw"
+  location            = azurerm_resource_group.default.location
+  resource_group_name = azurerm_resource_group.hub_rg.name
+  allocation_method   = "Static"
+  sku                 = "Standard"
 }
 
 resource "azurerm_firewall_policy" "base_policy" {
@@ -41,35 +41,35 @@ resource "azurerm_firewall_policy" "base_policy" {
   dns {
     proxy_enabled = true
   }
-  
+
 }
-resource "azurerm_firewall" "azure_firewall_instance" { 
-    name                        = "afw-${var.name}-${var.environment}"
-    location                    = azurerm_resource_group.default.location
-    resource_group_name         = azurerm_resource_group.hub_rg.name
-    firewall_policy_id          = azurerm_firewall_policy.base_policy.id   
+resource "azurerm_firewall" "azure_firewall_instance" {
+  name                = "afw-${var.name}-${var.environment}"
+  location            = azurerm_resource_group.default.location
+  resource_group_name = azurerm_resource_group.hub_rg.name
+  firewall_policy_id  = azurerm_firewall_policy.base_policy.id
 
-    ip_configuration {
-        name                    = "configuration"
-        subnet_id               = azurerm_subnet.azure_firewall.id 
-        public_ip_address_id    = azurerm_public_ip.azure_firewall.id
-    }
-
-    timeouts {
-      create = "60m"
-      delete = "2h"
+  ip_configuration {
+    name                 = "configuration"
+    subnet_id            = azurerm_subnet.azure_firewall.id
+    public_ip_address_id = azurerm_public_ip.azure_firewall.id
   }
-  depends_on = [ 
+
+  timeouts {
+    create = "60m"
+    delete = "2h"
+  }
+  depends_on = [
     azurerm_public_ip.azure_firewall,
     azurerm_subnet.azure_firewall,
     azurerm_firewall_policy_rule_collection_group.azure_firewall_rules_collection
-    ]
+  ]
 }
 
 resource "azurerm_monitor_diagnostic_setting" "azure_firewall_instance" {
-  name                        = "diagnostics-${var.name}-${var.environment}-${random_string.fw_diag_prefix.result}"
-  target_resource_id          = azurerm_firewall.azure_firewall_instance.id
-  log_analytics_workspace_id  = azurerm_log_analytics_workspace.default.id
+  name                       = "diagnostics-${var.name}-${var.environment}-${random_string.fw_diag_prefix.result}"
+  target_resource_id         = azurerm_firewall.azure_firewall_instance.id
+  log_analytics_workspace_id = azurerm_log_analytics_workspace.default.id
 
   log {
     category = "AzureFirewallApplicationRule"
@@ -95,7 +95,7 @@ resource "azurerm_monitor_diagnostic_setting" "azure_firewall_instance" {
       enabled = false
     }
   }
-  
+
 
   metric {
     category = "AllMetrics"
@@ -112,7 +112,7 @@ resource "azurerm_firewall_policy_rule_collection_group" "azure_firewall_rules_c
   firewall_policy_id = azurerm_firewall_policy.base_policy.id
   priority           = 100
 
-application_rule_collection {
+  application_rule_collection {
     name     = "afwp-base-app-rule-collection"
     priority = 200
     action   = "Allow"
@@ -125,9 +125,9 @@ application_rule_collection {
       }
       protocols {
         type = "Http"
-        port= 80
+        port = 80
       }
-      source_ip_groups = [azurerm_ip_group.ip_group_dsvm_subnet.id]      
+      source_ip_groups  = [azurerm_ip_group.ip_group_dsvm_subnet.id]
       destination_fqdns = ["*"]
     }
 
@@ -137,7 +137,7 @@ application_rule_collection {
         type = "Https"
         port = 443
       }
-      source_ip_groups = [azurerm_ip_group.ip_group_spoke.id]      
+      source_ip_groups      = [azurerm_ip_group.ip_group_spoke.id]
       destination_fqdn_tags = ["AzureKubernetesService"]
     }
 
@@ -147,8 +147,8 @@ application_rule_collection {
         type = "Https"
         port = 443
       }
-      source_ip_groups = [azurerm_ip_group.ip_group_spoke.id]      
-      destination_fqdns = ["api.snapcraft.io","motd.ubuntu.com",]
+      source_ip_groups  = [azurerm_ip_group.ip_group_spoke.id]
+      destination_fqdns = ["api.snapcraft.io", "motd.ubuntu.com", ]
     }
 
     rule {
@@ -157,11 +157,11 @@ application_rule_collection {
         type = "Http"
         port = 80
       }
-      source_ip_groups = [azurerm_ip_group.ip_group_spoke.id]      
+      source_ip_groups = [azurerm_ip_group.ip_group_spoke.id]
       destination_fqdns = ["crl.microsoft.com",
-                          "mscrl.microsoft.com",  
-                          "crl3.digicert.com",
-                          "ocsp.digicert.com"]
+        "mscrl.microsoft.com",
+        "crl3.digicert.com",
+      "ocsp.digicert.com"]
     }
 
     rule {
@@ -170,7 +170,7 @@ application_rule_collection {
         type = "Https"
         port = 443
       }
-      source_ip_groups = [azurerm_ip_group.ip_group_spoke.id]      
+      source_ip_groups  = [azurerm_ip_group.ip_group_spoke.id]
       destination_fqdns = ["github.com"]
     }
 
@@ -184,7 +184,7 @@ application_rule_collection {
         type = "Http"
         port = 80
       }
-      source_ip_groups = [azurerm_ip_group.ip_group_spoke.id]      
+      source_ip_groups  = [azurerm_ip_group.ip_group_spoke.id]
       destination_fqdns = ["raw.githubusercontent.com"]
     }
 
@@ -194,7 +194,7 @@ application_rule_collection {
         type = "Https"
         port = 443
       }
-      source_ip_groups = [azurerm_ip_group.ip_group_spoke.id]      
+      source_ip_groups  = [azurerm_ip_group.ip_group_spoke.id]
       destination_fqdns = ["*.prod.microsoftmetrics.com"]
     }
 
@@ -204,11 +204,11 @@ application_rule_collection {
         type = "Https"
         port = 443
       }
-      source_ip_groups = [azurerm_ip_group.ip_group_spoke.id]      
+      source_ip_groups = [azurerm_ip_group.ip_group_spoke.id]
       destination_fqdns = ["acs-mirror.azureedge.net",
-                           "*.docker.io",
-                           "production.cloudflare.docker.com",
-                          "*.azurecr.io"]
+        "*.docker.io",
+        "production.cloudflare.docker.com",
+      "*.azurecr.io"]
     }
 
     rule {
@@ -217,11 +217,11 @@ application_rule_collection {
         type = "Https"
         port = 443
       }
-      source_ip_groups = [azurerm_ip_group.ip_group_spoke.id]      
+      source_ip_groups  = [azurerm_ip_group.ip_group_spoke.id]
       destination_fqdns = ["login.microsoftonline.com"]
     }
 
-        rule {
+    rule {
       name = "graph.windows.net"
       protocols {
         type = "Http"
@@ -231,7 +231,7 @@ application_rule_collection {
         type = "Https"
         port = 443
       }
-      source_ip_groups = [azurerm_ip_group.ip_group_spoke.id]      
+      source_ip_groups  = [azurerm_ip_group.ip_group_spoke.id]
       destination_fqdns = ["graph.windows.net"]
     }
 
@@ -245,7 +245,7 @@ application_rule_collection {
         type = "Https"
         port = 443
       }
-      source_ip_groups = [azurerm_ip_group.ip_group_spoke.id]      
+      source_ip_groups  = [azurerm_ip_group.ip_group_spoke.id]
       destination_fqdns = ["anaconda.com", "*.anaconda.com"]
     }
 
@@ -259,10 +259,10 @@ application_rule_collection {
         type = "Https"
         port = 443
       }
-      source_ip_groups = [azurerm_ip_group.ip_group_spoke.id]      
+      source_ip_groups  = [azurerm_ip_group.ip_group_spoke.id]
       destination_fqdns = ["*.anaconda.org"]
     }
-    
+
     rule {
       name = "pypi.org"
       protocols {
@@ -273,7 +273,7 @@ application_rule_collection {
         type = "Https"
         port = 443
       }
-      source_ip_groups = [azurerm_ip_group.ip_group_spoke.id]      
+      source_ip_groups  = [azurerm_ip_group.ip_group_spoke.id]
       destination_fqdns = ["pypi.org"]
     }
 
@@ -287,7 +287,7 @@ application_rule_collection {
         type = "Https"
         port = 443
       }
-      source_ip_groups = [azurerm_ip_group.ip_group_spoke.id]      
+      source_ip_groups  = [azurerm_ip_group.ip_group_spoke.id]
       destination_fqdns = ["cloud.r-project.org"]
     }
 
@@ -301,7 +301,7 @@ application_rule_collection {
         type = "Https"
         port = 443
       }
-      source_ip_groups = [azurerm_ip_group.ip_group_spoke.id]      
+      source_ip_groups  = [azurerm_ip_group.ip_group_spoke.id]
       destination_fqdns = ["*pytorch.org"]
     }
 
@@ -315,7 +315,7 @@ application_rule_collection {
         type = "Https"
         port = 443
       }
-      source_ip_groups = [azurerm_ip_group.ip_group_spoke.id]      
+      source_ip_groups  = [azurerm_ip_group.ip_group_spoke.id]
       destination_fqdns = ["*.tensorflow.org"]
     }
 
@@ -329,7 +329,7 @@ application_rule_collection {
         type = "Https"
         port = 443
       }
-      source_ip_groups = [azurerm_ip_group.ip_group_spoke.id]      
+      source_ip_groups  = [azurerm_ip_group.ip_group_spoke.id]
       destination_fqdns = ["update.code.visualstudio.com", "*.vo.msecnd.net"]
     }
 
@@ -343,7 +343,7 @@ application_rule_collection {
         type = "Https"
         port = 443
       }
-      source_ip_groups = [azurerm_ip_group.ip_group_spoke.id]      
+      source_ip_groups  = [azurerm_ip_group.ip_group_spoke.id]
       destination_fqdns = ["dc.applicationinsights.azure.com"]
     }
 
@@ -357,7 +357,7 @@ application_rule_collection {
         type = "Https"
         port = 443
       }
-      source_ip_groups = [azurerm_ip_group.ip_group_spoke.id]      
+      source_ip_groups  = [azurerm_ip_group.ip_group_spoke.id]
       destination_fqdns = ["dc.applicationinsights.microsoft.com"]
     }
 
@@ -371,7 +371,7 @@ application_rule_collection {
         type = "Https"
         port = 443
       }
-      source_ip_groups = [azurerm_ip_group.ip_group_spoke.id]      
+      source_ip_groups  = [azurerm_ip_group.ip_group_spoke.id]
       destination_fqdns = ["dc.services.visualstudio.com"]
     }
 
@@ -385,12 +385,12 @@ application_rule_collection {
         type = "Https"
         port = 443
       }
-      source_ip_groups = [azurerm_ip_group.ip_group_spoke.id]      
+      source_ip_groups  = [azurerm_ip_group.ip_group_spoke.id]
       destination_fqdns = ["*.instances.azureml.net", "*.instances.azureml.ms"]
     }
   }
 
- network_rule_collection {
+  network_rule_collection {
     name     = "afwp-base-network-rule-collection"
     priority = 100
     action   = "Allow"
@@ -398,15 +398,15 @@ application_rule_collection {
     rule {
       name                  = "hub-to-spoke-rule"
       protocols             = ["Any"]
-      source_ip_groups      = [azurerm_ip_group.ip_group_spoke.id,azurerm_ip_group.ip_group_hub.id]   
-      destination_ip_groups = [azurerm_ip_group.ip_group_hub.id,azurerm_ip_group.ip_group_spoke.id]
+      source_ip_groups      = [azurerm_ip_group.ip_group_spoke.id, azurerm_ip_group.ip_group_hub.id]
+      destination_ip_groups = [azurerm_ip_group.ip_group_hub.id, azurerm_ip_group.ip_group_spoke.id]
       destination_ports     = ["*"]
     }
 
-     rule {
+    rule {
       name                  = "aks-global-network-rule"
       protocols             = ["TCP"]
-      source_ip_groups      = [azurerm_ip_group.ip_group_spoke.id]      
+      source_ip_groups      = [azurerm_ip_group.ip_group_spoke.id]
       destination_addresses = ["AzureCloud"]
       destination_ports     = ["443", "9000"]
     }
@@ -414,7 +414,7 @@ application_rule_collection {
     rule {
       name                  = "aks-ntp-network-rule"
       protocols             = ["UDP"]
-      source_ip_groups      = [azurerm_ip_group.ip_group_spoke.id]      
+      source_ip_groups      = [azurerm_ip_group.ip_group_spoke.id]
       destination_addresses = ["*"]
       destination_ports     = ["123"]
     }
@@ -422,7 +422,7 @@ application_rule_collection {
     rule {
       name                  = "Azure-Active-Directory"
       protocols             = ["TCP"]
-      source_ip_groups      = [azurerm_ip_group.ip_group_spoke.id]      
+      source_ip_groups      = [azurerm_ip_group.ip_group_spoke.id]
       destination_addresses = ["AzureActiveDirectory"]
       destination_ports     = ["*"]
     }
@@ -430,7 +430,7 @@ application_rule_collection {
     rule {
       name                  = "Azure-Machine-Learning"
       protocols             = ["TCP"]
-      source_ip_groups      = [azurerm_ip_group.ip_group_spoke.id]      
+      source_ip_groups      = [azurerm_ip_group.ip_group_spoke.id]
       destination_addresses = ["AzureMachineLearning"]
       destination_ports     = ["443"]
     }
@@ -438,7 +438,7 @@ application_rule_collection {
     rule {
       name                  = "Azure-Resource-Manager"
       protocols             = ["TCP"]
-      source_ip_groups      = [azurerm_ip_group.ip_group_spoke.id]      
+      source_ip_groups      = [azurerm_ip_group.ip_group_spoke.id]
       destination_addresses = ["AzureResourceManager"]
       destination_ports     = ["443"]
     }
@@ -446,7 +446,7 @@ application_rule_collection {
     rule {
       name                  = "Azure-Storage"
       protocols             = ["TCP"]
-      source_ip_groups      = [azurerm_ip_group.ip_group_spoke.id]      
+      source_ip_groups      = [azurerm_ip_group.ip_group_spoke.id]
       destination_addresses = ["Storage"]
       destination_ports     = ["443"]
     }
@@ -454,15 +454,15 @@ application_rule_collection {
     rule {
       name                  = "Azure-Front-Door-Frontend"
       protocols             = ["TCP"]
-      source_ip_groups      = [azurerm_ip_group.ip_group_spoke.id]      
-      destination_addresses = ["AzureFrontDoor.Frontend","AzureFrontDoor.FirstParty"]
+      source_ip_groups      = [azurerm_ip_group.ip_group_spoke.id]
+      destination_addresses = ["AzureFrontDoor.Frontend", "AzureFrontDoor.FirstParty"]
       destination_ports     = ["443"]
     }
 
     rule {
       name                  = "Azure-Container-Registry"
       protocols             = ["TCP"]
-      source_ip_groups      = [azurerm_ip_group.ip_group_spoke.id]      
+      source_ip_groups      = [azurerm_ip_group.ip_group_spoke.id]
       destination_addresses = ["AzureContainerRegistry"]
       destination_ports     = ["443"]
     }
@@ -470,7 +470,7 @@ application_rule_collection {
     rule {
       name                  = "Azure-Key-Vault"
       protocols             = ["TCP"]
-      source_ip_groups      = [azurerm_ip_group.ip_group_spoke.id]      
+      source_ip_groups      = [azurerm_ip_group.ip_group_spoke.id]
       destination_addresses = ["AzureKeyVault"]
       destination_ports     = ["443"]
     }
@@ -478,13 +478,13 @@ application_rule_collection {
     rule {
       name                  = "Microsoft-Container-Registry"
       protocols             = ["TCP"]
-      source_ip_groups      = [azurerm_ip_group.ip_group_spoke.id]      
+      source_ip_groups      = [azurerm_ip_group.ip_group_spoke.id]
       destination_addresses = ["MicrosoftContainerRegistry"]
       destination_ports     = ["443"]
     }
- }
- depends_on = [
-   azurerm_ip_group.ip_group_hub,
-   azurerm_ip_group.ip_group_spoke
- ]
+  }
+  depends_on = [
+    azurerm_ip_group.ip_group_hub,
+    azurerm_ip_group.ip_group_spoke
+  ]
 }
\ No newline at end of file
diff --git a/quickstart/301-machine-learning-hub-spoke-secure/bastion.tf b/quickstart/301-machine-learning-hub-spoke-secure/bastion.tf
index 89c81847..d06b3f3c 100644
--- a/quickstart/301-machine-learning-hub-spoke-secure/bastion.tf
+++ b/quickstart/301-machine-learning-hub-spoke-secure/bastion.tf
@@ -1,126 +1,126 @@
-resource "azurerm_public_ip" "azure_bastion" { 
-    name                        = "pip-azure-bastion"
-    location                    = azurerm_resource_group.default.location
-    resource_group_name         = azurerm_resource_group.hub_rg.name
-    allocation_method           = "Static"
-    sku                         = "Standard" 
+resource "azurerm_public_ip" "azure_bastion" {
+  name                = "pip-azure-bastion"
+  location            = azurerm_resource_group.default.location
+  resource_group_name = azurerm_resource_group.hub_rg.name
+  allocation_method   = "Static"
+  sku                 = "Standard"
 }
 
-resource "azurerm_network_security_group" "bastion_nsg" { 
-    name                        = "nsg-bastion"
-    location                    = azurerm_resource_group.default.location
-    resource_group_name         = azurerm_resource_group.hub_rg.name
+resource "azurerm_network_security_group" "bastion_nsg" {
+  name                = "nsg-bastion"
+  location            = azurerm_resource_group.default.location
+  resource_group_name = azurerm_resource_group.hub_rg.name
 
-    security_rule {      
-      name                        = "AllowHTTPSInbound"
-      priority                    = 100
-      direction                   = "Inbound"
-      access                      = "Allow"
-      protocol                    = "Tcp"
-      source_port_range           = "*"
-      destination_port_range      = "443"
-      source_address_prefix       = "Internet"
-      destination_address_prefix  = "*"
-    }
-    security_rule {
-      name                        = "AllowGatewayManagerInbound"
-      priority                    = 200
-      direction                   = "Inbound"
-      access                      = "Allow"
-      protocol                    = "Tcp"
-      source_port_range           = "*"
-      destination_port_range      = "443"
-      source_address_prefix       = "GatewayManager"
-      destination_address_prefix  = "*"
-    }
-    security_rule {
-      name                        = "AllowAzureLBInbound"
-      priority                    = 300
-      direction                   = "Inbound"
-      access                      = "Allow"
-      protocol                    = "Tcp"
-      source_port_range           = "*"
-      destination_port_range      = "443"
-      source_address_prefix       = "AzureLoadBalancer"
-      destination_address_prefix  = "*"
-    }
-    security_rule {
-      name                        = "AllowBastionHostCommunication"
-      priority                    = 400
-      direction                   = "Inbound"
-      access                      = "Allow"
-      protocol                    = "*"
-      source_port_range           = "*"
-      destination_port_ranges      = ["5701","8080"]
-      source_address_prefix       = "VirtualNetwork"
-      destination_address_prefix  = "VirtualNetwork"
-    }
-    security_rule {
-      name                        = "AllowRdpSshOutbound"
-      priority                    = 100
-      direction                   = "Outbound"
-      access                      = "Allow"
-      protocol                    = "Tcp"
-      source_port_range           = "*"
-      destination_port_ranges      = ["22", "3389"]
-      source_address_prefix       = "*"
-      destination_address_prefix  = "VirtualNetwork"
-    }
-      security_rule {
-      name                        = "AllowBastionHostCommunicationOutbound"
-      priority                    = 110
-      direction                   = "Outbound"
-      access                      = "Allow"
-      protocol                    = "Tcp"
-      source_port_range           = "*"
-      destination_port_ranges      = ["5701", "8080"]
-      source_address_prefix       = "VirtualNetwork"
-      destination_address_prefix  = "VirtualNetwork"
+  security_rule {
+    name                       = "AllowHTTPSInbound"
+    priority                   = 100
+    direction                  = "Inbound"
+    access                     = "Allow"
+    protocol                   = "Tcp"
+    source_port_range          = "*"
+    destination_port_range     = "443"
+    source_address_prefix      = "Internet"
+    destination_address_prefix = "*"
   }
-    security_rule {
-      name                        = "AllowAzureCloudOutbound"
-      priority                    = 120
-      direction                   = "Outbound"
-      access                      = "Allow"
-      protocol                    = "Tcp"
-      source_port_range           = "*"
-      destination_port_ranges      = ["443"]
-      source_address_prefix       = "*"
-      destination_address_prefix  = "AzureCloud"
-    }
-    security_rule {
-      name                        = "AllowGetSessionInformation"
-      priority                    = 130
-      direction                   = "Outbound"
-      access                      = "Allow"
-      protocol                    = "Tcp"
-      source_port_range           = "*"
-      destination_port_ranges      = ["80"]
-      source_address_prefix       = "*"
-      destination_address_prefix  = "Internet"
-}
-    
+  security_rule {
+    name                       = "AllowGatewayManagerInbound"
+    priority                   = 200
+    direction                  = "Inbound"
+    access                     = "Allow"
+    protocol                   = "Tcp"
+    source_port_range          = "*"
+    destination_port_range     = "443"
+    source_address_prefix      = "GatewayManager"
+    destination_address_prefix = "*"
+  }
+  security_rule {
+    name                       = "AllowAzureLBInbound"
+    priority                   = 300
+    direction                  = "Inbound"
+    access                     = "Allow"
+    protocol                   = "Tcp"
+    source_port_range          = "*"
+    destination_port_range     = "443"
+    source_address_prefix      = "AzureLoadBalancer"
+    destination_address_prefix = "*"
+  }
+  security_rule {
+    name                       = "AllowBastionHostCommunication"
+    priority                   = 400
+    direction                  = "Inbound"
+    access                     = "Allow"
+    protocol                   = "*"
+    source_port_range          = "*"
+    destination_port_ranges    = ["5701", "8080"]
+    source_address_prefix      = "VirtualNetwork"
+    destination_address_prefix = "VirtualNetwork"
+  }
+  security_rule {
+    name                       = "AllowRdpSshOutbound"
+    priority                   = 100
+    direction                  = "Outbound"
+    access                     = "Allow"
+    protocol                   = "Tcp"
+    source_port_range          = "*"
+    destination_port_ranges    = ["22", "3389"]
+    source_address_prefix      = "*"
+    destination_address_prefix = "VirtualNetwork"
+  }
+  security_rule {
+    name                       = "AllowBastionHostCommunicationOutbound"
+    priority                   = 110
+    direction                  = "Outbound"
+    access                     = "Allow"
+    protocol                   = "Tcp"
+    source_port_range          = "*"
+    destination_port_ranges    = ["5701", "8080"]
+    source_address_prefix      = "VirtualNetwork"
+    destination_address_prefix = "VirtualNetwork"
+  }
+  security_rule {
+    name                       = "AllowAzureCloudOutbound"
+    priority                   = 120
+    direction                  = "Outbound"
+    access                     = "Allow"
+    protocol                   = "Tcp"
+    source_port_range          = "*"
+    destination_port_ranges    = ["443"]
+    source_address_prefix      = "*"
+    destination_address_prefix = "AzureCloud"
+  }
+  security_rule {
+    name                       = "AllowGetSessionInformation"
+    priority                   = 130
+    direction                  = "Outbound"
+    access                     = "Allow"
+    protocol                   = "Tcp"
+    source_port_range          = "*"
+    destination_port_ranges    = ["80"]
+    source_address_prefix      = "*"
+    destination_address_prefix = "Internet"
+  }
+
 }
 
 resource "azurerm_subnet_network_security_group_association" "bastion_nsg_assoc" {
   subnet_id                 = azurerm_subnet.azure_bastion.id
   network_security_group_id = azurerm_network_security_group.bastion_nsg.id
-  depends_on = [ 
+  depends_on = [
     azurerm_bastion_host.azure_bastion_instance,
     azurerm_subnet_network_security_group_association.jumphost_nsg_assoc
-    ]
+  ]
 }
 
 
 resource "azurerm_bastion_host" "azure_bastion_instance" {
-    name                        = "bas-${var.name}-${var.environment}"
-    location                    = azurerm_resource_group.default.location
-    resource_group_name         = azurerm_resource_group.hub_rg.name
+  name                = "bas-${var.name}-${var.environment}"
+  location            = azurerm_resource_group.default.location
+  resource_group_name = azurerm_resource_group.hub_rg.name
 
-    ip_configuration { 
-        name                    = "configuration"
-        subnet_id               = azurerm_subnet.azure_bastion.id
-        public_ip_address_id    = azurerm_public_ip.azure_bastion.id 
-    }
+  ip_configuration {
+    name                 = "configuration"
+    subnet_id            = azurerm_subnet.azure_bastion.id
+    public_ip_address_id = azurerm_public_ip.azure_bastion.id
+  }
 }
 
diff --git a/quickstart/301-machine-learning-hub-spoke-secure/dsvm.tf b/quickstart/301-machine-learning-hub-spoke-secure/dsvm.tf
index 383fccd3..204e9b6e 100644
--- a/quickstart/301-machine-learning-hub-spoke-secure/dsvm.tf
+++ b/quickstart/301-machine-learning-hub-spoke-secure/dsvm.tf
@@ -1,48 +1,48 @@
-resource "azurerm_network_interface" "dsvm" { 
-    name                              = "nic-${var.dsvm_name}"
-    location                          = azurerm_resource_group.default.location
-    resource_group_name               = azurerm_resource_group.default.name
-    
-    ip_configuration { 
-        name                          = "configuration"
-        subnet_id                     = azurerm_subnet.snet-jumphost.id 
-        private_ip_address_allocation = "Dynamic"        
-    }    
+resource "azurerm_network_interface" "dsvm" {
+  name                = "nic-${var.dsvm_name}"
+  location            = azurerm_resource_group.default.location
+  resource_group_name = azurerm_resource_group.default.name
+
+  ip_configuration {
+    name                          = "configuration"
+    subnet_id                     = azurerm_subnet.snet-jumphost.id
+    private_ip_address_allocation = "Dynamic"
+  }
 }
 
 resource "azurerm_windows_virtual_machine" "dsvm" {
-  name                  = var.dsvm_name
-  location              = azurerm_resource_group.default.location
-  resource_group_name   = azurerm_resource_group.default.name
+  name                = var.dsvm_name
+  location            = azurerm_resource_group.default.location
+  resource_group_name = azurerm_resource_group.default.name
   network_interface_ids = [
-        azurerm_network_interface.dsvm.id
-    ]
-  size               = "Standard_DS3_v2"
- 
+    azurerm_network_interface.dsvm.id
+  ]
+  size = "Standard_DS3_v2"
+
   source_image_reference {
     publisher = "microsoft-dsvm"
     offer     = "dsvm-win-2019"
     sku       = "server-2019"
     version   = "latest"
   }
- 
+
   os_disk {
-    name              = "osdisk-${var.dsvm_name}"
-    caching           = "ReadWrite"  
+    name                 = "osdisk-${var.dsvm_name}"
+    caching              = "ReadWrite"
     storage_account_type = "Premium_LRS"
   }
- 
+
   identity {
     type = "SystemAssigned"
   }
-  computer_name      = var.dsvm_name
-  admin_username     = var.dsvm_admin_username
-  admin_password     = var.dsvm_host_password
-  
+  computer_name  = var.dsvm_name
+  admin_username = var.dsvm_admin_username
+  admin_password = var.dsvm_host_password
+
   provision_vm_agent = true
- 
+
   timeouts {
-      create = "60m"
-      delete = "2h"
+    create = "60m"
+    delete = "2h"
   }
 }
diff --git a/quickstart/301-machine-learning-hub-spoke-secure/main.tf b/quickstart/301-machine-learning-hub-spoke-secure/main.tf
index 124361e0..64a6602c 100644
--- a/quickstart/301-machine-learning-hub-spoke-secure/main.tf
+++ b/quickstart/301-machine-learning-hub-spoke-secure/main.tf
@@ -24,5 +24,5 @@ resource "azurerm_resource_group" "default" {
 resource "azurerm_resource_group" "hub_rg" {
   name     = "rg-hub-${var.name}-${var.environment}"
   location = var.location
-  
+
 }
\ No newline at end of file
diff --git a/quickstart/301-machine-learning-hub-spoke-secure/network-hub.tf b/quickstart/301-machine-learning-hub-spoke-secure/network-hub.tf
index cf0bb055..28c29ac5 100644
--- a/quickstart/301-machine-learning-hub-spoke-secure/network-hub.tf
+++ b/quickstart/301-machine-learning-hub-spoke-secure/network-hub.tf
@@ -8,28 +8,28 @@ resource "azurerm_virtual_network" "hub" {
 }
 
 resource "azurerm_subnet" "snet-jumphost" {
-  name                          = "snet-jumphost"
-  resource_group_name           = azurerm_resource_group.hub_rg.name
-  virtual_network_name          = azurerm_virtual_network.hub.name
-  address_prefixes              = var.jumphost_subnet_address_space
+  name                 = "snet-jumphost"
+  resource_group_name  = azurerm_resource_group.hub_rg.name
+  virtual_network_name = azurerm_virtual_network.hub.name
+  address_prefixes     = var.jumphost_subnet_address_space
 
 }
 
 
 resource "azurerm_subnet" "azure_bastion" {
-    name                        = "AzureBastionSubnet"
-    resource_group_name         = azurerm_resource_group.hub_rg.name
-    virtual_network_name        = azurerm_virtual_network.hub.name
-    address_prefixes            = var.bastion_subnet_address_space
-    
-} 
+  name                 = "AzureBastionSubnet"
+  resource_group_name  = azurerm_resource_group.hub_rg.name
+  virtual_network_name = azurerm_virtual_network.hub.name
+  address_prefixes     = var.bastion_subnet_address_space
+
+}
 resource "azurerm_subnet" "azure_firewall" {
-    name                        = "AzureFirewallSubnet"
-    resource_group_name         = azurerm_resource_group.hub_rg.name
-    virtual_network_name        = azurerm_virtual_network.hub.name
-    address_prefixes            = var.firewall_subnet_address_space
-    
-} 
+  name                 = "AzureFirewallSubnet"
+  resource_group_name  = azurerm_resource_group.hub_rg.name
+  virtual_network_name = azurerm_virtual_network.hub.name
+  address_prefixes     = var.firewall_subnet_address_space
+
+}
 
 #Vnet Peering
 
@@ -62,7 +62,7 @@ resource "azurerm_virtual_network_peering" "direction2" {
     azurerm_virtual_network.hub,
     azurerm_virtual_network.default
   ]
-  
+
 }
 
 # Private DNS Zones
@@ -140,14 +140,14 @@ resource "azurerm_private_dns_zone_virtual_network_link" "vnetlinknbs" {
 
 # NSG for jump_host Subnet
 
-resource "azurerm_network_security_group" "jump_host" { 
-    name                        = "nsg-jumphost-subnet"
-    location                    = azurerm_resource_group.hub_rg.location
-    resource_group_name         = azurerm_resource_group.hub_rg.name
+resource "azurerm_network_security_group" "jump_host" {
+  name                = "nsg-jumphost-subnet"
+  location            = azurerm_resource_group.hub_rg.location
+  resource_group_name = azurerm_resource_group.hub_rg.name
 }
 
 resource "azurerm_subnet_network_security_group_association" "jumphost_nsg_assoc" {
-  subnet_id                 = azurerm_subnet.snet-jumphost.id 
+  subnet_id                 = azurerm_subnet.snet-jumphost.id
   network_security_group_id = azurerm_network_security_group.jump_host.id
   depends_on = [
     azurerm_network_interface.dsvm
@@ -162,11 +162,11 @@ resource "azurerm_route_table" "jumphost_rt" {
 }
 
 resource "azurerm_route" "jumphost-fw-route" {
-  name                = "udr-Default"
-  resource_group_name = azurerm_resource_group.default.name
-  route_table_name    = azurerm_route_table.jumphost_rt.name
-  address_prefix      = "0.0.0.0/0"
-  next_hop_type       = "VirtualAppliance"
+  name                   = "udr-Default"
+  resource_group_name    = azurerm_resource_group.default.name
+  route_table_name       = azurerm_route_table.jumphost_rt.name
+  address_prefix         = "0.0.0.0/0"
+  next_hop_type          = "VirtualAppliance"
   next_hop_in_ip_address = azurerm_firewall.azure_firewall_instance.ip_configuration[0].private_ip_address
 }
 
diff --git a/quickstart/301-machine-learning-hub-spoke-secure/network-spoke.tf b/quickstart/301-machine-learning-hub-spoke-secure/network-spoke.tf
index 75a98046..7d00fdf9 100644
--- a/quickstart/301-machine-learning-hub-spoke-secure/network-spoke.tf
+++ b/quickstart/301-machine-learning-hub-spoke-secure/network-spoke.tf
@@ -93,11 +93,11 @@ resource "azurerm_route_table" "rt-training" {
 }
 
 resource "azurerm_route" "training-Internet-Route" {
-  name                = "udr-Default"
-  resource_group_name = azurerm_resource_group.default.name
-  route_table_name    = azurerm_route_table.rt-training.name
-  address_prefix      = "0.0.0.0/0"
-  next_hop_type       = "VirtualAppliance"
+  name                   = "udr-Default"
+  resource_group_name    = azurerm_resource_group.default.name
+  route_table_name       = azurerm_route_table.rt-training.name
+  address_prefix         = "0.0.0.0/0"
+  next_hop_type          = "VirtualAppliance"
   next_hop_in_ip_address = azurerm_firewall.azure_firewall_instance.ip_configuration[0].private_ip_address
 }
 
@@ -130,11 +130,11 @@ resource "azurerm_route_table" "rt-aks" {
 }
 
 resource "azurerm_route" "aks-default-Route" {
-  name                = "udr-Default"
-  resource_group_name = azurerm_resource_group.default.name
-  route_table_name    = azurerm_route_table.rt-aks.name
-  address_prefix      = "0.0.0.0/0"
-  next_hop_type       = "VirtualAppliance"
+  name                   = "udr-Default"
+  resource_group_name    = azurerm_resource_group.default.name
+  route_table_name       = azurerm_route_table.rt-aks.name
+  address_prefix         = "0.0.0.0/0"
+  next_hop_type          = "VirtualAppliance"
   next_hop_in_ip_address = azurerm_firewall.azure_firewall_instance.ip_configuration[0].private_ip_address
 }
 

From a4f39d340c9886526124653c5e496cdafbc63945 Mon Sep 17 00:00:00 2001
From: ryhud <rhudson@microsoft.com>
Date: Fri, 12 Nov 2021 15:38:18 -0500
Subject: [PATCH 17/20] tf version 1.0

---
 quickstart/301-machine-learning-hub-spoke-secure/main.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/quickstart/301-machine-learning-hub-spoke-secure/main.tf b/quickstart/301-machine-learning-hub-spoke-secure/main.tf
index 64a6602c..e67f4af8 100644
--- a/quickstart/301-machine-learning-hub-spoke-secure/main.tf
+++ b/quickstart/301-machine-learning-hub-spoke-secure/main.tf
@@ -1,5 +1,5 @@
 terraform {
-  required_version = ">=0.15.0"
+  required_version = ">=1.0"
 
   required_providers {
     azurerm = {

From 91b12bb8b952829340770bc1b9a80fd9a92f9988 Mon Sep 17 00:00:00 2001
From: ryhud <rhudson@microsoft.com>
Date: Fri, 12 Nov 2021 16:20:50 -0500
Subject: [PATCH 18/20] updating tf version 1.0

---
 quickstart/101-machine-learning/main.tf                         | 2 +-
 quickstart/201-machine-learning-moderately-secure/main.tf       | 2 +-
 .../main.tf                                                     | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/quickstart/101-machine-learning/main.tf b/quickstart/101-machine-learning/main.tf
index b7d6655b..e7dc0655 100644
--- a/quickstart/101-machine-learning/main.tf
+++ b/quickstart/101-machine-learning/main.tf
@@ -1,5 +1,5 @@
 terraform {
-  required_version = ">=0.15.0"
+  required_version = ">=1.0"
 
   required_providers {
     azurerm = {
diff --git a/quickstart/201-machine-learning-moderately-secure/main.tf b/quickstart/201-machine-learning-moderately-secure/main.tf
index b5e0c3a8..21befa79 100644
--- a/quickstart/201-machine-learning-moderately-secure/main.tf
+++ b/quickstart/201-machine-learning-moderately-secure/main.tf
@@ -1,5 +1,5 @@
 terraform {
-  required_version = ">=0.15.0"
+  required_version = ">=1.0"
 
   required_providers {
     azurerm = {
diff --git a/quickstart/202-machine-learning-moderately-secure-existing-VNet/main.tf b/quickstart/202-machine-learning-moderately-secure-existing-VNet/main.tf
index b5e0c3a8..21befa79 100644
--- a/quickstart/202-machine-learning-moderately-secure-existing-VNet/main.tf
+++ b/quickstart/202-machine-learning-moderately-secure-existing-VNet/main.tf
@@ -1,5 +1,5 @@
 terraform {
-  required_version = ">=0.15.0"
+  required_version = ">=1.0"
 
   required_providers {
     azurerm = {

From 057a0330cad028fb783d8d870ffb457c23698be7 Mon Sep 17 00:00:00 2001
From: ryhud <rhudson@microsoft.com>
Date: Fri, 12 Nov 2021 16:33:12 -0500
Subject: [PATCH 19/20] terraform fmt

---
 .../bastion.tf                                | 218 +++++++++---------
 .../dsvm.tf                                   |  56 ++---
 .../network.tf                                |   8 +-
 .../workspace.tf                              |   2 +-
 4 files changed, 142 insertions(+), 142 deletions(-)

diff --git a/quickstart/201-machine-learning-moderately-secure/bastion.tf b/quickstart/201-machine-learning-moderately-secure/bastion.tf
index d8cae3d8..dcea9044 100644
--- a/quickstart/201-machine-learning-moderately-secure/bastion.tf
+++ b/quickstart/201-machine-learning-moderately-secure/bastion.tf
@@ -1,125 +1,125 @@
-resource "azurerm_public_ip" "azure_bastion" { 
-    name                        = "pip-azure-bastion"
-    location                    = azurerm_resource_group.default.location
-    resource_group_name         = azurerm_resource_group.default.name
-    allocation_method           = "Static"
-    sku                         = "Standard" 
+resource "azurerm_public_ip" "azure_bastion" {
+  name                = "pip-azure-bastion"
+  location            = azurerm_resource_group.default.location
+  resource_group_name = azurerm_resource_group.default.name
+  allocation_method   = "Static"
+  sku                 = "Standard"
 }
 
-resource "azurerm_network_security_group" "bastion_nsg" { 
-    name                        = "nsg-bastion"
-    location                    = azurerm_resource_group.default.location
-    resource_group_name         = azurerm_resource_group.default.name
+resource "azurerm_network_security_group" "bastion_nsg" {
+  name                = "nsg-bastion"
+  location            = azurerm_resource_group.default.location
+  resource_group_name = azurerm_resource_group.default.name
 
-    security_rule {      
-      name                        = "AllowHTTPSInbound"
-      priority                    = 100
-      direction                   = "Inbound"
-      access                      = "Allow"
-      protocol                    = "Tcp"
-      source_port_range           = "*"
-      destination_port_range      = "443"
-      source_address_prefix       = "Internet"
-      destination_address_prefix  = "*"
-    }
-    security_rule {
-      name                        = "AllowGatewayManagerInbound"
-      priority                    = 200
-      direction                   = "Inbound"
-      access                      = "Allow"
-      protocol                    = "Tcp"
-      source_port_range           = "*"
-      destination_port_range      = "443"
-      source_address_prefix       = "GatewayManager"
-      destination_address_prefix  = "*"
-    }
-    security_rule {
-      name                        = "AllowAzureLBInbound"
-      priority                    = 300
-      direction                   = "Inbound"
-      access                      = "Allow"
-      protocol                    = "Tcp"
-      source_port_range           = "*"
-      destination_port_range      = "443"
-      source_address_prefix       = "AzureLoadBalancer"
-      destination_address_prefix  = "*"
-    }
-    security_rule {
-      name                        = "AllowBastionHostCommunication"
-      priority                    = 400
-      direction                   = "Inbound"
-      access                      = "Allow"
-      protocol                    = "*"
-      source_port_range           = "*"
-      destination_port_ranges      = ["5701","8080"]
-      source_address_prefix       = "VirtualNetwork"
-      destination_address_prefix  = "VirtualNetwork"
-    }
-    security_rule {
-      name                        = "AllowRdpSshOutbound"
-      priority                    = 100
-      direction                   = "Outbound"
-      access                      = "Allow"
-      protocol                    = "Tcp"
-      source_port_range           = "*"
-      destination_port_ranges      = ["22", "3389"]
-      source_address_prefix       = "*"
-      destination_address_prefix  = "VirtualNetwork"
-    }
-      security_rule {
-      name                        = "AllowBastionHostCommunicationOutbound"
-      priority                    = 110
-      direction                   = "Outbound"
-      access                      = "Allow"
-      protocol                    = "Tcp"
-      source_port_range           = "*"
-      destination_port_ranges      = ["5701", "8080"]
-      source_address_prefix       = "VirtualNetwork"
-      destination_address_prefix  = "VirtualNetwork"
+  security_rule {
+    name                       = "AllowHTTPSInbound"
+    priority                   = 100
+    direction                  = "Inbound"
+    access                     = "Allow"
+    protocol                   = "Tcp"
+    source_port_range          = "*"
+    destination_port_range     = "443"
+    source_address_prefix      = "Internet"
+    destination_address_prefix = "*"
   }
-    security_rule {
-      name                        = "AllowAzureCloudOutbound"
-      priority                    = 120
-      direction                   = "Outbound"
-      access                      = "Allow"
-      protocol                    = "Tcp"
-      source_port_range           = "*"
-      destination_port_ranges      = ["443"]
-      source_address_prefix       = "*"
-      destination_address_prefix  = "AzureCloud"
-    }
-    security_rule {
-      name                        = "AllowGetSessionInformation"
-      priority                    = 130
-      direction                   = "Outbound"
-      access                      = "Allow"
-      protocol                    = "Tcp"
-      source_port_range           = "*"
-      destination_port_ranges      = ["80"]
-      source_address_prefix       = "*"
-      destination_address_prefix  = "Internet"
-}
-    
+  security_rule {
+    name                       = "AllowGatewayManagerInbound"
+    priority                   = 200
+    direction                  = "Inbound"
+    access                     = "Allow"
+    protocol                   = "Tcp"
+    source_port_range          = "*"
+    destination_port_range     = "443"
+    source_address_prefix      = "GatewayManager"
+    destination_address_prefix = "*"
+  }
+  security_rule {
+    name                       = "AllowAzureLBInbound"
+    priority                   = 300
+    direction                  = "Inbound"
+    access                     = "Allow"
+    protocol                   = "Tcp"
+    source_port_range          = "*"
+    destination_port_range     = "443"
+    source_address_prefix      = "AzureLoadBalancer"
+    destination_address_prefix = "*"
+  }
+  security_rule {
+    name                       = "AllowBastionHostCommunication"
+    priority                   = 400
+    direction                  = "Inbound"
+    access                     = "Allow"
+    protocol                   = "*"
+    source_port_range          = "*"
+    destination_port_ranges    = ["5701", "8080"]
+    source_address_prefix      = "VirtualNetwork"
+    destination_address_prefix = "VirtualNetwork"
+  }
+  security_rule {
+    name                       = "AllowRdpSshOutbound"
+    priority                   = 100
+    direction                  = "Outbound"
+    access                     = "Allow"
+    protocol                   = "Tcp"
+    source_port_range          = "*"
+    destination_port_ranges    = ["22", "3389"]
+    source_address_prefix      = "*"
+    destination_address_prefix = "VirtualNetwork"
+  }
+  security_rule {
+    name                       = "AllowBastionHostCommunicationOutbound"
+    priority                   = 110
+    direction                  = "Outbound"
+    access                     = "Allow"
+    protocol                   = "Tcp"
+    source_port_range          = "*"
+    destination_port_ranges    = ["5701", "8080"]
+    source_address_prefix      = "VirtualNetwork"
+    destination_address_prefix = "VirtualNetwork"
+  }
+  security_rule {
+    name                       = "AllowAzureCloudOutbound"
+    priority                   = 120
+    direction                  = "Outbound"
+    access                     = "Allow"
+    protocol                   = "Tcp"
+    source_port_range          = "*"
+    destination_port_ranges    = ["443"]
+    source_address_prefix      = "*"
+    destination_address_prefix = "AzureCloud"
+  }
+  security_rule {
+    name                       = "AllowGetSessionInformation"
+    priority                   = 130
+    direction                  = "Outbound"
+    access                     = "Allow"
+    protocol                   = "Tcp"
+    source_port_range          = "*"
+    destination_port_ranges    = ["80"]
+    source_address_prefix      = "*"
+    destination_address_prefix = "Internet"
+  }
+
 }
 
 resource "azurerm_subnet_network_security_group_association" "bastion_nsg_assoc" {
   subnet_id                 = azurerm_subnet.azure_bastion.id
   network_security_group_id = azurerm_network_security_group.bastion_nsg.id
-  depends_on = [ 
-    azurerm_bastion_host.azure_bastion_instance    
-    ]
+  depends_on = [
+    azurerm_bastion_host.azure_bastion_instance
+  ]
 }
 
 
 resource "azurerm_bastion_host" "azure_bastion_instance" {
-    name                        = "bas-${var.name}-${var.environment}"
-    location                    = azurerm_resource_group.default.location
-    resource_group_name         = azurerm_resource_group.default.name
+  name                = "bas-${var.name}-${var.environment}"
+  location            = azurerm_resource_group.default.location
+  resource_group_name = azurerm_resource_group.default.name
 
-    ip_configuration { 
-        name                    = "configuration"
-        subnet_id               = azurerm_subnet.azure_bastion.id
-        public_ip_address_id    = azurerm_public_ip.azure_bastion.id 
-    }
+  ip_configuration {
+    name                 = "configuration"
+    subnet_id            = azurerm_subnet.azure_bastion.id
+    public_ip_address_id = azurerm_public_ip.azure_bastion.id
+  }
 }
 
diff --git a/quickstart/201-machine-learning-moderately-secure/dsvm.tf b/quickstart/201-machine-learning-moderately-secure/dsvm.tf
index 5beed51f..d01ddebe 100644
--- a/quickstart/201-machine-learning-moderately-secure/dsvm.tf
+++ b/quickstart/201-machine-learning-moderately-secure/dsvm.tf
@@ -1,48 +1,48 @@
-resource "azurerm_network_interface" "dsvm" { 
-    name                              = "nic-${var.dsvm_name}"
-    location                          = azurerm_resource_group.default.location
-    resource_group_name               = azurerm_resource_group.default.name
-    
-    ip_configuration { 
-        name                          = "configuration"
-        subnet_id                     = azurerm_subnet.snet-dsvm.id 
-        private_ip_address_allocation = "Dynamic"        
-    }    
+resource "azurerm_network_interface" "dsvm" {
+  name                = "nic-${var.dsvm_name}"
+  location            = azurerm_resource_group.default.location
+  resource_group_name = azurerm_resource_group.default.name
+
+  ip_configuration {
+    name                          = "configuration"
+    subnet_id                     = azurerm_subnet.snet-dsvm.id
+    private_ip_address_allocation = "Dynamic"
+  }
 }
 
 resource "azurerm_windows_virtual_machine" "dsvm" {
-  name                  = var.dsvm_name
-  location              = azurerm_resource_group.default.location
-  resource_group_name   = azurerm_resource_group.default.name
+  name                = var.dsvm_name
+  location            = azurerm_resource_group.default.location
+  resource_group_name = azurerm_resource_group.default.name
   network_interface_ids = [
-        azurerm_network_interface.dsvm.id
-    ]
-  size               = "Standard_DS3_v2"
- 
+    azurerm_network_interface.dsvm.id
+  ]
+  size = "Standard_DS3_v2"
+
   source_image_reference {
     publisher = "microsoft-dsvm"
     offer     = "dsvm-win-2019"
     sku       = "server-2019"
     version   = "latest"
   }
- 
+
   os_disk {
-    name              = "osdisk-${var.dsvm_name}"
-    caching           = "ReadWrite"  
+    name                 = "osdisk-${var.dsvm_name}"
+    caching              = "ReadWrite"
     storage_account_type = "Premium_LRS"
   }
- 
+
   identity {
     type = "SystemAssigned"
   }
-  computer_name      = var.dsvm_name
-  admin_username     = var.dsvm_admin_username
-  admin_password     = var.dsvm_host_password
-  
+  computer_name  = var.dsvm_name
+  admin_username = var.dsvm_admin_username
+  admin_password = var.dsvm_host_password
+
   provision_vm_agent = true
- 
+
   timeouts {
-      create = "60m"
-      delete = "2h"
+    create = "60m"
+    delete = "2h"
   }
 }
diff --git a/quickstart/201-machine-learning-moderately-secure/network.tf b/quickstart/201-machine-learning-moderately-secure/network.tf
index 11bf0c28..0911c8eb 100644
--- a/quickstart/201-machine-learning-moderately-secure/network.tf
+++ b/quickstart/201-machine-learning-moderately-secure/network.tf
@@ -39,10 +39,10 @@ resource "azurerm_subnet" "snet-dsvm" {
 }
 
 resource "azurerm_subnet" "azure_bastion" {
-    name                        = "AzureBastionSubnet"
-    resource_group_name         = azurerm_resource_group.default.name
-    virtual_network_name        = azurerm_virtual_network.default.name
-    address_prefixes            = var.bastion_subnet_address_space
+  name                 = "AzureBastionSubnet"
+  resource_group_name  = azurerm_resource_group.default.name
+  virtual_network_name = azurerm_virtual_network.default.name
+  address_prefixes     = var.bastion_subnet_address_space
 }
 
 # Private DNS Zones
diff --git a/quickstart/201-machine-learning-moderately-secure/workspace.tf b/quickstart/201-machine-learning-moderately-secure/workspace.tf
index dc0036cc..7b6ba44a 100644
--- a/quickstart/201-machine-learning-moderately-secure/workspace.tf
+++ b/quickstart/201-machine-learning-moderately-secure/workspace.tf
@@ -63,7 +63,7 @@ resource "azurerm_machine_learning_workspace" "default" {
   # Args of use when using an Azure Private Link configuration
   public_network_access_enabled = false
   image_build_compute_name      = var.image_build_compute_name
-  depends_on = [    
+  depends_on = [
     azurerm_private_endpoint.kv_ple,
     azurerm_private_endpoint.st_ple_blob,
     azurerm_private_endpoint.storage_ple_file,

From f42bebeab3a3c87d1fcac831e0c086f07acfa9fa Mon Sep 17 00:00:00 2001
From: Ryan Hudson <rhudson@microsoft.com>
Date: Sat, 13 Nov 2021 11:43:26 -0500
Subject: [PATCH 20/20] Delete .gitignore

---
 .gitignore | 1 -
 1 file changed, 1 deletion(-)
 delete mode 100644 .gitignore

diff --git a/.gitignore b/.gitignore
deleted file mode 100644
index 8b137891..00000000
--- a/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-