diff --git a/.gitignore b/.gitignore index e69de29b..ec9e5b1d 100644 --- a/.gitignore +++ b/.gitignore @@ -0,0 +1,4 @@ +quickstart/301-machine-learning-hub-spoke-secure/*.terraform.lock.hcl +quickstart/301-machine-learning-hub-spoke-secure/*.tfstate +quickstart/301-machine-learning-hub-spoke-secure/.terraform/providers/registry.terraform.io/hashicorp/azurerm/2.79.1/windows_amd64/terraform-provider-azurerm_v2.79.1_x5.exe +quickstart/301-machine-learning-hub-spoke-secure/.terraform/providers/registry.terraform.io/hashicorp/random/3.1.0/windows_amd64/terraform-provider-random_v3.1.0_x5.exe diff --git a/quickstart/301-machine-learning-hub-spoke-secure/azure-firewall.tf b/quickstart/301-machine-learning-hub-spoke-secure/azure-firewall.tf index 23365dd0..fddc61ee 100644 --- a/quickstart/301-machine-learning-hub-spoke-secure/azure-firewall.tf +++ b/quickstart/301-machine-learning-hub-spoke-secure/azure-firewall.tf @@ -1,3 +1,25 @@ + +resource "azurerm_ip_group" "ip_group_hub" { + name = "hub-ipgroup" + location = azurerm_resource_group.hub_rg.location + resource_group_name = azurerm_resource_group.hub_rg.name + cidrs = var.vnet_hub_address_space +} + +resource "azurerm_ip_group" "ip_group_spoke" { + name = "mlw-spoke-ipgroup" + location = azurerm_resource_group.hub_rg.location + resource_group_name = azurerm_resource_group.hub_rg.name + cidrs = var.vnet_address_space +} + +resource "azurerm_ip_group" "ip_group_dsvm_subnet" { + name = "dsvm-subnet-ipgroup" + location = azurerm_resource_group.hub_rg.location + resource_group_name = azurerm_resource_group.hub_rg.name + cidrs = var.jumphost_subnet_address_space +} + resource "azurerm_public_ip" "azure_firewall" { name = "pip-azfw" location = azurerm_resource_group.default.location @@ -13,10 +35,7 @@ resource "azurerm_firewall_policy" "base_policy" { dns { proxy_enabled = true } - depends_on = [ - azurerm_virtual_network_peering.direction1, - azurerm_virtual_network_peering.direction2 - ] + } resource "azurerm_firewall" "azure_firewall_instance" { name = "afw-${var.name}-${var.environment}" @@ -34,12 +53,15 @@ resource "azurerm_firewall" "azure_firewall_instance" { create = "60m" delete = "2h" } - depends_on = [ azurerm_public_ip.azure_firewall, - azurerm_firewall_policy.base_policy] + depends_on = [ + azurerm_public_ip.azure_firewall, + azurerm_subnet.azure_firewall, + azurerm_firewall_policy_rule_collection_group.azure_firewall_rules_collection + ] } resource "azurerm_monitor_diagnostic_setting" "azure_firewall_instance" { - name = "diagnostics" + name = "diagnostics-${var.name}-${var.environment}" target_resource_id = azurerm_firewall.azure_firewall_instance.id log_analytics_workspace_id = azurerm_log_analytics_workspace.default.id @@ -77,4 +99,358 @@ resource "azurerm_monitor_diagnostic_setting" "azure_firewall_instance" { } } +} + +resource "azurerm_firewall_policy_rule_collection_group" "azure_firewall_rules_collection" { + name = "afwp-base-rule-collection-group" + firewall_policy_id = azurerm_firewall_policy.base_policy.id + priority = 100 + +application_rule_collection { + name = "afwp-base-app-rule-collection" + priority = 200 + action = "Allow" + + rule { + name = "dsvm-to-internet" + protocols { + type = "Https" + port = 443 + } + protocols { + type = "Http" + port= 80 + } + source_ip_groups = [azurerm_ip_group.ip_group_dsvm_subnet.id] + destination_fqdns = ["*"] + } + + rule { + name = "aks-service-tag" + protocols { + type = "Https" + port = 443 + } + source_ip_groups = [azurerm_ip_group.ip_group_spoke.id] + destination_fqdn_tags = ["AzureKubernetesService"] + } + + rule { + name = "ubuntu-libraries" + protocols { + type = "Https" + port = 443 + } + source_ip_groups = [azurerm_ip_group.ip_group_spoke.id] + destination_fqdns = ["api.snapcraft.io","motd.ubuntu.com",] + } + + rule { + name = "microsoft-crls" + protocols { + type = "Http" + port = 80 + } + source_ip_groups = [azurerm_ip_group.ip_group_spoke.id] + destination_fqdns = ["crl.microsoft.com", + "mscrl.microsoft.com", + "crl3.digicert.com", + "ocsp.digicert.com"] + } + + rule { + name = "github-rules" + protocols { + type = "Https" + port = 443 + } + source_ip_groups = [azurerm_ip_group.ip_group_spoke.id] + destination_fqdns = ["github.com"] + } + + rule { + name = "microsoft-metrics-rules" + protocols { + type = "Https" + port = 443 + } + source_ip_groups = [azurerm_ip_group.ip_group_spoke.id] + destination_fqdns = ["*.prod.microsoftmetrics.com"] + } + + rule { + name = "aks-acs-rules" + protocols { + type = "Https" + port = 443 + } + source_ip_groups = [azurerm_ip_group.ip_group_spoke.id] + destination_fqdns = ["acs-mirror.azureedge.net", + "*.docker.io", + "production.cloudflare.docker.com", + "*.azurecr.io"] + } + + rule { + name = "microsoft-login-rules" + protocols { + type = "Https" + port = 443 + } + source_ip_groups = [azurerm_ip_group.ip_group_spoke.id] + destination_fqdns = ["login.microsoftonline.com"] + } + + rule { + name = "graph.windows.net" + protocols { + type = "Http" + port = 80 + } + protocols { + type = "Https" + port = 443 + } + source_ip_groups = [azurerm_ip_group.ip_group_spoke.id] + destination_fqdns = ["graph.windows.net"] + } + + rule { + name = "anaconda.com" + protocols { + type = "Http" + port = 80 + } + protocols { + type = "Https" + port = 443 + } + source_ip_groups = [azurerm_ip_group.ip_group_spoke.id] + destination_fqdns = ["anaconda.com", "*.anaconda.com"] + } + + rule { + name = "anaconda.org" + protocols { + type = "Http" + port = 80 + } + protocols { + type = "Https" + port = 443 + } + source_ip_groups = [azurerm_ip_group.ip_group_spoke.id] + destination_fqdns = ["*.anaconda.org"] + } + + rule { + name = "pypi.org" + protocols { + type = "Http" + port = 80 + } + protocols { + type = "Https" + port = 443 + } + source_ip_groups = [azurerm_ip_group.ip_group_spoke.id] + destination_fqdns = ["pypi.org"] + } + + rule { + name = "cloud.r-project.org" + protocols { + type = "Http" + port = 80 + } + protocols { + type = "Https" + port = 443 + } + source_ip_groups = [azurerm_ip_group.ip_group_spoke.id] + destination_fqdns = ["cloud.r-project.org"] + } + + rule { + name = "pytorch.org" + protocols { + type = "Http" + port = 80 + } + protocols { + type = "Https" + port = 443 + } + source_ip_groups = [azurerm_ip_group.ip_group_spoke.id] + destination_fqdns = ["*pytorch.org"] + } + + rule { + name = "tensorflow.org" + protocols { + type = "Http" + port = 80 + } + protocols { + type = "Https" + port = 443 + } + source_ip_groups = [azurerm_ip_group.ip_group_spoke.id] + destination_fqdns = ["*.tensorflow.org"] + } + + rule { + name = "update.code.visualstudio.com" + protocols { + type = "Http" + port = 80 + } + protocols { + type = "Https" + port = 443 + } + source_ip_groups = [azurerm_ip_group.ip_group_spoke.id] + destination_fqdns = ["update.code.visualstudio.com", "*.vo.msecnd.net"] + } + + rule { + name = "dc.applicationinsights.azure.com" + protocols { + type = "Http" + port = 80 + } + protocols { + type = "Https" + port = 443 + } + source_ip_groups = [azurerm_ip_group.ip_group_spoke.id] + destination_fqdns = ["dc.applicationinsights.azure.com"] + } + + rule { + name = "dc.applicationinsights.microsoft.com" + protocols { + type = "Http" + port = 80 + } + protocols { + type = "Https" + port = 443 + } + source_ip_groups = [azurerm_ip_group.ip_group_spoke.id] + destination_fqdns = ["dc.applicationinsights.microsoft.com"] + } + + rule { + name = "dc.services.visualstudio.com" + protocols { + type = "Http" + port = 80 + } + protocols { + type = "Https" + port = 443 + } + source_ip_groups = [azurerm_ip_group.ip_group_spoke.id] + destination_fqdns = ["dc.services.visualstudio.com"] + } + } + + network_rule_collection { + name = "afwp-base-network-rule-collection" + priority = 100 + action = "Allow" + + rule { + name = "hub-to-spoke-rule" + protocols = ["Any"] + source_ip_groups = [azurerm_ip_group.ip_group_spoke.id] + destination_ip_groups = [azurerm_ip_group.ip_group_hub.id] + destination_ports = ["*"] + } + + rule { + name = "aks-global-network-rule" + protocols = ["TCP"] + source_ip_groups = [azurerm_ip_group.ip_group_spoke.id] + destination_addresses = ["AzureCloud"] + destination_ports = ["443", "9000"] + } + + rule { + name = "aks-ntp-network-rule" + protocols = ["UDP"] + source_ip_groups = [azurerm_ip_group.ip_group_spoke.id] + destination_addresses = ["*"] + destination_ports = ["123"] + } + + rule { + name = "Azure-Active-Directory" + protocols = ["TCP"] + source_ip_groups = [azurerm_ip_group.ip_group_spoke.id] + destination_addresses = ["AzureActiveDirectory"] + destination_ports = ["*"] + } + + rule { + name = "Azure-Machine-Learning" + protocols = ["TCP"] + source_ip_groups = [azurerm_ip_group.ip_group_spoke.id] + destination_addresses = ["AzureMachineLearning"] + destination_ports = ["443"] + } + + rule { + name = "Azure-Resource-Manager" + protocols = ["TCP"] + source_ip_groups = [azurerm_ip_group.ip_group_spoke.id] + destination_addresses = ["AzureResourceManager"] + destination_ports = ["443"] + } + + rule { + name = "Azure-Storage" + protocols = ["TCP"] + source_ip_groups = [azurerm_ip_group.ip_group_spoke.id] + destination_addresses = ["Storage"] + destination_ports = ["443"] + } + + rule { + name = "Azure-Front-Door-Frontend" + protocols = ["TCP"] + source_ip_groups = [azurerm_ip_group.ip_group_spoke.id] + destination_addresses = ["AzureFrontDoor.Frontend"] + destination_ports = ["443"] + } + + rule { + name = "Azure-Container-Registry" + protocols = ["TCP"] + source_ip_groups = [azurerm_ip_group.ip_group_spoke.id] + destination_addresses = ["AzureContainerRegistry"] + destination_ports = ["443"] + } + + rule { + name = "Azure-Key-Vault" + protocols = ["TCP"] + source_ip_groups = [azurerm_ip_group.ip_group_spoke.id] + destination_addresses = ["AzureKeyVault"] + destination_ports = ["443"] + } + + rule { + name = "Microsoft-Container-Registry" + protocols = ["TCP"] + source_ip_groups = [azurerm_ip_group.ip_group_spoke.id] + destination_addresses = ["MicrosoftContainerRegistry"] + destination_ports = ["443"] + } + } + depends_on = [ + azurerm_ip_group.ip_group_hub, + azurerm_ip_group.ip_group_spoke + ] } \ No newline at end of file diff --git a/quickstart/301-machine-learning-hub-spoke-secure/bastion.tf b/quickstart/301-machine-learning-hub-spoke-secure/bastion.tf index e83f90d8..89c81847 100644 --- a/quickstart/301-machine-learning-hub-spoke-secure/bastion.tf +++ b/quickstart/301-machine-learning-hub-spoke-secure/bastion.tf @@ -105,7 +105,10 @@ resource "azurerm_network_security_group" "bastion_nsg" { resource "azurerm_subnet_network_security_group_association" "bastion_nsg_assoc" { subnet_id = azurerm_subnet.azure_bastion.id network_security_group_id = azurerm_network_security_group.bastion_nsg.id - depends_on = [ azurerm_bastion_host.azure_bastion_instance ] + depends_on = [ + azurerm_bastion_host.azure_bastion_instance, + azurerm_subnet_network_security_group_association.jumphost_nsg_assoc + ] } diff --git a/quickstart/301-machine-learning-hub-spoke-secure/compute.tf b/quickstart/301-machine-learning-hub-spoke-secure/compute.tf index b95541b8..520031a7 100644 --- a/quickstart/301-machine-learning-hub-spoke-secure/compute.tf +++ b/quickstart/301-machine-learning-hub-spoke-secure/compute.tf @@ -6,7 +6,7 @@ resource "random_string" "ci_prefix" { number = false } -/* # Compute instance +# Compute instance resource "azurerm_machine_learning_compute_instance" "compute_instance" { name = "${random_string.ci_prefix.result}instance" location = azurerm_resource_group.default.location @@ -18,7 +18,7 @@ resource "azurerm_machine_learning_compute_instance" "compute_instance" { azurerm_private_endpoint.mlw_ple ] } -*/ + # Compute cluster resource "azurerm_machine_learning_compute_cluster" "compute" { name = "cpu-cluster" @@ -37,5 +37,8 @@ resource "azurerm_machine_learning_compute_cluster" "compute" { max_node_count = 3 scale_down_nodes_after_idle_duration = "PT15M" # 15 minutes } + depends_on = [ + azurerm_private_endpoint.mlw_ple + ] } \ No newline at end of file diff --git a/quickstart/301-machine-learning-hub-spoke-secure/network-hub.tf b/quickstart/301-machine-learning-hub-spoke-secure/network-hub.tf index 94c3d5b1..cf0bb055 100644 --- a/quickstart/301-machine-learning-hub-spoke-secure/network-hub.tf +++ b/quickstart/301-machine-learning-hub-spoke-secure/network-hub.tf @@ -42,7 +42,11 @@ resource "azurerm_virtual_network_peering" "direction1" { allow_forwarded_traffic = false allow_gateway_transit = false use_remote_gateways = false - + depends_on = [ + azurerm_virtual_network.hub, + azurerm_virtual_network.default + ] + } resource "azurerm_virtual_network_peering" "direction2" { @@ -54,7 +58,11 @@ resource "azurerm_virtual_network_peering" "direction2" { allow_forwarded_traffic = false allow_gateway_transit = false use_remote_gateways = false - + depends_on = [ + azurerm_virtual_network.hub, + azurerm_virtual_network.default + ] + } # Private DNS Zones @@ -141,4 +149,28 @@ resource "azurerm_network_security_group" "jump_host" { resource "azurerm_subnet_network_security_group_association" "jumphost_nsg_assoc" { subnet_id = azurerm_subnet.snet-jumphost.id network_security_group_id = azurerm_network_security_group.jump_host.id + depends_on = [ + azurerm_network_interface.dsvm + ] } + +# Route Table for Jump host subnet +resource "azurerm_route_table" "jumphost_rt" { + name = "rt-jumphost" + location = azurerm_resource_group.default.location + resource_group_name = azurerm_resource_group.default.name +} + +resource "azurerm_route" "jumphost-fw-route" { + name = "udr-Default" + resource_group_name = azurerm_resource_group.default.name + route_table_name = azurerm_route_table.jumphost_rt.name + address_prefix = "0.0.0.0/0" + next_hop_type = "VirtualAppliance" + next_hop_in_ip_address = azurerm_firewall.azure_firewall_instance.ip_configuration[0].private_ip_address +} + +resource "azurerm_subnet_route_table_association" "rt-jumphost-link" { + subnet_id = azurerm_subnet.snet-jumphost.id + route_table_id = azurerm_route_table.jumphost_rt.id +} \ No newline at end of file diff --git a/quickstart/301-machine-learning-hub-spoke-secure/network-spoke.tf b/quickstart/301-machine-learning-hub-spoke-secure/network-spoke.tf index 0d5568d3..75a98046 100644 --- a/quickstart/301-machine-learning-hub-spoke-secure/network-spoke.tf +++ b/quickstart/301-machine-learning-hub-spoke-secure/network-spoke.tf @@ -4,6 +4,11 @@ resource "azurerm_virtual_network" "default" { address_space = var.vnet_address_space location = azurerm_resource_group.default.location resource_group_name = azurerm_resource_group.default.name + dns_servers = [azurerm_firewall.azure_firewall_instance.ip_configuration[0].private_ip_address] + depends_on = [ + azurerm_virtual_network.hub, + azurerm_firewall.azure_firewall_instance + ] } resource "azurerm_subnet" "snet-training" { @@ -88,15 +93,16 @@ resource "azurerm_route_table" "rt-training" { } resource "azurerm_route" "training-Internet-Route" { - name = "Internet" + name = "udr-Default" resource_group_name = azurerm_resource_group.default.name route_table_name = azurerm_route_table.rt-training.name address_prefix = "0.0.0.0/0" - next_hop_type = "Internet" + next_hop_type = "VirtualAppliance" + next_hop_in_ip_address = azurerm_firewall.azure_firewall_instance.ip_configuration[0].private_ip_address } resource "azurerm_route" "training-AzureMLRoute" { - name = "AzureMLRoute" + name = "udr-AzureML" resource_group_name = azurerm_resource_group.default.name route_table_name = azurerm_route_table.rt-training.name address_prefix = "AzureMachineLearning" @@ -104,7 +110,7 @@ resource "azurerm_route" "training-AzureMLRoute" { } resource "azurerm_route" "training-BatchRoute" { - name = "BatchRoute" + name = "udr-Batch" resource_group_name = azurerm_resource_group.default.name route_table_name = azurerm_route_table.rt-training.name address_prefix = "BatchNodeManagement" @@ -123,12 +129,13 @@ resource "azurerm_route_table" "rt-aks" { resource_group_name = azurerm_resource_group.default.name } -resource "azurerm_route" "aks-Internet-Route" { - name = "Internet" +resource "azurerm_route" "aks-default-Route" { + name = "udr-Default" resource_group_name = azurerm_resource_group.default.name route_table_name = azurerm_route_table.rt-aks.name address_prefix = "0.0.0.0/0" - next_hop_type = "Internet" + next_hop_type = "VirtualAppliance" + next_hop_in_ip_address = azurerm_firewall.azure_firewall_instance.ip_configuration[0].private_ip_address } resource "azurerm_subnet_route_table_association" "rt-aks-link" { diff --git a/quickstart/301-machine-learning-hub-spoke-secure/workspace.tf b/quickstart/301-machine-learning-hub-spoke-secure/workspace.tf index f9be016d..b749f10a 100644 --- a/quickstart/301-machine-learning-hub-spoke-secure/workspace.tf +++ b/quickstart/301-machine-learning-hub-spoke-secure/workspace.tf @@ -182,4 +182,7 @@ resource "azurerm_machine_learning_compute_cluster" "image-builder" { identity { type = "SystemAssigned" } + depends_on = [ + azurerm_private_endpoint.mlw_ple + ] }