From 063a26bb31f3b6f9c4149b19e637837179f99829 Mon Sep 17 00:00:00 2001 From: murggu Date: Tue, 8 Nov 2022 12:47:54 +0100 Subject: [PATCH 1/9] Add synapse 101 --- quickstart/101-synapse/.gitignore | 11 ++++ quickstart/101-synapse/README.md | 51 ++++++++++++++++++ quickstart/101-synapse/locals.tf | 10 ++++ quickstart/101-synapse/main.tf | 24 +++++++++ quickstart/101-synapse/storage_account.tf | 36 +++++++++++++ quickstart/101-synapse/synapse_pools.tf | 28 ++++++++++ quickstart/101-synapse/synapse_workspace.tf | 32 +++++++++++ .../101-synapse/terraform.tfvars.example | 12 +++++ quickstart/101-synapse/variables.tf | 54 +++++++++++++++++++ 9 files changed, 258 insertions(+) create mode 100644 quickstart/101-synapse/.gitignore create mode 100644 quickstart/101-synapse/README.md create mode 100644 quickstart/101-synapse/locals.tf create mode 100644 quickstart/101-synapse/main.tf create mode 100644 quickstart/101-synapse/storage_account.tf create mode 100644 quickstart/101-synapse/synapse_pools.tf create mode 100644 quickstart/101-synapse/synapse_workspace.tf create mode 100644 quickstart/101-synapse/terraform.tfvars.example create mode 100644 quickstart/101-synapse/variables.tf diff --git a/quickstart/101-synapse/.gitignore b/quickstart/101-synapse/.gitignore new file mode 100644 index 00000000..cc744186 --- /dev/null +++ b/quickstart/101-synapse/.gitignore @@ -0,0 +1,11 @@ +# Terraform specific + +.terraform + +.terraform.lock.hcl +terraform.tfstate +terraform.tfstate.backup +.terraform.tfstate.lock.info +terraform.tfvars +**.tfbackend +state/ \ No newline at end of file diff --git a/quickstart/101-synapse/README.md b/quickstart/101-synapse/README.md new file mode 100644 index 00000000..ab8dd2d8 --- /dev/null +++ b/quickstart/101-synapse/README.md @@ -0,0 +1,51 @@ +# Azure Synapse Analytics workspace (public network connectivity) + +This deployment configuration specifies an [Azure Synapse Analytics workspace](https://learn.microsoft.com/en-us/azure/synapse-analytics/get-started-create-workspace), +and its associated resources including Azure Data Lake Storage (gen2), Synapse Spark Pool and Synapse SQL Pool. + +This configuration describes the minimal set of resources you require to get started with Azure Synapse Analytics. + +Network connectivity to the workspace is allowed over public endpoints, making this configuration suitable for open source projects or pilot environments. + +## Resources + + +| Terraform Resource Type | Description | +| - | - | +| `azurerm_resource_group` | The resource group all resources get deployed into. | +| `azurerm_storage_account` | An Azure Storage instance associated to the Azure Synapse Analytics workspace. | +| `azurerm_synapse_workspace` | An Azure Synapse Analytics workspace instance. | +| `azurerm_synapse_spark_pool` | An Azure Synapse Analytics spark pool. | +| `azurerm_synapse_sql_pool` | An Azure Synapse Analytics dedicated SQL pool. | + +## Variables + +| Name | Description | Default | +|-|-|-| +| name | Name of the deployment | - | +| environment | The deployment environment name (used for pre- and postfixing resource names) | dev | +| location | The Azure region used for deployments | East US | +| aad_admin.login | The login name of the Azure AD Administrator of this Synapse Workspace | - | +| aad_admin.object_id| The object id of the Azure AD Administrator of this Synapse Workspace | - | +| aad_admin.tenant_id| The tenant id of the Azure AD Administrator of this Synapse Workspace | - | +| synadmin_username| Specifies The login name of the SQL administrator | sqladminuser | +| synadmin_password| The Password associated with the sql_administrator_login for the SQL administrator | ThisIsNotVerySecure! | +| enable_syn_sparkpool| A feature flag to enable/disable the Spark pool | false | +| enable_syn_sqlpool| A feature flag to enable/disable the SQL pool | false | + +## Usage + +1. Copy `terraform.tfvars.example` to `terraform.tfvars` +2. Update `terraform.tfvars` with your desired values +3. Run Terraform + ```console + $ terraform init + $ terraform plan + $ terraform apply + ``` + +## Learn more + +- If you are new to Azure Synapse Analytics, see [Azure Synapse Analytics service](https://azure.microsoft.com/services/synapse-analytics/) and [Azure Synapse Analytics documentation](https://learn.microsoft.com/azure/synapse-analytics/overview-what-is). +- To learn more about security configurations in Azure Synapse Analytics, see [Azure Synapse Analytics security white paper](https://learn.microsoft.com/azure/synapse-analytics/guidance/security-white-paper-introduction). +- For all configurations of Azure Synapse Analytics in Terraform, see [Terraform Hashicorp AzureRM provider documentation](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/synapse_workspace). diff --git a/quickstart/101-synapse/locals.tf b/quickstart/101-synapse/locals.tf new file mode 100644 index 00000000..2d4911c5 --- /dev/null +++ b/quickstart/101-synapse/locals.tf @@ -0,0 +1,10 @@ +locals { + tags = { + Toolkit = "Terraform" + } + + safe_name = replace(var.name, "-", "") + safe_environment = replace(var.environment, "-", "") + + basename = "${var.name}-${var.environment}" +} \ No newline at end of file diff --git a/quickstart/101-synapse/main.tf b/quickstart/101-synapse/main.tf new file mode 100644 index 00000000..c0dc6988 --- /dev/null +++ b/quickstart/101-synapse/main.tf @@ -0,0 +1,24 @@ +terraform { + required_providers { + azurerm = { + version = "= 3.30.0" + } + } +} + +provider "azurerm" { + features {} +} + +data "azurerm_client_config" "current" {} + +data "http" "ip" { + url = "https://ifconfig.me" +} + +resource "azurerm_resource_group" "default" { + name = "rg-${local.basename}" + location = var.location + + tags = local.tags +} \ No newline at end of file diff --git a/quickstart/101-synapse/storage_account.tf b/quickstart/101-synapse/storage_account.tf new file mode 100644 index 00000000..d7c9a9e2 --- /dev/null +++ b/quickstart/101-synapse/storage_account.tf @@ -0,0 +1,36 @@ +resource "azurerm_storage_account" "default" { + name = "st${local.safe_name}${local.safe_environment}" + resource_group_name = azurerm_resource_group.default.name + location = azurerm_resource_group.default.location + account_tier = "Standard" + account_replication_type = "LRS" + account_kind = "StorageV2" + is_hns_enabled = true +} + +resource "azurerm_role_assignment" "sbdc_current_user" { + scope = azurerm_storage_account.default.id + role_definition_name = "Storage Blob Data Contributor" + principal_id = data.azurerm_client_config.current.object_id +} + +resource "azurerm_role_assignment" "sbdc_syn_ws" { + scope = azurerm_storage_account.default.id + role_definition_name = "Storage Blob Data Contributor" + principal_id = azurerm_synapse_workspace.default.identity[0].principal_id +} + +resource "azurerm_role_assignment" "c_syn_ws" { + scope = azurerm_storage_account.default.id + role_definition_name = "Contributor" + principal_id = azurerm_synapse_workspace.default.identity[0].principal_id +} + +resource "azurerm_storage_data_lake_gen2_filesystem" "default" { + name = "default" + storage_account_id = azurerm_storage_account.default.id + + depends_on = [ + azurerm_role_assignment.sbdc_current_user + ] +} \ No newline at end of file diff --git a/quickstart/101-synapse/synapse_pools.tf b/quickstart/101-synapse/synapse_pools.tf new file mode 100644 index 00000000..baf3f571 --- /dev/null +++ b/quickstart/101-synapse/synapse_pools.tf @@ -0,0 +1,28 @@ +# Sql Pool + +resource "azurerm_synapse_sql_pool" "syn_pool_sql" { + name = "syndp01" + synapse_workspace_id = azurerm_synapse_workspace.default.id + sku_name = "DW100c" + create_mode = "Default" + count = var.enable_syn_sqlpool ? 1 : 0 +} + +# Spark Pool + +resource "azurerm_synapse_spark_pool" "syn_pool_spark" { + name = "synsp01" + synapse_workspace_id = azurerm_synapse_workspace.default.id + node_size_family = "MemoryOptimized" + node_size = "Small" + count = var.enable_syn_sparkpool ? 1 : 0 + + auto_scale { + max_node_count = 50 + min_node_count = 3 + } + + auto_pause { + delay_in_minutes = 15 + } +} \ No newline at end of file diff --git a/quickstart/101-synapse/synapse_workspace.tf b/quickstart/101-synapse/synapse_workspace.tf new file mode 100644 index 00000000..59687938 --- /dev/null +++ b/quickstart/101-synapse/synapse_workspace.tf @@ -0,0 +1,32 @@ +resource "azurerm_synapse_workspace" "default" { + name = "syn-${local.basename}" + resource_group_name = azurerm_resource_group.default.name + location = azurerm_resource_group.default.location + storage_data_lake_gen2_filesystem_id = azurerm_storage_data_lake_gen2_filesystem.default.id + + sql_administrator_login = var.synadmin_username + sql_administrator_login_password = var.synadmin_password + + managed_resource_group_name = "${azurerm_resource_group.default.name}-syn-managed" + + public_network_access_enabled = true + + aad_admin { + login = var.aad_login.name + object_id = var.aad_login.object_id + tenant_id = var.aad_login.tenant_id + } + + identity { + type = "SystemAssigned" + } + + tags = local.tags +} + +resource "azurerm_synapse_firewall_rule" "allow_my_ip" { + name = "AllowMyPublicIp" + synapse_workspace_id = azurerm_synapse_workspace.default.id + start_ip_address = data.http.ip.body + end_ip_address = data.http.ip.body +} diff --git a/quickstart/101-synapse/terraform.tfvars.example b/quickstart/101-synapse/terraform.tfvars.example new file mode 100644 index 00000000..9dfa8a8e --- /dev/null +++ b/quickstart/101-synapse/terraform.tfvars.example @@ -0,0 +1,12 @@ +name = "syn101" +environment = "dev" +location = "East US" + +aad_login = { + name = "azureuser@contoso.com" + object_id = "00000000-0000-0000-0000-000000000000" + tenant_id = "00000000-0000-0000-0000-000000000000" +} + +enable_syn_sparkpool = true +enable_syn_sqlpool = true \ No newline at end of file diff --git a/quickstart/101-synapse/variables.tf b/quickstart/101-synapse/variables.tf new file mode 100644 index 00000000..4bfb92fc --- /dev/null +++ b/quickstart/101-synapse/variables.tf @@ -0,0 +1,54 @@ +variable "name" { + type = string + description = "Name of the deployment" +} + +variable "environment" { + type = string + description = "Name of the environment" + default = "dev" +} + +variable "location" { + type = string + description = "Location of the resources" + default = "East US" +} + +variable "aad_login" { + description = "AAD login" + type = object({ + name = string + object_id = string + tenant_id = string + }) + default = { + name = "AzureAD Admin" + object_id = "00000000-0000-0000-0000-000000000000" + tenant_id = "00000000-0000-0000-0000-000000000000" + } +} + +variable "synadmin_username" { + type = string + description = "Specifies The login name of the SQL administrator" + default = "sqladminuser" +} + +variable "synadmin_password" { + type = string + description = "The Password associated with the sql_administrator_login for the SQL administrator" + default = "ThisIsNotVerySecure!" +} + +variable "enable_syn_sparkpool" { + type = bool + description = "Variable to enable or disable Synapse Spark pool deployment" + default = false +} + +variable "enable_syn_sqlpool" { + type = bool + description = "Variable to enable or disable Synapse Dedicated SQL pool deployment" + default = false +} \ No newline at end of file From a165449f646267408836d1c5ee050deacfd6a546 Mon Sep 17 00:00:00 2001 From: murggu Date: Tue, 8 Nov 2022 12:52:09 +0100 Subject: [PATCH 2/9] Update README.md --- quickstart/101-synapse/README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/quickstart/101-synapse/README.md b/quickstart/101-synapse/README.md index ab8dd2d8..2f829ddc 100644 --- a/quickstart/101-synapse/README.md +++ b/quickstart/101-synapse/README.md @@ -25,11 +25,11 @@ Network connectivity to the workspace is allowed over public endpoints, making t | name | Name of the deployment | - | | environment | The deployment environment name (used for pre- and postfixing resource names) | dev | | location | The Azure region used for deployments | East US | -| aad_admin.login | The login name of the Azure AD Administrator of this Synapse Workspace | - | -| aad_admin.object_id| The object id of the Azure AD Administrator of this Synapse Workspace | - | -| aad_admin.tenant_id| The tenant id of the Azure AD Administrator of this Synapse Workspace | - | -| synadmin_username| Specifies The login name of the SQL administrator | sqladminuser | -| synadmin_password| The Password associated with the sql_administrator_login for the SQL administrator | ThisIsNotVerySecure! | +| aad_admin.login | The login name of the Azure AD Administrator of the Synapse Workspace | - | +| aad_admin.object_id| The object id of the Azure AD Administrator of the Synapse Workspace | - | +| aad_admin.tenant_id| The tenant id of the Azure AD Administrator of the Synapse Workspace | - | +| synadmin_username| Specifies the login name of the SQL administrator | sqladminuser | +| synadmin_password| The Password associated with the synadmin_username for the SQL administrator | ThisIsNotVerySecure! | | enable_syn_sparkpool| A feature flag to enable/disable the Spark pool | false | | enable_syn_sqlpool| A feature flag to enable/disable the SQL pool | false | From ebf66eab0b36adf58e87c98188e35a3aa6a6af1f Mon Sep 17 00:00:00 2001 From: murggu Date: Tue, 8 Nov 2022 12:58:29 +0100 Subject: [PATCH 3/9] add tags --- quickstart/101-synapse/storage_account.tf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/quickstart/101-synapse/storage_account.tf b/quickstart/101-synapse/storage_account.tf index d7c9a9e2..39f23dbc 100644 --- a/quickstart/101-synapse/storage_account.tf +++ b/quickstart/101-synapse/storage_account.tf @@ -6,6 +6,8 @@ resource "azurerm_storage_account" "default" { account_replication_type = "LRS" account_kind = "StorageV2" is_hns_enabled = true + + tags = local.tags } resource "azurerm_role_assignment" "sbdc_current_user" { From d65c97c30714a8be606b7a92300198cbf24323e5 Mon Sep 17 00:00:00 2001 From: murggu Date: Tue, 8 Nov 2022 13:47:23 +0100 Subject: [PATCH 4/9] Add synapse 201 --- quickstart/101-synapse/locals.tf | 8 +- quickstart/101-synapse/main.tf | 2 - quickstart/101-synapse/storage_account.tf | 4 +- quickstart/101-synapse/synapse_workspace.tf | 4 - quickstart/201-synapse-secure/.gitignore | 11 ++ quickstart/201-synapse-secure/README.md | 64 ++++++++++ quickstart/201-synapse-secure/bastion.tf | 19 +++ quickstart/201-synapse-secure/jumphost.tf | 87 +++++++++++++ quickstart/201-synapse-secure/locals.tf | 4 + quickstart/201-synapse-secure/main.tf | 22 ++++ quickstart/201-synapse-secure/network.tf | 24 ++++ .../201-synapse-secure/storage_account.tf | 115 +++++++++++++++++ .../201-synapse-secure/synapse_pools.tf | 28 +++++ .../synapse_private_link_hub.tf | 33 +++++ .../201-synapse-secure/synapse_workspace.tf | 116 ++++++++++++++++++ .../terraform.tfvars.example | 12 ++ quickstart/201-synapse-secure/variables.tf | 66 ++++++++++ 17 files changed, 603 insertions(+), 16 deletions(-) create mode 100644 quickstart/201-synapse-secure/.gitignore create mode 100644 quickstart/201-synapse-secure/README.md create mode 100644 quickstart/201-synapse-secure/bastion.tf create mode 100644 quickstart/201-synapse-secure/jumphost.tf create mode 100644 quickstart/201-synapse-secure/locals.tf create mode 100644 quickstart/201-synapse-secure/main.tf create mode 100644 quickstart/201-synapse-secure/network.tf create mode 100644 quickstart/201-synapse-secure/storage_account.tf create mode 100644 quickstart/201-synapse-secure/synapse_pools.tf create mode 100644 quickstart/201-synapse-secure/synapse_private_link_hub.tf create mode 100644 quickstart/201-synapse-secure/synapse_workspace.tf create mode 100644 quickstart/201-synapse-secure/terraform.tfvars.example create mode 100644 quickstart/201-synapse-secure/variables.tf diff --git a/quickstart/101-synapse/locals.tf b/quickstart/101-synapse/locals.tf index 2d4911c5..6e7e93e2 100644 --- a/quickstart/101-synapse/locals.tf +++ b/quickstart/101-synapse/locals.tf @@ -1,10 +1,4 @@ locals { - tags = { - Toolkit = "Terraform" - } - - safe_name = replace(var.name, "-", "") - safe_environment = replace(var.environment, "-", "") - basename = "${var.name}-${var.environment}" + safe_basename = replace(local.basename, "-", "") } \ No newline at end of file diff --git a/quickstart/101-synapse/main.tf b/quickstart/101-synapse/main.tf index c0dc6988..dfa3bd98 100644 --- a/quickstart/101-synapse/main.tf +++ b/quickstart/101-synapse/main.tf @@ -19,6 +19,4 @@ data "http" "ip" { resource "azurerm_resource_group" "default" { name = "rg-${local.basename}" location = var.location - - tags = local.tags } \ No newline at end of file diff --git a/quickstart/101-synapse/storage_account.tf b/quickstart/101-synapse/storage_account.tf index 39f23dbc..08e072c9 100644 --- a/quickstart/101-synapse/storage_account.tf +++ b/quickstart/101-synapse/storage_account.tf @@ -1,13 +1,11 @@ resource "azurerm_storage_account" "default" { - name = "st${local.safe_name}${local.safe_environment}" + name = "st${local.safe_basename}" resource_group_name = azurerm_resource_group.default.name location = azurerm_resource_group.default.location account_tier = "Standard" account_replication_type = "LRS" account_kind = "StorageV2" is_hns_enabled = true - - tags = local.tags } resource "azurerm_role_assignment" "sbdc_current_user" { diff --git a/quickstart/101-synapse/synapse_workspace.tf b/quickstart/101-synapse/synapse_workspace.tf index 59687938..5673dedd 100644 --- a/quickstart/101-synapse/synapse_workspace.tf +++ b/quickstart/101-synapse/synapse_workspace.tf @@ -9,8 +9,6 @@ resource "azurerm_synapse_workspace" "default" { managed_resource_group_name = "${azurerm_resource_group.default.name}-syn-managed" - public_network_access_enabled = true - aad_admin { login = var.aad_login.name object_id = var.aad_login.object_id @@ -20,8 +18,6 @@ resource "azurerm_synapse_workspace" "default" { identity { type = "SystemAssigned" } - - tags = local.tags } resource "azurerm_synapse_firewall_rule" "allow_my_ip" { diff --git a/quickstart/201-synapse-secure/.gitignore b/quickstart/201-synapse-secure/.gitignore new file mode 100644 index 00000000..cc744186 --- /dev/null +++ b/quickstart/201-synapse-secure/.gitignore @@ -0,0 +1,11 @@ +# Terraform specific + +.terraform + +.terraform.lock.hcl +terraform.tfstate +terraform.tfstate.backup +.terraform.tfstate.lock.info +terraform.tfvars +**.tfbackend +state/ \ No newline at end of file diff --git a/quickstart/201-synapse-secure/README.md b/quickstart/201-synapse-secure/README.md new file mode 100644 index 00000000..69fa2ece --- /dev/null +++ b/quickstart/201-synapse-secure/README.md @@ -0,0 +1,64 @@ +# Azure Synapse Analytics workspace (moderately secure network set up) + +This deployment configuration specifies an [Azure Synapse Analytics workspace](https://learn.microsoft.com/en-us/azure/synapse-analytics/get-started-create-workspace), +and its associated resources including Azure Data Lake Storage (gen2), Synapse Spark Pool and Synapse SQL Pool. + +In addition to these core services, this configuration specifies any networking components that are required to set up Azure Synapse Analytics +for private network connectivity using [Azure Private Link](https://docs.microsoft.com/en-us/azure/private-link/). + +This configuration describes the minimal set of resources you require to get started with Azure Synapse Analytics in a network-isolated set-up. This configuration creates new network components. Use Azure Bastion to securely connect to the Virtual Machine. + +## Resources + +| Terraform Resource Type | Description | +| - | - | +| `azurerm_resource_group` | The resource group all resources get deployed into | +| `azurerm_bastion_host` | An Azure Bastion Instance to securely RDP/SSH into Virtual Machines deployed into the Virtual Network | +| `azurerm_windows_virtual_machine` | A Windows Data Science Virtual Machine used for connecting to the Azure Machine Learning workspace | +| `azurerm_application_insights` | An Azure Application Insights instance associated to the Azure Machine Learning workspace | +| `azurerm_key_vault` | An Azure Key Vault instance associated to the Azure Machine Learning workspace | +| `azurerm_storage_account` | An Azure Storage instance associated to the Azure Machine Learning workspace | +| `azurerm_container_registry` | An Azure Container Registry instance associated to the Azure Machine Learning workspace | +| `azurerm_machine_learning_workspace` | An Azure Machine Learning workspace instance | +| `azurerm_virtual_network` | An Azure Machine Learning workspace instance | +| `azurerm_subnet` | An Azure Machine Learning workspace instance | +| `azurerm_private_dns_zone` | Private DNS Zones for FQDNs required for Azure Machine Learning and associated resources | +| `azurerm_private_dns_zone_virtual_network_link` | Virtual network links of the Private DNS Zones to the virtual network resource | +| `azurerm_private_endpoint` | Private Endpoints for the Azure Machine Learning workspace and associated resources | +| `azurerm_machine_learning_compute_instance` | An Azure Machine Learning compute instance a single-node managed compute. | +| `azurerm_machine_learning_compute_cluster` | An Azure Machine Learning compute cluster as multi-node shared and managed compute. | +| `azurerm_network_security_group` | Network security group with required inbound and outbound rules for Azure Machine Learning. | + +## Variables + +| Name | Description | Default | +|-|-|-| +| name | Name of the deployment | - | +| environment | The deployment environment name (used for pre- and postfixing resource names) | dev | +| location | The Azure region used for deployments | East US | +| vnet_address_space | Address space of the virtual network | ["10.0.0.0/16"] | +| training_subnet_address_space | Address space of the training subnet | ["10.0.1.0/24"] | +| aks_subnet_address_space | Address space of the aks subnet | ["10.0.2.0/23"] | +| ml_subnet_address_space | Address space of the ML workspace subnet | ["10.0.0.0/24"] | +| image_build_compute_name | Name of the compute cluster to be created and configured for building docker images (Azure ML Environments) | image-builder | +| dsvm_name | Name of the Windows Data Science VM resource | vmdsvm01 | +| dsvm_admin_username | Admin username of the Windows Data Science VM | azureadmin | +| dsvm_host_password | Password for the admin username of the Data Science VM | - | + + +## Usage + +1. Copy `terraform.tfvars.example` to `terraform.tfvars` +2. Update `terraform.tfvars` with your desired values +3. Run Terraform + ```console + $ terraform init + $ terraform plan + $ terraform apply + ``` + +## Learn more + +- If you are new to Azure Synapse Analytics, see [Azure Synapse Analytics service](https://azure.microsoft.com/services/synapse-analytics/) and [Azure Synapse Analytics documentation](https://learn.microsoft.com/azure/synapse-analytics/overview-what-is). +- To learn more about security configurations in Azure Synapse Analytics, see [Azure Synapse Analytics security white paper](https://learn.microsoft.com/azure/synapse-analytics/guidance/security-white-paper-introduction). +- For all configurations of Azure Synapse Analytics in Terraform, see [Terraform Hashicorp AzureRM provider documentation](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/synapse_workspace). \ No newline at end of file diff --git a/quickstart/201-synapse-secure/bastion.tf b/quickstart/201-synapse-secure/bastion.tf new file mode 100644 index 00000000..4c245c6f --- /dev/null +++ b/quickstart/201-synapse-secure/bastion.tf @@ -0,0 +1,19 @@ +resource "azurerm_bastion_host" "default" { + name = "bas-${local.basename}" + location = azurerm_resource_group.default.location + resource_group_name = azurerm_resource_group.default.name + + ip_configuration { + name = "configuration" + subnet_id = azurerm_subnet.bastion.id + public_ip_address_id = azurerm_public_ip.default.id + } +} + +resource "azurerm_public_ip" "default" { + name = "pip-${local.basename}" + location = azurerm_resource_group.default.location + resource_group_name = azurerm_resource_group.default.name + allocation_method = "Static" + sku = "Standard" +} \ No newline at end of file diff --git a/quickstart/201-synapse-secure/jumphost.tf b/quickstart/201-synapse-secure/jumphost.tf new file mode 100644 index 00000000..957c8400 --- /dev/null +++ b/quickstart/201-synapse-secure/jumphost.tf @@ -0,0 +1,87 @@ +resource "azurerm_virtual_machine" "jumphost" { + name = "wvm-${local.basename}" + location = azurerm_resource_group.default.location + resource_group_name = azurerm_resource_group.default.name + network_interface_ids = [azurerm_network_interface.jumphost_nic.id] + vm_size = "Standard_DS3_v2" + + delete_os_disk_on_termination = true + delete_data_disks_on_termination = true + + storage_image_reference { + publisher = "microsoft-dsvm" + offer = "dsvm-win-2019" + sku = "server-2019" + version = "latest" + } + + os_profile { + computer_name = "jumphost" + admin_username = var.jumphost_username + admin_password = var.jumphost_password + } + + os_profile_windows_config { + provision_vm_agent = true + enable_automatic_upgrades = true + } + + identity { + type = "SystemAssigned" + } + + storage_os_disk { + name = "disk-${local.basename}" + caching = "ReadWrite" + create_option = "FromImage" + managed_disk_type = "StandardSSD_LRS" + } +} + +resource "azurerm_network_interface" "jumphost_nic" { + name = "nic-${local.basename}" + location = azurerm_resource_group.default.location + resource_group_name = azurerm_resource_group.default.name + + ip_configuration { + name = "configuration" + private_ip_address_allocation = "Dynamic" + subnet_id = azurerm_subnet.default.id + } +} + +resource "azurerm_network_security_group" "jumphost_nsg" { + name = "nsg-${local.basename}" + location = azurerm_resource_group.default.location + resource_group_name = azurerm_resource_group.default.name + + security_rule { + name = "RDP" + priority = 1010 + direction = "Inbound" + access = "Allow" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = 3389 + source_address_prefix = "*" + destination_address_prefix = "*" + } +} + +resource "azurerm_network_interface_security_group_association" "syn_jumphost_nsg_association" { + network_interface_id = azurerm_network_interface.jumphost_nic.id + network_security_group_id = azurerm_network_security_group.jumphost_nsg.id +} + +resource "azurerm_dev_test_global_vm_shutdown_schedule" "syn_jumphost_schedule" { + virtual_machine_id = azurerm_virtual_machine.jumphost.id + location = azurerm_resource_group.default.location + enabled = true + + daily_recurrence_time = "2000" + timezone = "W. Europe Standard Time" + + notification_settings { + enabled = false + } +} \ No newline at end of file diff --git a/quickstart/201-synapse-secure/locals.tf b/quickstart/201-synapse-secure/locals.tf new file mode 100644 index 00000000..6e7e93e2 --- /dev/null +++ b/quickstart/201-synapse-secure/locals.tf @@ -0,0 +1,4 @@ +locals { + basename = "${var.name}-${var.environment}" + safe_basename = replace(local.basename, "-", "") +} \ No newline at end of file diff --git a/quickstart/201-synapse-secure/main.tf b/quickstart/201-synapse-secure/main.tf new file mode 100644 index 00000000..dfa3bd98 --- /dev/null +++ b/quickstart/201-synapse-secure/main.tf @@ -0,0 +1,22 @@ +terraform { + required_providers { + azurerm = { + version = "= 3.30.0" + } + } +} + +provider "azurerm" { + features {} +} + +data "azurerm_client_config" "current" {} + +data "http" "ip" { + url = "https://ifconfig.me" +} + +resource "azurerm_resource_group" "default" { + name = "rg-${local.basename}" + location = var.location +} \ No newline at end of file diff --git a/quickstart/201-synapse-secure/network.tf b/quickstart/201-synapse-secure/network.tf new file mode 100644 index 00000000..567e5ac3 --- /dev/null +++ b/quickstart/201-synapse-secure/network.tf @@ -0,0 +1,24 @@ +resource "azurerm_virtual_network" "default" { + name = "vnet-${local.basename}" + address_space = ["10.0.0.0/16"] + location = azurerm_resource_group.default.location + resource_group_name = azurerm_resource_group.default.name +} + +# Subnets + +resource "azurerm_subnet" "default" { + name = "snet-${local.basename}" + resource_group_name = azurerm_resource_group.default.name + virtual_network_name = azurerm_virtual_network.default.name + address_prefixes = ["10.0.1.0/24"] + service_endpoints = [] + enforce_private_link_endpoint_network_policies = true +} + +resource "azurerm_subnet" "bastion" { + name = "AzureBastionSubnet" + resource_group_name = azurerm_resource_group.default.name + virtual_network_name = azurerm_virtual_network.default.name + address_prefixes = ["10.0.10.0/27"] +} \ No newline at end of file diff --git a/quickstart/201-synapse-secure/storage_account.tf b/quickstart/201-synapse-secure/storage_account.tf new file mode 100644 index 00000000..756c0353 --- /dev/null +++ b/quickstart/201-synapse-secure/storage_account.tf @@ -0,0 +1,115 @@ +resource "azurerm_storage_account" "default" { + name = "st${local.safe_basename}" + resource_group_name = azurerm_resource_group.default.name + location = azurerm_resource_group.default.location + account_tier = "Standard" + account_replication_type = "LRS" + account_kind = "StorageV2" + is_hns_enabled = true +} + +resource "azurerm_role_assignment" "sbdc_current_user" { + scope = azurerm_storage_account.default.id + role_definition_name = "Storage Blob Data Contributor" + principal_id = data.azurerm_client_config.current.object_id +} + +resource "azurerm_role_assignment" "sbdc_syn_ws" { + scope = azurerm_storage_account.default.id + role_definition_name = "Storage Blob Data Contributor" + principal_id = azurerm_synapse_workspace.default.identity[0].principal_id +} + +resource "azurerm_role_assignment" "c_syn_ws" { + scope = azurerm_storage_account.default.id + role_definition_name = "Contributor" + principal_id = azurerm_synapse_workspace.default.identity[0].principal_id +} + +resource "azurerm_storage_data_lake_gen2_filesystem" "default" { + name = "default" + storage_account_id = azurerm_storage_account.default.id + + depends_on = [ + azurerm_role_assignment.sbdc_current_user + ] +} + +# Virtual Network & Firewall configuration + +resource "azurerm_storage_account_network_rules" "firewall_rules" { + storage_account_id = azurerm_storage_account.default.id + + default_action = "Deny" + ip_rules = [data.http.ip.body] + virtual_network_subnet_ids = [] + bypass = ["None"] +} + +# DNS Zones + +resource "azurerm_private_dns_zone" "zone_blob" { + name = "privatelink.blob.core.windows.net" + resource_group_name = azurerm_resource_group.default.name +} + +resource "azurerm_private_dns_zone" "zone_dfs" { + name = "privatelink.dfs.core.windows.net" + resource_group_name = azurerm_resource_group.default.name +} + +# Linking of DNS zones to Virtual Network + +resource "azurerm_private_dns_zone_virtual_network_link" "zone_blob_link" { + name = "${local.basename}_link_blob" + resource_group_name = azurerm_resource_group.default.name + private_dns_zone_name = azurerm_private_dns_zone.zone_blob.name + virtual_network_id = azurerm_virtual_network.default.id +} + +resource "azurerm_private_dns_zone_virtual_network_link" "zone_dfs_link" { + name = "${local.basename}_link_dfs" + resource_group_name = azurerm_resource_group.default.name + private_dns_zone_name = azurerm_private_dns_zone.zone_dfs.name + virtual_network_id = azurerm_virtual_network.default.id +} + +# Private Endpoint configuration + +resource "azurerm_private_endpoint" "pe_blob" { + name = "pe-${azurerm_storage_account.default.name}-blob" + location = azurerm_resource_group.default.location + resource_group_name = azurerm_resource_group.default.name + subnet_id = azurerm_subnet.default.id + + private_service_connection { + name = "psc-blob-${local.basename}" + private_connection_resource_id = azurerm_storage_account.default.id + subresource_names = ["blob"] + is_manual_connection = false + } + + private_dns_zone_group { + name = "private-dns-zone-group-blob" + private_dns_zone_ids = [azurerm_private_dns_zone.zone_blob.id] + } +} + +resource "azurerm_private_endpoint" "pe_dfs" { + name = "pe-${azurerm_storage_account.default.name}-dfs" + location = azurerm_resource_group.default.location + resource_group_name = azurerm_resource_group.default.name + subnet_id = azurerm_subnet.default.id + + private_service_connection { + name = "psc-dfs-${local.basename}" + private_connection_resource_id = azurerm_storage_account.default.id + subresource_names = ["dfs"] + is_manual_connection = false + } + + private_dns_zone_group { + name = "private-dns-zone-group-dfs" + private_dns_zone_ids = [azurerm_private_dns_zone.zone_dfs.id] + } +} \ No newline at end of file diff --git a/quickstart/201-synapse-secure/synapse_pools.tf b/quickstart/201-synapse-secure/synapse_pools.tf new file mode 100644 index 00000000..baf3f571 --- /dev/null +++ b/quickstart/201-synapse-secure/synapse_pools.tf @@ -0,0 +1,28 @@ +# Sql Pool + +resource "azurerm_synapse_sql_pool" "syn_pool_sql" { + name = "syndp01" + synapse_workspace_id = azurerm_synapse_workspace.default.id + sku_name = "DW100c" + create_mode = "Default" + count = var.enable_syn_sqlpool ? 1 : 0 +} + +# Spark Pool + +resource "azurerm_synapse_spark_pool" "syn_pool_spark" { + name = "synsp01" + synapse_workspace_id = azurerm_synapse_workspace.default.id + node_size_family = "MemoryOptimized" + node_size = "Small" + count = var.enable_syn_sparkpool ? 1 : 0 + + auto_scale { + max_node_count = 50 + min_node_count = 3 + } + + auto_pause { + delay_in_minutes = 15 + } +} \ No newline at end of file diff --git a/quickstart/201-synapse-secure/synapse_private_link_hub.tf b/quickstart/201-synapse-secure/synapse_private_link_hub.tf new file mode 100644 index 00000000..86b6edb0 --- /dev/null +++ b/quickstart/201-synapse-secure/synapse_private_link_hub.tf @@ -0,0 +1,33 @@ +resource "azurerm_synapse_private_link_hub" "default" { + name = "synplh${local.safe_basename}" + resource_group_name = azurerm_resource_group.default.name + location = azurerm_resource_group.default.location +} + +# DNS Zones + +resource "azurerm_private_dns_zone" "zone_web" { + name = "privatelink.azuresynapse.net" + resource_group_name = azurerm_resource_group.default.name +} + +# Private Endpoint configuration + +resource "azurerm_private_endpoint" "pe_web" { + name = "pe-${azurerm_synapse_private_link_hub.default.name}-web" + location = azurerm_resource_group.default.location + resource_group_name = azurerm_resource_group.default.name + subnet_id = azurerm_subnet.default.id + + private_service_connection { + name = "psc-web-${local.basename}" + private_connection_resource_id = azurerm_synapse_private_link_hub.default.id + subresource_names = ["web"] + is_manual_connection = false + } + + private_dns_zone_group { + name = "private-dns-zone-group-syn-web" + private_dns_zone_ids = [azurerm_private_dns_zone.zone_web.id] + } +} \ No newline at end of file diff --git a/quickstart/201-synapse-secure/synapse_workspace.tf b/quickstart/201-synapse-secure/synapse_workspace.tf new file mode 100644 index 00000000..848c860a --- /dev/null +++ b/quickstart/201-synapse-secure/synapse_workspace.tf @@ -0,0 +1,116 @@ +resource "azurerm_synapse_workspace" "default" { + name = "syn-${local.basename}" + resource_group_name = azurerm_resource_group.default.name + location = azurerm_resource_group.default.location + storage_data_lake_gen2_filesystem_id = azurerm_storage_data_lake_gen2_filesystem.default.id + + sql_administrator_login = var.synadmin_username + sql_administrator_login_password = var.synadmin_password + + managed_virtual_network_enabled = true + managed_resource_group_name = "${azurerm_resource_group.default.name}-syn-managed" + + aad_admin { + login = var.aad_login.name + object_id = var.aad_login.object_id + tenant_id = var.aad_login.tenant_id + } + + identity { + type = "SystemAssigned" + } +} + +resource "azurerm_synapse_firewall_rule" "allow_my_ip" { + name = "AllowMyPublicIp" + synapse_workspace_id = azurerm_synapse_workspace.default.id + start_ip_address = data.http.ip.body + end_ip_address = data.http.ip.body +} + +# DNS Zones + +resource "azurerm_private_dns_zone" "zone_dev" { + name = "privatelink.dev.azuresynapse.net" + resource_group_name = azurerm_resource_group.default.name +} + +resource "azurerm_private_dns_zone" "zone_sql" { + name = "privatelink.sql.azuresynapse.net" + resource_group_name = azurerm_resource_group.default.name +} + +# Linking of DNS zones to Virtual Network + +resource "azurerm_private_dns_zone_virtual_network_link" "zone_dev_link" { + name = "${local.basename}_link_dev" + resource_group_name = azurerm_resource_group.default.name + private_dns_zone_name = azurerm_private_dns_zone.zone_dev.name + virtual_network_id = azurerm_virtual_network.default.id +} + +resource "azurerm_private_dns_zone_virtual_network_link" "zone_sql_link" { + name = "${local.basename}_link_sql" + resource_group_name = azurerm_resource_group.default.name + private_dns_zone_name = azurerm_private_dns_zone.zone_sql.name + virtual_network_id = azurerm_virtual_network.default.id +} + +# Private Endpoint configuration + +resource "azurerm_private_endpoint" "pe_dev" { + name = "pe-${azurerm_synapse_workspace.default.name}-dev" + location = azurerm_resource_group.default.location + resource_group_name = azurerm_resource_group.default.name + subnet_id = azurerm_subnet.default.id + + private_service_connection { + name = "psc-dev-${local.basename}" + private_connection_resource_id = azurerm_synapse_workspace.default.id + subresource_names = ["dev"] + is_manual_connection = false + } + + private_dns_zone_group { + name = "private-dns-zone-group-dev" + private_dns_zone_ids = [azurerm_private_dns_zone.zone_dev.id] + } +} + +resource "azurerm_private_endpoint" "pe_sql" { + name = "pe-${azurerm_synapse_workspace.default.name}-sql" + location = azurerm_resource_group.default.location + resource_group_name = azurerm_resource_group.default.name + subnet_id = azurerm_subnet.default.id + + private_service_connection { + name = "psc-sql-${local.basename}" + private_connection_resource_id = azurerm_synapse_workspace.default.id + subresource_names = ["sql"] + is_manual_connection = false + } + + private_dns_zone_group { + name = "private-dns-zone-group-sql" + private_dns_zone_ids = [azurerm_private_dns_zone.zone_sql.id] + } +} + +resource "azurerm_private_endpoint" "pe_sqlondemand" { + name = "pe-${azurerm_synapse_workspace.default.name}-sqlondemand" + location = azurerm_resource_group.default.location + resource_group_name = azurerm_resource_group.default.name + subnet_id = azurerm_subnet.default.id + + private_service_connection { + name = "psc-sqlondemand-${local.basename}" + private_connection_resource_id = azurerm_synapse_workspace.default.id + subresource_names = ["sqlondemand"] + is_manual_connection = false + } + + private_dns_zone_group { + name = "private-dns-zone-group-sqlondemand" + private_dns_zone_ids = [azurerm_private_dns_zone.zone_sql.id] + } +} diff --git a/quickstart/201-synapse-secure/terraform.tfvars.example b/quickstart/201-synapse-secure/terraform.tfvars.example new file mode 100644 index 00000000..9dfa8a8e --- /dev/null +++ b/quickstart/201-synapse-secure/terraform.tfvars.example @@ -0,0 +1,12 @@ +name = "syn101" +environment = "dev" +location = "East US" + +aad_login = { + name = "azureuser@contoso.com" + object_id = "00000000-0000-0000-0000-000000000000" + tenant_id = "00000000-0000-0000-0000-000000000000" +} + +enable_syn_sparkpool = true +enable_syn_sqlpool = true \ No newline at end of file diff --git a/quickstart/201-synapse-secure/variables.tf b/quickstart/201-synapse-secure/variables.tf new file mode 100644 index 00000000..ce26c0e4 --- /dev/null +++ b/quickstart/201-synapse-secure/variables.tf @@ -0,0 +1,66 @@ +variable "name" { + type = string + description = "Name of the deployment" +} + +variable "environment" { + type = string + description = "Name of the environment" + default = "dev" +} + +variable "location" { + type = string + description = "Location of the resources" + default = "East US" +} + +variable "aad_login" { + description = "AAD login" + type = object({ + name = string + object_id = string + tenant_id = string + }) + default = { + name = "AzureAD Admin" + object_id = "00000000-0000-0000-0000-000000000000" + tenant_id = "00000000-0000-0000-0000-000000000000" + } +} + +variable "jumphost_username" { + type = string + description = "Admin username of the VM" + default = "azureuser" +} + +variable "jumphost_password" { + type = string + description = "Password for the admin username of the VM" + default = "ThisIsNotVerySecure!" +} + +variable "synadmin_username" { + type = string + description = "Specifies The login name of the SQL administrator" + default = "sqladminuser" +} + +variable "synadmin_password" { + type = string + description = "The Password associated with the sql_administrator_login for the SQL administrator" + default = "ThisIsNotVerySecure!" +} + +variable "enable_syn_sparkpool" { + type = bool + description = "Variable to enable or disable Synapse Spark pool deployment" + default = false +} + +variable "enable_syn_sqlpool" { + type = bool + description = "Variable to enable or disable Synapse Dedicated SQL pool deployment" + default = false +} \ No newline at end of file From a0ddf338d3224c0839b110c753a802ee0bc42858 Mon Sep 17 00:00:00 2001 From: murggu Date: Tue, 8 Nov 2022 13:57:31 +0100 Subject: [PATCH 5/9] update readme.md --- quickstart/201-synapse-secure/README.md | 44 +++++++++++-------------- 1 file changed, 20 insertions(+), 24 deletions(-) diff --git a/quickstart/201-synapse-secure/README.md b/quickstart/201-synapse-secure/README.md index 69fa2ece..541774e2 100644 --- a/quickstart/201-synapse-secure/README.md +++ b/quickstart/201-synapse-secure/README.md @@ -12,22 +12,16 @@ This configuration describes the minimal set of resources you require to get sta | Terraform Resource Type | Description | | - | - | -| `azurerm_resource_group` | The resource group all resources get deployed into | -| `azurerm_bastion_host` | An Azure Bastion Instance to securely RDP/SSH into Virtual Machines deployed into the Virtual Network | -| `azurerm_windows_virtual_machine` | A Windows Data Science Virtual Machine used for connecting to the Azure Machine Learning workspace | -| `azurerm_application_insights` | An Azure Application Insights instance associated to the Azure Machine Learning workspace | -| `azurerm_key_vault` | An Azure Key Vault instance associated to the Azure Machine Learning workspace | -| `azurerm_storage_account` | An Azure Storage instance associated to the Azure Machine Learning workspace | -| `azurerm_container_registry` | An Azure Container Registry instance associated to the Azure Machine Learning workspace | -| `azurerm_machine_learning_workspace` | An Azure Machine Learning workspace instance | -| `azurerm_virtual_network` | An Azure Machine Learning workspace instance | -| `azurerm_subnet` | An Azure Machine Learning workspace instance | -| `azurerm_private_dns_zone` | Private DNS Zones for FQDNs required for Azure Machine Learning and associated resources | -| `azurerm_private_dns_zone_virtual_network_link` | Virtual network links of the Private DNS Zones to the virtual network resource | -| `azurerm_private_endpoint` | Private Endpoints for the Azure Machine Learning workspace and associated resources | -| `azurerm_machine_learning_compute_instance` | An Azure Machine Learning compute instance a single-node managed compute. | -| `azurerm_machine_learning_compute_cluster` | An Azure Machine Learning compute cluster as multi-node shared and managed compute. | -| `azurerm_network_security_group` | Network security group with required inbound and outbound rules for Azure Machine Learning. | +| `azurerm_resource_group` | The resource group all resources get deployed into. | +| `azurerm_bastion_host` | An Azure Bastion Instance to securely RDP into Virtual Machines deployed into the Virtual Network. | +| `azurerm_windows_virtual_machine` | A Windows Data Science Virtual Machine (jumphost) used for connecting to the Azure Synapse Analytics workspace. | +| `azurerm_storage_account` | An Azure Storage instance associated to the Azure Synapse Analytics workspace. | +| `azurerm_synapse_workspace` | An Azure Synapse Analytics workspace instance. | +| `azurerm_synapse_private_link_hub` | An Azure Synapse Private Link Hub to allow securely connecting to Synapse Studio. | +| `azurerm_synapse_spark_pool` | An Azure Synapse Analytics spark pool. | +| `azurerm_synapse_sql_pool` | An Azure Synapse Analytics dedicated SQL pool. | +| `azurerm_virtual_network` | A default vnet. | +| `azurerm_subnet` | `bastion` and `default` subnets. | ## Variables @@ -36,14 +30,16 @@ This configuration describes the minimal set of resources you require to get sta | name | Name of the deployment | - | | environment | The deployment environment name (used for pre- and postfixing resource names) | dev | | location | The Azure region used for deployments | East US | -| vnet_address_space | Address space of the virtual network | ["10.0.0.0/16"] | -| training_subnet_address_space | Address space of the training subnet | ["10.0.1.0/24"] | -| aks_subnet_address_space | Address space of the aks subnet | ["10.0.2.0/23"] | -| ml_subnet_address_space | Address space of the ML workspace subnet | ["10.0.0.0/24"] | -| image_build_compute_name | Name of the compute cluster to be created and configured for building docker images (Azure ML Environments) | image-builder | -| dsvm_name | Name of the Windows Data Science VM resource | vmdsvm01 | -| dsvm_admin_username | Admin username of the Windows Data Science VM | azureadmin | -| dsvm_host_password | Password for the admin username of the Data Science VM | - | +| aad_admin.login | The login name of the Azure AD Administrator of the Synapse Workspace | - | +| aad_admin.object_id| The object id of the Azure AD Administrator of the Synapse Workspace | - | +| aad_admin.tenant_id| The tenant id of the Azure AD Administrator of the Synapse Workspace | - | +| synadmin_username| Specifies the login name of the SQL administrator | sqladminuser | +| synadmin_password| The Password associated with the synadmin_username for the SQL administrator | ThisIsNotVerySecure! | +| jumphost_username| Admin username of the VM | azureuser | +| jumphost_password| Password for the admin username of the VM | ThisIsNotVerySecure! | +| enable_syn_sparkpool| A feature flag to enable/disable the Spark pool | false | +| enable_syn_sqlpool| A feature flag to enable/disable the SQL pool | false | + ## Usage From eb6b615ae331a90828fbfa8a8f437bad988c0bc6 Mon Sep 17 00:00:00 2001 From: murggu Date: Fri, 18 Nov 2022 22:12:00 +0100 Subject: [PATCH 6/9] fix comments --- quickstart/101-synapse/README.md | 7 +++---- quickstart/101-synapse/locals.tf | 2 +- quickstart/101-synapse/main.tf | 12 ------------ quickstart/101-synapse/providers.tf | 11 +++++++++++ quickstart/101-synapse/synapse_workspace.tf | 4 ++-- quickstart/101-synapse/variables.tf | 2 -- quickstart/201-synapse-secure/README.md | 8 +++----- quickstart/201-synapse-secure/locals.tf | 2 +- quickstart/201-synapse-secure/main.tf | 12 ------------ quickstart/201-synapse-secure/network.tf | 12 ++++++------ quickstart/201-synapse-secure/providers.tf | 11 +++++++++++ quickstart/201-synapse-secure/storage_account.tf | 2 +- quickstart/201-synapse-secure/synapse_workspace.tf | 11 +++-------- quickstart/201-synapse-secure/variables.tf | 4 ---- 14 files changed, 42 insertions(+), 58 deletions(-) create mode 100644 quickstart/101-synapse/providers.tf create mode 100644 quickstart/201-synapse-secure/providers.tf diff --git a/quickstart/101-synapse/README.md b/quickstart/101-synapse/README.md index 2f829ddc..d97d3350 100644 --- a/quickstart/101-synapse/README.md +++ b/quickstart/101-synapse/README.md @@ -9,7 +9,6 @@ Network connectivity to the workspace is allowed over public endpoints, making t ## Resources - | Terraform Resource Type | Description | | - | - | | `azurerm_resource_group` | The resource group all resources get deployed into. | @@ -46,6 +45,6 @@ Network connectivity to the workspace is allowed over public endpoints, making t ## Learn more -- If you are new to Azure Synapse Analytics, see [Azure Synapse Analytics service](https://azure.microsoft.com/services/synapse-analytics/) and [Azure Synapse Analytics documentation](https://learn.microsoft.com/azure/synapse-analytics/overview-what-is). -- To learn more about security configurations in Azure Synapse Analytics, see [Azure Synapse Analytics security white paper](https://learn.microsoft.com/azure/synapse-analytics/guidance/security-white-paper-introduction). -- For all configurations of Azure Synapse Analytics in Terraform, see [Terraform Hashicorp AzureRM provider documentation](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/synapse_workspace). +- If you are new to Azure Synapse Analytics, see [Azure Synapse Analytics service](https://azure.microsoft.com/services/synapse-analytics/) and [Azure Synapse Analytics documentation](https://learn.microsoft.com/azure/synapse-analytics/guidance/success-by-design-introduction). +- To learn more about security configurations in Azure Synapse Analytics, see [Azure Synapse Analytics security white paper](https://learn.microsoft.com/azure/synapse-analytics/guidance/security-white-paper-introduction) and watch [Success with Synapse - Security videos](https://www.youtube.com/playlist?list=PLzUAjXZBFU9OWYjSI5TdlpMV0ltAjLaNw). +- For all configurations of Azure Synapse Analytics in Terraform, see [Terraform Hashicorp AzureRM provider documentation](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/synapse_workspace). \ No newline at end of file diff --git a/quickstart/101-synapse/locals.tf b/quickstart/101-synapse/locals.tf index 6e7e93e2..a0b2c59b 100644 --- a/quickstart/101-synapse/locals.tf +++ b/quickstart/101-synapse/locals.tf @@ -1,4 +1,4 @@ locals { - basename = "${var.name}-${var.environment}" + basename = "${var.name}-${var.environment}" safe_basename = replace(local.basename, "-", "") } \ No newline at end of file diff --git a/quickstart/101-synapse/main.tf b/quickstart/101-synapse/main.tf index dfa3bd98..c87b0966 100644 --- a/quickstart/101-synapse/main.tf +++ b/quickstart/101-synapse/main.tf @@ -1,15 +1,3 @@ -terraform { - required_providers { - azurerm = { - version = "= 3.30.0" - } - } -} - -provider "azurerm" { - features {} -} - data "azurerm_client_config" "current" {} data "http" "ip" { diff --git a/quickstart/101-synapse/providers.tf b/quickstart/101-synapse/providers.tf new file mode 100644 index 00000000..356fa182 --- /dev/null +++ b/quickstart/101-synapse/providers.tf @@ -0,0 +1,11 @@ +terraform { + required_providers { + azurerm = { + version = "= 3.32.0" + } + } +} + +provider "azurerm" { + features {} +} \ No newline at end of file diff --git a/quickstart/101-synapse/synapse_workspace.tf b/quickstart/101-synapse/synapse_workspace.tf index 5673dedd..4d64ba80 100644 --- a/quickstart/101-synapse/synapse_workspace.tf +++ b/quickstart/101-synapse/synapse_workspace.tf @@ -23,6 +23,6 @@ resource "azurerm_synapse_workspace" "default" { resource "azurerm_synapse_firewall_rule" "allow_my_ip" { name = "AllowMyPublicIp" synapse_workspace_id = azurerm_synapse_workspace.default.id - start_ip_address = data.http.ip.body - end_ip_address = data.http.ip.body + start_ip_address = data.http.ip.response_body + end_ip_address = data.http.ip.response_body } diff --git a/quickstart/101-synapse/variables.tf b/quickstart/101-synapse/variables.tf index 4bfb92fc..3a8deab5 100644 --- a/quickstart/101-synapse/variables.tf +++ b/quickstart/101-synapse/variables.tf @@ -32,13 +32,11 @@ variable "aad_login" { variable "synadmin_username" { type = string description = "Specifies The login name of the SQL administrator" - default = "sqladminuser" } variable "synadmin_password" { type = string description = "The Password associated with the sql_administrator_login for the SQL administrator" - default = "ThisIsNotVerySecure!" } variable "enable_syn_sparkpool" { diff --git a/quickstart/201-synapse-secure/README.md b/quickstart/201-synapse-secure/README.md index 541774e2..1c004d39 100644 --- a/quickstart/201-synapse-secure/README.md +++ b/quickstart/201-synapse-secure/README.md @@ -6,7 +6,7 @@ and its associated resources including Azure Data Lake Storage (gen2), Synapse S In addition to these core services, this configuration specifies any networking components that are required to set up Azure Synapse Analytics for private network connectivity using [Azure Private Link](https://docs.microsoft.com/en-us/azure/private-link/). -This configuration describes the minimal set of resources you require to get started with Azure Synapse Analytics in a network-isolated set-up. This configuration creates new network components. Use Azure Bastion to securely connect to the Virtual Machine. +This configuration describes the minimal set of resources you require to get started with Azure Synapse Analytics in a network-isolated set-up. This configuration creates new network components. Use Azure Bastion to securely connect to the Virtual Machine. ## Resources @@ -40,8 +40,6 @@ This configuration describes the minimal set of resources you require to get sta | enable_syn_sparkpool| A feature flag to enable/disable the Spark pool | false | | enable_syn_sqlpool| A feature flag to enable/disable the SQL pool | false | - - ## Usage 1. Copy `terraform.tfvars.example` to `terraform.tfvars` @@ -55,6 +53,6 @@ This configuration describes the minimal set of resources you require to get sta ## Learn more -- If you are new to Azure Synapse Analytics, see [Azure Synapse Analytics service](https://azure.microsoft.com/services/synapse-analytics/) and [Azure Synapse Analytics documentation](https://learn.microsoft.com/azure/synapse-analytics/overview-what-is). -- To learn more about security configurations in Azure Synapse Analytics, see [Azure Synapse Analytics security white paper](https://learn.microsoft.com/azure/synapse-analytics/guidance/security-white-paper-introduction). +- If you are new to Azure Synapse Analytics, see [Azure Synapse Analytics service](https://azure.microsoft.com/services/synapse-analytics/) and [Azure Synapse Analytics documentation](https://learn.microsoft.com/azure/synapse-analytics/guidance/success-by-design-introduction). +- To learn more about security configurations in Azure Synapse Analytics, see [Azure Synapse Analytics security white paper](https://learn.microsoft.com/azure/synapse-analytics/guidance/security-white-paper-introduction) and watch [Success with Synapse - Security videos](https://www.youtube.com/playlist?list=PLzUAjXZBFU9OWYjSI5TdlpMV0ltAjLaNw). - For all configurations of Azure Synapse Analytics in Terraform, see [Terraform Hashicorp AzureRM provider documentation](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/synapse_workspace). \ No newline at end of file diff --git a/quickstart/201-synapse-secure/locals.tf b/quickstart/201-synapse-secure/locals.tf index 6e7e93e2..a0b2c59b 100644 --- a/quickstart/201-synapse-secure/locals.tf +++ b/quickstart/201-synapse-secure/locals.tf @@ -1,4 +1,4 @@ locals { - basename = "${var.name}-${var.environment}" + basename = "${var.name}-${var.environment}" safe_basename = replace(local.basename, "-", "") } \ No newline at end of file diff --git a/quickstart/201-synapse-secure/main.tf b/quickstart/201-synapse-secure/main.tf index dfa3bd98..c87b0966 100644 --- a/quickstart/201-synapse-secure/main.tf +++ b/quickstart/201-synapse-secure/main.tf @@ -1,15 +1,3 @@ -terraform { - required_providers { - azurerm = { - version = "= 3.30.0" - } - } -} - -provider "azurerm" { - features {} -} - data "azurerm_client_config" "current" {} data "http" "ip" { diff --git a/quickstart/201-synapse-secure/network.tf b/quickstart/201-synapse-secure/network.tf index 567e5ac3..59e5536e 100644 --- a/quickstart/201-synapse-secure/network.tf +++ b/quickstart/201-synapse-secure/network.tf @@ -8,12 +8,12 @@ resource "azurerm_virtual_network" "default" { # Subnets resource "azurerm_subnet" "default" { - name = "snet-${local.basename}" - resource_group_name = azurerm_resource_group.default.name - virtual_network_name = azurerm_virtual_network.default.name - address_prefixes = ["10.0.1.0/24"] - service_endpoints = [] - enforce_private_link_endpoint_network_policies = true + name = "snet-${local.basename}" + resource_group_name = azurerm_resource_group.default.name + virtual_network_name = azurerm_virtual_network.default.name + address_prefixes = ["10.0.1.0/24"] + service_endpoints = [] + private_endpoint_network_policies_enabled = true } resource "azurerm_subnet" "bastion" { diff --git a/quickstart/201-synapse-secure/providers.tf b/quickstart/201-synapse-secure/providers.tf new file mode 100644 index 00000000..356fa182 --- /dev/null +++ b/quickstart/201-synapse-secure/providers.tf @@ -0,0 +1,11 @@ +terraform { + required_providers { + azurerm = { + version = "= 3.32.0" + } + } +} + +provider "azurerm" { + features {} +} \ No newline at end of file diff --git a/quickstart/201-synapse-secure/storage_account.tf b/quickstart/201-synapse-secure/storage_account.tf index 756c0353..9e2bdf11 100644 --- a/quickstart/201-synapse-secure/storage_account.tf +++ b/quickstart/201-synapse-secure/storage_account.tf @@ -41,7 +41,7 @@ resource "azurerm_storage_account_network_rules" "firewall_rules" { storage_account_id = azurerm_storage_account.default.id default_action = "Deny" - ip_rules = [data.http.ip.body] + ip_rules = [data.http.ip.response_body] virtual_network_subnet_ids = [] bypass = ["None"] } diff --git a/quickstart/201-synapse-secure/synapse_workspace.tf b/quickstart/201-synapse-secure/synapse_workspace.tf index 848c860a..80438596 100644 --- a/quickstart/201-synapse-secure/synapse_workspace.tf +++ b/quickstart/201-synapse-secure/synapse_workspace.tf @@ -8,7 +8,9 @@ resource "azurerm_synapse_workspace" "default" { sql_administrator_login_password = var.synadmin_password managed_virtual_network_enabled = true - managed_resource_group_name = "${azurerm_resource_group.default.name}-syn-managed" + managed_resource_group_name = "${azurerm_resource_group.default.name}-syn-managed" + + public_network_access_enabled = false aad_admin { login = var.aad_login.name @@ -21,13 +23,6 @@ resource "azurerm_synapse_workspace" "default" { } } -resource "azurerm_synapse_firewall_rule" "allow_my_ip" { - name = "AllowMyPublicIp" - synapse_workspace_id = azurerm_synapse_workspace.default.id - start_ip_address = data.http.ip.body - end_ip_address = data.http.ip.body -} - # DNS Zones resource "azurerm_private_dns_zone" "zone_dev" { diff --git a/quickstart/201-synapse-secure/variables.tf b/quickstart/201-synapse-secure/variables.tf index ce26c0e4..ecf212d5 100644 --- a/quickstart/201-synapse-secure/variables.tf +++ b/quickstart/201-synapse-secure/variables.tf @@ -32,25 +32,21 @@ variable "aad_login" { variable "jumphost_username" { type = string description = "Admin username of the VM" - default = "azureuser" } variable "jumphost_password" { type = string description = "Password for the admin username of the VM" - default = "ThisIsNotVerySecure!" } variable "synadmin_username" { type = string description = "Specifies The login name of the SQL administrator" - default = "sqladminuser" } variable "synadmin_password" { type = string description = "The Password associated with the sql_administrator_login for the SQL administrator" - default = "ThisIsNotVerySecure!" } variable "enable_syn_sparkpool" { From d5a7e742a289b8b44f10f6202af4762e034adfe3 Mon Sep 17 00:00:00 2001 From: murggu Date: Fri, 18 Nov 2022 22:14:08 +0100 Subject: [PATCH 7/9] fix tfvars example --- quickstart/101-synapse/terraform.tfvars.example | 11 +++++++---- .../201-synapse-secure/terraform.tfvars.example | 14 ++++++++++---- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/quickstart/101-synapse/terraform.tfvars.example b/quickstart/101-synapse/terraform.tfvars.example index 9dfa8a8e..dba34cc8 100644 --- a/quickstart/101-synapse/terraform.tfvars.example +++ b/quickstart/101-synapse/terraform.tfvars.example @@ -1,6 +1,9 @@ -name = "syn101" +name = "syn101" environment = "dev" -location = "East US" +location = "East US" + +synadmin_username = "sqladminuser" +synadmin_password = "ThisIsNotVerySecure!" aad_login = { name = "azureuser@contoso.com" @@ -8,5 +11,5 @@ aad_login = { tenant_id = "00000000-0000-0000-0000-000000000000" } -enable_syn_sparkpool = true -enable_syn_sqlpool = true \ No newline at end of file +enable_syn_sparkpool = false +enable_syn_sqlpool = false \ No newline at end of file diff --git a/quickstart/201-synapse-secure/terraform.tfvars.example b/quickstart/201-synapse-secure/terraform.tfvars.example index 9dfa8a8e..b28f018e 100644 --- a/quickstart/201-synapse-secure/terraform.tfvars.example +++ b/quickstart/201-synapse-secure/terraform.tfvars.example @@ -1,6 +1,12 @@ -name = "syn101" +name = "syn201" environment = "dev" -location = "East US" +location = "East US" + +jumphost_username = "azureuser" +jumphost_password = "ThisIsNotVerySecure!" + +synadmin_username = "sqladminuser" +synadmin_password = "ThisIsNotVerySecure!" aad_login = { name = "azureuser@contoso.com" @@ -8,5 +14,5 @@ aad_login = { tenant_id = "00000000-0000-0000-0000-000000000000" } -enable_syn_sparkpool = true -enable_syn_sqlpool = true \ No newline at end of file +enable_syn_sparkpool = false +enable_syn_sqlpool = false \ No newline at end of file From 664d1069b824dd8203c64da2e26f00f8c16c080f Mon Sep 17 00:00:00 2001 From: murggu Date: Fri, 18 Nov 2022 22:17:36 +0100 Subject: [PATCH 8/9] remova .gitignores --- quickstart/101-synapse/.gitignore | 11 ----------- quickstart/201-synapse-secure/.gitignore | 11 ----------- 2 files changed, 22 deletions(-) delete mode 100644 quickstart/101-synapse/.gitignore delete mode 100644 quickstart/201-synapse-secure/.gitignore diff --git a/quickstart/101-synapse/.gitignore b/quickstart/101-synapse/.gitignore deleted file mode 100644 index cc744186..00000000 --- a/quickstart/101-synapse/.gitignore +++ /dev/null @@ -1,11 +0,0 @@ -# Terraform specific - -.terraform - -.terraform.lock.hcl -terraform.tfstate -terraform.tfstate.backup -.terraform.tfstate.lock.info -terraform.tfvars -**.tfbackend -state/ \ No newline at end of file diff --git a/quickstart/201-synapse-secure/.gitignore b/quickstart/201-synapse-secure/.gitignore deleted file mode 100644 index cc744186..00000000 --- a/quickstart/201-synapse-secure/.gitignore +++ /dev/null @@ -1,11 +0,0 @@ -# Terraform specific - -.terraform - -.terraform.lock.hcl -terraform.tfstate -terraform.tfstate.backup -.terraform.tfstate.lock.info -terraform.tfvars -**.tfbackend -state/ \ No newline at end of file From f40d33642cffde0d04ced33bed07b6deefdd6c51 Mon Sep 17 00:00:00 2001 From: murggu Date: Mon, 21 Nov 2022 11:35:48 +0100 Subject: [PATCH 9/9] Fix PR comments --- quickstart/101-synapse/README.md | 4 ++-- quickstart/101-synapse/variables.tf | 5 ----- quickstart/201-synapse-secure/README.md | 8 ++++---- quickstart/201-synapse-secure/variables.tf | 5 ----- 4 files changed, 6 insertions(+), 16 deletions(-) diff --git a/quickstart/101-synapse/README.md b/quickstart/101-synapse/README.md index d97d3350..f18f4690 100644 --- a/quickstart/101-synapse/README.md +++ b/quickstart/101-synapse/README.md @@ -27,8 +27,8 @@ Network connectivity to the workspace is allowed over public endpoints, making t | aad_admin.login | The login name of the Azure AD Administrator of the Synapse Workspace | - | | aad_admin.object_id| The object id of the Azure AD Administrator of the Synapse Workspace | - | | aad_admin.tenant_id| The tenant id of the Azure AD Administrator of the Synapse Workspace | - | -| synadmin_username| Specifies the login name of the SQL administrator | sqladminuser | -| synadmin_password| The Password associated with the synadmin_username for the SQL administrator | ThisIsNotVerySecure! | +| synadmin_username| Specifies the login name of the SQL administrator | - | +| synadmin_password| The Password associated with the synadmin_username for the SQL administrator | - | | enable_syn_sparkpool| A feature flag to enable/disable the Spark pool | false | | enable_syn_sqlpool| A feature flag to enable/disable the SQL pool | false | diff --git a/quickstart/101-synapse/variables.tf b/quickstart/101-synapse/variables.tf index 3a8deab5..fe14d367 100644 --- a/quickstart/101-synapse/variables.tf +++ b/quickstart/101-synapse/variables.tf @@ -22,11 +22,6 @@ variable "aad_login" { object_id = string tenant_id = string }) - default = { - name = "AzureAD Admin" - object_id = "00000000-0000-0000-0000-000000000000" - tenant_id = "00000000-0000-0000-0000-000000000000" - } } variable "synadmin_username" { diff --git a/quickstart/201-synapse-secure/README.md b/quickstart/201-synapse-secure/README.md index 1c004d39..0f8a3441 100644 --- a/quickstart/201-synapse-secure/README.md +++ b/quickstart/201-synapse-secure/README.md @@ -33,10 +33,10 @@ This configuration describes the minimal set of resources you require to get sta | aad_admin.login | The login name of the Azure AD Administrator of the Synapse Workspace | - | | aad_admin.object_id| The object id of the Azure AD Administrator of the Synapse Workspace | - | | aad_admin.tenant_id| The tenant id of the Azure AD Administrator of the Synapse Workspace | - | -| synadmin_username| Specifies the login name of the SQL administrator | sqladminuser | -| synadmin_password| The Password associated with the synadmin_username for the SQL administrator | ThisIsNotVerySecure! | -| jumphost_username| Admin username of the VM | azureuser | -| jumphost_password| Password for the admin username of the VM | ThisIsNotVerySecure! | +| synadmin_username| Specifies the login name of the SQL administrator | - | +| synadmin_password| The Password associated with the synadmin_username for the SQL administrator | - | +| jumphost_username| Admin username of the VM | - | +| jumphost_password| Password for the admin username of the VM | - | | enable_syn_sparkpool| A feature flag to enable/disable the Spark pool | false | | enable_syn_sqlpool| A feature flag to enable/disable the SQL pool | false | diff --git a/quickstart/201-synapse-secure/variables.tf b/quickstart/201-synapse-secure/variables.tf index ecf212d5..15d78c4f 100644 --- a/quickstart/201-synapse-secure/variables.tf +++ b/quickstart/201-synapse-secure/variables.tf @@ -22,11 +22,6 @@ variable "aad_login" { object_id = string tenant_id = string }) - default = { - name = "AzureAD Admin" - object_id = "00000000-0000-0000-0000-000000000000" - tenant_id = "00000000-0000-0000-0000-000000000000" - } } variable "jumphost_username" {