From 713cbfc0a79a108622caef0de428e12a205dd3e0 Mon Sep 17 00:00:00 2001 From: Pengfei Ni Date: Mon, 24 Aug 2020 13:49:38 +0800 Subject: [PATCH] Add quickstart sample for private AKS cluster with custom DNS --- quickstart/301-aks-private-cluster/README.md | 55 ++++++++++++++++ quickstart/301-aks-private-cluster/aks.tf | 55 ++++++++++++++++ .../301-aks-private-cluster/dns-zone-link.sh | 35 ++++++++++ .../301-aks-private-cluster/dns-zone.tf | 13 ++++ quickstart/301-aks-private-cluster/main.tf | 4 ++ quickstart/301-aks-private-cluster/output.tf | 7 ++ .../301-aks-private-cluster/variables.tf | 65 +++++++++++++++++++ 7 files changed, 234 insertions(+) create mode 100644 quickstart/301-aks-private-cluster/README.md create mode 100644 quickstart/301-aks-private-cluster/aks.tf create mode 100755 quickstart/301-aks-private-cluster/dns-zone-link.sh create mode 100644 quickstart/301-aks-private-cluster/dns-zone.tf create mode 100644 quickstart/301-aks-private-cluster/main.tf create mode 100644 quickstart/301-aks-private-cluster/output.tf create mode 100644 quickstart/301-aks-private-cluster/variables.tf diff --git a/quickstart/301-aks-private-cluster/README.md b/quickstart/301-aks-private-cluster/README.md new file mode 100644 index 00000000..281a557d --- /dev/null +++ b/quickstart/301-aks-private-cluster/README.md @@ -0,0 +1,55 @@ +# Private Azure Kubernetes Service with Custom DNS Server + +This template deploys a private Azure Kubernetes Service cluster configured with custom DNS server. A new VNet with provided +custom DNS server would be provisioned and the AKS cluster is deployed into this new VNet. The DNS server's VNet would +also be linked to AKS provisioned private DNS zone, so that AKS cluster's private FQDN could be resolved successfully on +custom DNS server. + +To use this template, ensure the following pre-requirements have been set: + +* Azure CLI and terraform installed locally +* Pre-configure DNS servers outside of AKS VNet +* Forward AKS cluster FQDN `azmk8s.io` (or only private cluster FQDN `privatelink..azmk8s.io`) to Azure DNS `168.63.129.16` +* Get the DNS servers IP address, which would be set in `custom_dns` +* Get the DNS server's VNet resource ID, which would be set in `custom_dns_vnet_id` + +## Resources + +| Terraform Resource Type | Description | +|-------------------------|-------------| +| `azurerm_resource_group` | The resource group all resources are deployed into | +| `azurerm_virtual_network` | The VNet that AKS cluster would be deployed on | +| `azurerm_subnet` | The subnet that AKS cluster would be deployed on | +| `azurerm_kubernetes_cluster` | The AKS cluster | +| `null_resource.dns_zone_link` | Link custom DNS server's VNet to AKS private DNS zone| + +## Variables + +| Name | Description | +|------|-------------| +| `resource_group_name` | Name of the Azure resource group| +| `cluster_name` | Name of the AKS cluster| +| `custom_dns` | IP of custom DNS server| +| `custom_dns_vnet_id` |Resource ID of the Azure VNet that holds custom DNS server| +| `client_id` | The service principal ID| +| `client_secret` | The service principal password| +| `agent_count` | The number of K8S nodes to provision| +| `kubernetes_version` | The version of K8S to provision| +| `ssh_public_key` | The SSH public key of K8S nodes | +| `dns_prefix` | The DNS prefix of AKS cluster | +| `location` | The location of Azure resources | + +## Usage + +```sh +terraform plan \ + -var 'resource_group_name=aks-quickstart' \ + -var 'cluster_name=aks' \ + -var 'custom_dns=' \ + -var 'custom_dns_vnet_id=' \ + -var 'client_id=' \ + -var 'client_secret=' \ + -out demo.tfplan + +terraform apply demo.tfplan +``` diff --git a/quickstart/301-aks-private-cluster/aks.tf b/quickstart/301-aks-private-cluster/aks.tf new file mode 100644 index 00000000..54e8a776 --- /dev/null +++ b/quickstart/301-aks-private-cluster/aks.tf @@ -0,0 +1,55 @@ +resource "azurerm_resource_group" "k8s" { + name = "${var.resource_group_name}" + location = "${var.location}" +} + +resource "azurerm_virtual_network" "myvnet" { + name = "myvnet" + location = "${azurerm_resource_group.k8s.location}" + resource_group_name = "${azurerm_resource_group.k8s.name}" + address_space = ["10.240.0.0/16"] + dns_servers = ["${var.custom_dns}"] +} + +resource "azurerm_subnet" "mysubnet" { + name = "mysubnet" + resource_group_name = "${azurerm_resource_group.k8s.name}" + address_prefixes = ["10.240.10.0/24"] + virtual_network_name = "${azurerm_virtual_network.myvnet.name}" +} + +resource "azurerm_kubernetes_cluster" "k8s" { + name = "${var.cluster_name}" + location = "${azurerm_resource_group.k8s.location}" + resource_group_name = "${azurerm_resource_group.k8s.name}" + dns_prefix = "${var.dns_prefix}" + kubernetes_version = "${var.kubernetes_version}" + private_cluster_enabled = true + + linux_profile { + admin_username = "azureuser" + + ssh_key { + key_data = "${file("${var.ssh_public_key}")}" + } + } + + default_node_pool { + name = "agentpool1" + node_count = "${var.agent_count}" + vm_size = "Standard_D2_v3" + os_disk_size_gb = 30 + vnet_subnet_id = "${azurerm_subnet.mysubnet.id}" + } + + service_principal { + client_id = "${var.client_id}" + client_secret = "${var.client_secret}" + } + + network_profile { + network_plugin = "azure" + load_balancer_sku = "standard" + } + +} diff --git a/quickstart/301-aks-private-cluster/dns-zone-link.sh b/quickstart/301-aks-private-cluster/dns-zone-link.sh new file mode 100755 index 00000000..8577f0e5 --- /dev/null +++ b/quickstart/301-aks-private-cluster/dns-zone-link.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# Query AKS private DNS zone and then link it to custom DNS VNet. +set -e + +wait-for-node-resource-group() { + NODE_RESOURCE_GROUP=$(az aks show -g ${AKS_RESOURCE_GROUP} -n ${AKS_CLUSTER_NAME} -o json 2>/dev/null | jq -r '.nodeResourceGroup') + while [ "${NODE_RESOURCE_GROUP}" == "" ]; do + echo "Waiting for node resource group..." + sleep 5 + NODE_RESOURCE_GROUP=$(az aks show -g ${AKS_RESOURCE_GROUP} -n ${AKS_CLUSTER_NAME} -o json 2>/dev/null | jq -r '.nodeResourceGroup') + done +} + +wait-for-private-dns-zone() { + length=$(az network private-dns zone list -g ${NODE_RESOURCE_GROUP} -o json 2>/dev/null | jq '. | length') + while [[ $length -eq 0 ]]; do + echo "Waiting for private DNS zone..." + sleep 5 + length=$(az network private-dns zone list -g ${NODE_RESOURCE_GROUP} -o json 2>/dev/null | jq '. | length') + done +} + +wait-for-node-resource-group +NODE_RESOURCE_GROUP=$(az aks show -g ${AKS_RESOURCE_GROUP} -n ${AKS_CLUSTER_NAME} -o json 2>/dev/null | jq -r '.nodeResourceGroup') +wait-for-private-dns-zone +ZONE_NAME=$(az network private-dns zone list -g ${NODE_RESOURCE_GROUP} -o json | jq -r '.[0].name') +echo "Get the private DNS Zone ${ZONE_NAME}" +az network private-dns link vnet create -o json \ + -g ${NODE_RESOURCE_GROUP} \ + -n private-dns-link \ + --registration-enabled false \ + --zone-name ${ZONE_NAME} \ + --virtual-network ${DNS_VNET} + +echo "Private DNS Zone ${ZONE_NAME} has been linked to VNet ${DNS_VNET}." diff --git a/quickstart/301-aks-private-cluster/dns-zone.tf b/quickstart/301-aks-private-cluster/dns-zone.tf new file mode 100644 index 00000000..6fd0c2da --- /dev/null +++ b/quickstart/301-aks-private-cluster/dns-zone.tf @@ -0,0 +1,13 @@ +resource "null_resource" "dns_zone_link" { + + provisioner "local-exec" { + interpreter = ["bash"] + command = "dns-zone-link.sh" + + environment = { + DNS_VNET = "${var.custom_dns_vnet_id}" + AKS_RESOURCE_GROUP="${var.resource_group_name}" + AKS_CLUSTER_NAME="${var.cluster_name}" + } + } +} diff --git a/quickstart/301-aks-private-cluster/main.tf b/quickstart/301-aks-private-cluster/main.tf new file mode 100644 index 00000000..dfcd1220 --- /dev/null +++ b/quickstart/301-aks-private-cluster/main.tf @@ -0,0 +1,4 @@ +provider "azurerm" { + version = "~>2.11" + features {} +} diff --git a/quickstart/301-aks-private-cluster/output.tf b/quickstart/301-aks-private-cluster/output.tf new file mode 100644 index 00000000..e69f29e0 --- /dev/null +++ b/quickstart/301-aks-private-cluster/output.tf @@ -0,0 +1,7 @@ +output "kube_config" { + value = "${azurerm_kubernetes_cluster.k8s.kube_config_raw}" +} + +output "host" { + value = "${azurerm_kubernetes_cluster.k8s.kube_config.0.host}" +} diff --git a/quickstart/301-aks-private-cluster/variables.tf b/quickstart/301-aks-private-cluster/variables.tf new file mode 100644 index 00000000..b1a128fa --- /dev/null +++ b/quickstart/301-aks-private-cluster/variables.tf @@ -0,0 +1,65 @@ +variable resource_group_name { + type = string + description = "Name of the Azure resource group" + default = "aks-quickstart" +} + +variable cluster_name { + type = string + description = "Name of the AKS cluster" + default = "demo-private" +} + +variable custom_dns { + type = string + description = "IP of custom DNS server" + default = "168.63.129.16" +} + +variable custom_dns_vnet_id { + type = string + description = "Resource ID of the Azure VNet that holds custom DNS server" + default = "" +} + +variable "client_id" { + type = string + description = "The service principal ID" + default = "" +} + +variable "client_secret" { + type = string + description = "The service principal password" + default = "" +} + +variable "agent_count" { + type = string + description = "The number of K8S nodes to provision" + default = 3 +} + +variable "kubernetes_version" { + type = string + description = "The version of K8S to provision" + default = "1.17.9" +} + +variable "ssh_public_key" { + type = string + description = "The SSH public key of K8S nodes" + default = "~/.ssh/id_rsa.pub" +} + +variable "dns_prefix" { + type = string + description = "The DNS prefix" + default = "aks" +} + +variable location { + type = string + description = "The location" + default = "East US" +}