update aks enterprise
This commit is contained in:
Joey Lorich
parent
415e3caa97
commit
5f390fa368
@ -1,4 +1,4 @@
|
||||
# Azure Kubernetes Service
|
||||
# AKS with Log Analytics
|
||||
|
||||
|
||||
This template deploys an [Azure Kubernetes Service](https://www.terraform.io/docs/providers/azurerm/r/kubernetes_cluster.html) instance which sends system and container logs to Azure Log Analytics, which can be visualized with the Container Monitoring solution.
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
# Azure Kubernetes Service
|
||||
# AKS with an Admin Dashboard
|
||||
|
||||
|
||||
This template deploys an [Azure Kubernetes Service](https://www.terraform.io/docs/providers/azurerm/r/kubernetes_cluster.html) instance with Role Based Access Control (RBAC) enabled. With this, by default the robust Kubernetes dashboard has no rights to view or make changes to the cluster. In this template we leverage the Kubernetes provider to provision a role binding for the Dashboard accoutn to give it `cluster-admin` rights - something we shoudl not do in production but can be very useful in development.
|
||||
|
||||
36
quickstart/301-aks-enterprise/aks.tf
Normal file
36
quickstart/301-aks-enterprise/aks.tf
Normal file
@ -0,0 +1,36 @@
|
||||
resource "azurerm_kubernetes_cluster" "default" {
|
||||
name = "${var.name}-aks"
|
||||
location = "${azurerm_resource_group.default.location}"
|
||||
resource_group_name = "${azurerm_resource_group.default.name}"
|
||||
dns_prefix = "${var.dns_prefix}-${var.name}-aks-${var.environment}"
|
||||
depends_on = ["azurerm_role_assignment.default"]
|
||||
|
||||
agent_pool_profile {
|
||||
name = "default"
|
||||
count = "${var.node_count}"
|
||||
vm_size = "${var.node_type}"
|
||||
os_type = "${var.node_os}"
|
||||
os_disk_size_gb = 30
|
||||
vnet_subnet_id = "${azurerm_subnet.aks.id}"
|
||||
}
|
||||
|
||||
service_principal {
|
||||
client_id = "${azuread_application.default.application_id}"
|
||||
client_secret = "${azuread_service_principal_password.default.value}"
|
||||
}
|
||||
|
||||
role_based_access_control {
|
||||
enabled = true
|
||||
}
|
||||
|
||||
network_profile {
|
||||
network_plugin = "azure"
|
||||
}
|
||||
|
||||
addon_profile {
|
||||
oms_agent {
|
||||
enabled = true
|
||||
log_analytics_workspace_id = "${azurerm_log_analytics_workspace.default.id}"
|
||||
}
|
||||
}
|
||||
}
|
||||
24
quickstart/301-aks-enterprise/azuread.tf
Normal file
24
quickstart/301-aks-enterprise/azuread.tf
Normal file
@ -0,0 +1,24 @@
|
||||
resource "azuread_application" "default" {
|
||||
name = "${var.name}-${var.environment}"
|
||||
}
|
||||
|
||||
resource "azuread_service_principal" "default" {
|
||||
application_id = "${azuread_application.default.application_id}"
|
||||
}
|
||||
|
||||
resource "random_string" "password" {
|
||||
length = 32
|
||||
special = true
|
||||
}
|
||||
|
||||
resource "azuread_service_principal_password" "default" {
|
||||
service_principal_id = "${azuread_service_principal.default.id}"
|
||||
value = "${random_string.password.result}"
|
||||
end_date = "2099-01-01T01:00:00Z"
|
||||
}
|
||||
|
||||
resource "azurerm_role_assignment" "default" {
|
||||
scope = "${data.azurerm_subscription.current.id}/resourceGroups/${azurerm_resource_group.default.name}"
|
||||
role_definition_name = "Network Contributor"
|
||||
principal_id = "${azuread_service_principal.default.id}"
|
||||
}
|
||||
39
quickstart/301-aks-enterprise/helm.tf
Normal file
39
quickstart/301-aks-enterprise/helm.tf
Normal file
@ -0,0 +1,39 @@
|
||||
# Define the helm provider to use the AKS cluster
|
||||
provider "helm" {
|
||||
kubernetes {
|
||||
host = "${azurerm_kubernetes_cluster.default.kube_config.0.host}"
|
||||
|
||||
client_certificate = "${base64decode(azurerm_kubernetes_cluster.default.kube_config.0.client_certificate)}"
|
||||
client_key = "${base64decode(azurerm_kubernetes_cluster.default.kube_config.0.client_key)}"
|
||||
cluster_ca_certificate = "${base64decode(azurerm_kubernetes_cluster.default.kube_config.0.cluster_ca_certificate)}"
|
||||
}
|
||||
|
||||
service_account = "tiller"
|
||||
}
|
||||
|
||||
# Install a load-balanced nginx-ingress controller onto the cluster
|
||||
resource "helm_release" "ingress" {
|
||||
name = "nginx-ingress"
|
||||
chart = "stable/nginx-ingress"
|
||||
namespace = "kube-system"
|
||||
|
||||
values = [<<EOF
|
||||
controller:
|
||||
replicaCount: 2
|
||||
service:
|
||||
loadBalancerIP: ${var.ingress_load_balancer_ip}
|
||||
annotations:
|
||||
service.beta.kubernetes.io/azure-load-balancer-internal: "true"
|
||||
service.beta.kubernetes.io/azure-load-balancer-internal-subnet: "${azurerm_subnet.ingress.name}"
|
||||
EOF
|
||||
]
|
||||
|
||||
depends_on = ["kubernetes_cluster_role_binding.tiller"]
|
||||
}
|
||||
|
||||
resource "helm_release" "ghost" {
|
||||
name = "ghost-blog"
|
||||
chart = "bitnami/ghost"
|
||||
|
||||
depends_on = ["kubernetes_cluster_role_binding.tiller"]
|
||||
}
|
||||
55
quickstart/301-aks-enterprise/kubernetes.tf
Normal file
55
quickstart/301-aks-enterprise/kubernetes.tf
Normal file
@ -0,0 +1,55 @@
|
||||
# Define Kubernetes provider to use the AKS cluster
|
||||
provider "kubernetes" {
|
||||
host = "${azurerm_kubernetes_cluster.default.kube_config.0.host}"
|
||||
|
||||
client_certificate = "${base64decode(azurerm_kubernetes_cluster.default.kube_config.0.client_certificate)}"
|
||||
client_key = "${base64decode(azurerm_kubernetes_cluster.default.kube_config.0.client_key)}"
|
||||
cluster_ca_certificate = "${base64decode(azurerm_kubernetes_cluster.default.kube_config.0.cluster_ca_certificate)}"
|
||||
}
|
||||
|
||||
# Create a service account for the Helm Tiller
|
||||
resource "kubernetes_service_account" "tiller" {
|
||||
metadata {
|
||||
name = "tiller"
|
||||
namespace = "kube-system"
|
||||
}
|
||||
}
|
||||
|
||||
# Grant cluster-admin rights to the Tiller Service Account
|
||||
resource "kubernetes_cluster_role_binding" "tiller" {
|
||||
metadata {
|
||||
name = "${kubernetes_service_account.tiller.metadata.0.name}"
|
||||
}
|
||||
|
||||
role_ref {
|
||||
api_group = "rbac.authorization.k8s.io"
|
||||
kind = "ClusterRole"
|
||||
name = "cluster-admin"
|
||||
}
|
||||
|
||||
subject {
|
||||
kind = "ServiceAccount"
|
||||
name = "${kubernetes_service_account.tiller.metadata.0.name}"
|
||||
namespace = "kube-system"
|
||||
}
|
||||
}
|
||||
|
||||
# Grant cluster-admin rights to the default service account
|
||||
# This is a terrible idea in general, but a feature of the game is killing other pods
|
||||
resource "kubernetes_cluster_role_binding" "default" {
|
||||
metadata {
|
||||
name = "default"
|
||||
}
|
||||
|
||||
role_ref {
|
||||
api_group = "rbac.authorization.k8s.io"
|
||||
kind = "ClusterRole"
|
||||
name = "cluster-admin"
|
||||
}
|
||||
|
||||
subject {
|
||||
kind = "ServiceAccount"
|
||||
name = "default"
|
||||
namespace = "default"
|
||||
}
|
||||
}
|
||||
6
quickstart/301-aks-enterprise/main.tf
Normal file
6
quickstart/301-aks-enterprise/main.tf
Normal file
@ -0,0 +1,6 @@
|
||||
resource "azurerm_resource_group" "default" {
|
||||
name = "${var.name}-${var.environment}-rg"
|
||||
location = "${var.location}"
|
||||
}
|
||||
|
||||
data "azurerm_subscription" "current" {}
|
||||
27
quickstart/301-aks-enterprise/monitoring.tf
Normal file
27
quickstart/301-aks-enterprise/monitoring.tf
Normal file
@ -0,0 +1,27 @@
|
||||
resource "azurerm_application_insights" "default" {
|
||||
name = "${var.name}-${var.environment}-ai"
|
||||
location = "${azurerm_resource_group.default.location}"
|
||||
resource_group_name = "${azurerm_resource_group.default.name}"
|
||||
application_type = "Web"
|
||||
}
|
||||
|
||||
resource "azurerm_log_analytics_workspace" "default" {
|
||||
name = "${var.name}-${var.environment}-law"
|
||||
location = "${azurerm_resource_group.default.location}"
|
||||
resource_group_name = "${azurerm_resource_group.default.name}"
|
||||
sku = "PerGB2018"
|
||||
retention_in_days = 30
|
||||
}
|
||||
|
||||
resource "azurerm_log_analytics_solution" "default" {
|
||||
solution_name = "ContainerInsights"
|
||||
location = "${azurerm_log_analytics_workspace.default.location}"
|
||||
resource_group_name = "${azurerm_resource_group.default.name}"
|
||||
workspace_resource_id = "${azurerm_log_analytics_workspace.default.id}"
|
||||
workspace_name = "${azurerm_log_analytics_workspace.default.name}"
|
||||
|
||||
plan {
|
||||
publisher = "Microsoft"
|
||||
product = "OMSGallery/ContainerInsights"
|
||||
}
|
||||
}
|
||||
176
quickstart/301-aks-enterprise/networking.tf
Normal file
176
quickstart/301-aks-enterprise/networking.tf
Normal file
@ -0,0 +1,176 @@
|
||||
# Virtual Network to deploy resources into
|
||||
resource "azurerm_virtual_network" "default" {
|
||||
name = "${var.name}-vnet"
|
||||
location = "${azurerm_resource_group.default.location}"
|
||||
resource_group_name = "${azurerm_resource_group.default.name}"
|
||||
address_space = ["${var.vnet_address_space}"]
|
||||
}
|
||||
|
||||
# Subnets
|
||||
resource "azurerm_subnet" "aks" {
|
||||
name = "${var.name}-aks-subnet"
|
||||
resource_group_name = "${azurerm_resource_group.default.name}"
|
||||
address_prefix = "${var.vnet_aks_subnet_space}"
|
||||
virtual_network_name = "${azurerm_virtual_network.default.name}"
|
||||
}
|
||||
|
||||
resource "azurerm_subnet" "ingress" {
|
||||
name = "${var.name}-ingress-subnet"
|
||||
resource_group_name = "${azurerm_resource_group.default.name}"
|
||||
virtual_network_name = "${azurerm_virtual_network.default.name}"
|
||||
address_prefix = "${var.vnet_ingress_subnet_space}"
|
||||
}
|
||||
|
||||
resource "azurerm_subnet" "gateway" {
|
||||
name = "${var.name}-gateway-subnet"
|
||||
resource_group_name = "${azurerm_resource_group.default.name}"
|
||||
virtual_network_name = "${azurerm_virtual_network.default.name}"
|
||||
address_prefix = "${var.vnet_gateway_subnet_space}"
|
||||
}
|
||||
|
||||
# Network security groups
|
||||
resource azurerm_network_security_group "aks" {
|
||||
name = "${var.name}-aks-nsg"
|
||||
location = "${azurerm_resource_group.default.location}"
|
||||
resource_group_name = "${azurerm_resource_group.default.name}"
|
||||
}
|
||||
|
||||
resource azurerm_network_security_group "ingress" {
|
||||
name = "${var.name}-ingress-nsg"
|
||||
location = "${azurerm_resource_group.default.location}"
|
||||
resource_group_name = "${azurerm_resource_group.default.name}"
|
||||
}
|
||||
|
||||
resource azurerm_network_security_group "gateway" {
|
||||
name = "${var.name}-gateway-nsg"
|
||||
location = "${azurerm_resource_group.default.location}"
|
||||
resource_group_name = "${azurerm_resource_group.default.name}"
|
||||
}
|
||||
|
||||
# Network security group associations
|
||||
resource "azurerm_subnet_network_security_group_association" "aks" {
|
||||
subnet_id = "${azurerm_subnet.aks.id}"
|
||||
network_security_group_id = "${azurerm_network_security_group.aks.id}"
|
||||
}
|
||||
|
||||
resource "azurerm_subnet_network_security_group_association" "ingress" {
|
||||
subnet_id = "${azurerm_subnet.ingress.id}"
|
||||
network_security_group_id = "${azurerm_network_security_group.ingress.id}"
|
||||
}
|
||||
|
||||
resource "azurerm_subnet_network_security_group_association" "gateway" {
|
||||
subnet_id = "${azurerm_subnet.gateway.id}"
|
||||
network_security_group_id = "${azurerm_network_security_group.gateway.id}"
|
||||
}
|
||||
|
||||
|
||||
locals {
|
||||
|
||||
gateway_name = "${var.dns_prefix}-${var.name}-${var.environment}-gateway"
|
||||
gateway_ip_name = "${var.dns_prefix}-${var.name}-${var.environment}-gateway-ip"
|
||||
gateway_ip_config_name = "${var.name}-gateway-ipconfig"
|
||||
frontend_port_name = "${var.name}-gateway-feport"
|
||||
frontend_ip_configuration_name = "${var.name}-gateway-feip"
|
||||
backend_address_pool_name = "${var.name}-gateway-bepool"
|
||||
http_setting_name = "${var.name}-gateway-http"
|
||||
probe_name = "${var.name}-gateway-probe"
|
||||
listener_name = "${var.name}-gateway-lstn"
|
||||
ssl_name = "${var.name}-gateway-ssl"
|
||||
url_path_map_name = "${var.name}-gateway-urlpath"
|
||||
url_path_map_rule_name = "${var.name}-gateway-urlrule"
|
||||
request_routing_rule_name = "${var.name}-gateway-router"
|
||||
}
|
||||
|
||||
resource "azurerm_public_ip" "gateway" {
|
||||
name = "${local.gateway_ip_name}"
|
||||
resource_group_name = "${azurerm_resource_group.default.name}"
|
||||
location = "${azurerm_resource_group.default.location}"
|
||||
domain_name_label = "${local.gateway_name}"
|
||||
allocation_method = "Static"
|
||||
sku = "Standard"
|
||||
}
|
||||
|
||||
resource "azurerm_application_gateway" "gateway" {
|
||||
name = "${local.gateway_name}"
|
||||
resource_group_name = "${azurerm_resource_group.default.name}"
|
||||
location = "${azurerm_resource_group.default.location}"
|
||||
|
||||
sku {
|
||||
name = "WAF_v2"
|
||||
tier = "WAF_v2"
|
||||
capacity = "${var.gateway_instance_count}"
|
||||
}
|
||||
|
||||
gateway_ip_configuration {
|
||||
name = "${local.gateway_ip_config_name}"
|
||||
subnet_id = "${azurerm_subnet.gateway.id}"
|
||||
}
|
||||
|
||||
frontend_port {
|
||||
name = "${local.frontend_port_name}-http"
|
||||
port = 80
|
||||
}
|
||||
|
||||
frontend_port {
|
||||
name = "${local.frontend_port_name}-https"
|
||||
port = 443
|
||||
}
|
||||
|
||||
frontend_ip_configuration {
|
||||
name = "${local.frontend_ip_configuration_name}"
|
||||
public_ip_address_id = "${azurerm_public_ip.gateway.id}"
|
||||
}
|
||||
|
||||
backend_address_pool {
|
||||
name = "${local.backend_address_pool_name}"
|
||||
ip_addresses = ["${var.ingress_load_balancer_ip}"]
|
||||
}
|
||||
|
||||
backend_http_settings {
|
||||
name = "${local.http_setting_name}"
|
||||
cookie_based_affinity = "Disabled"
|
||||
port = 80
|
||||
protocol = "http"
|
||||
request_timeout = 1
|
||||
probe_name = "${local.probe_name}"
|
||||
}
|
||||
|
||||
http_listener {
|
||||
name = "${local.listener_name}-http"
|
||||
frontend_ip_configuration_name = "${local.frontend_ip_configuration_name}"
|
||||
frontend_port_name = "${local.frontend_port_name}-http"
|
||||
protocol = "http"
|
||||
}
|
||||
|
||||
probe {
|
||||
name = "${local.probe_name}"
|
||||
protocol = "http"
|
||||
path = "/nginx-health"
|
||||
interval = 30
|
||||
timeout = 30
|
||||
unhealthy_threshold = 3
|
||||
host = "${var.ingress_load_balancer_ip}"
|
||||
}
|
||||
|
||||
request_routing_rule {
|
||||
name = "${local.request_routing_rule_name}-http"
|
||||
rule_type = "PathBasedRouting"
|
||||
http_listener_name = "${local.listener_name}-http"
|
||||
url_path_map_name = "${local.url_path_map_name}"
|
||||
}
|
||||
|
||||
url_path_map {
|
||||
name = "${local.url_path_map_name}"
|
||||
default_backend_address_pool_name = "${local.backend_address_pool_name}"
|
||||
default_backend_http_settings_name = "${local.http_setting_name}"
|
||||
|
||||
path_rule {
|
||||
name = "${local.url_path_map_rule_name}"
|
||||
backend_address_pool_name = "${local.backend_address_pool_name}"
|
||||
backend_http_settings_name = "${local.http_setting_name}"
|
||||
paths = [
|
||||
"/*"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -18,23 +18,634 @@ This template deploys an Azure Kubernetes Service cluster configured for common
|
||||
| `environment` | The depolyment environment name (used for postfixing resource names) |
|
||||
| `prefix` | A prefix for globally-unique dns-based resources |
|
||||
| `location` | The Azure Region to deploy these resources in |
|
||||
| `plan_tier` | The App Service Plan tier to deploy |
|
||||
| `plan_sku` | The App Service Plan SKU to deploy|
|
||||
| `azuread_application` | The Azure AD Application for the cluster |
|
||||
| `azuread_service_principal` | The service principal for the AAD App |
|
||||
| `azuread_service_principal_password` | The password for the AAD App SP |
|
||||
| `azurerm_application_gateway` | The App Gateway that will front all traffic |
|
||||
| `azurerm_application_insights` | An App Insights instance to collect application data |
|
||||
| `azurerm_kubernetes_cluster` | The AKS Cluster |
|
||||
| `azurerm_log_analytics_solution` | Enables the Log Analytics container monitoring solution |
|
||||
| `azurerm_log_analytics_workspace` | A Log Analytics workspace to send all AKS telemetry to |
|
||||
| `azurerm_network_security_group` | A network security group for AKS pods |
|
||||
| `azurerm_network_security_group` | A network security group for our App Gateway |
|
||||
| `azurerm_network_security_group` | A network security group for ingress services |
|
||||
| `azurerm_public_ip` | A public IP for the App Gateway |
|
||||
| `azurerm_role_assignment` | Assigns rights for AKS to be able to modify the networks |
|
||||
| `azurerm_subnet` | A subnet for AKS Pods |
|
||||
| `azurerm_subnet` | A subnet for our App Gateway |
|
||||
| `azurerm_subnet` | A subnet for ingress services |
|
||||
| `azurerm_virtual_network` | A Virtual network for our cluster |
|
||||
| `helm_release` | Installs Ghost blogging software on the AKS cluster as a demo |
|
||||
| `helm_release` | Installs nginx ingress on the AKS cluster |
|
||||
| `kubernetes_cluster_role_binding` | Permissions for the Helm Timmer accoutn |
|
||||
| `kubernetes_service_account` | A Service account for Helm |
|
||||
|
||||
|
||||
## Usage
|
||||
|
||||
```bash
|
||||
terraform plan \
|
||||
-var 'name=demo-docker' \
|
||||
-var 'environment=staging' \
|
||||
-var 'location=West US 2'
|
||||
-var 'prefix=tfquickstard' \
|
||||
-var 'plan_tier=standard' \
|
||||
-var 'plan_sku=s1' \
|
||||
-out demo.tfplan
|
||||
>terraform plan
|
||||
Refreshing Terraform state in-memory prior to plan...
|
||||
The refreshed state will be used to calculate this plan, but will not be
|
||||
persisted to local or remote state storage.
|
||||
|
||||
terraform apply demo.tfplan
|
||||
data.azurerm_subscription.current: Refreshing state...
|
||||
|
||||
------------------------------------------------------------------------
|
||||
|
||||
An execution plan has been generated and is shown below.
|
||||
Resource actions are indicated with the following symbols:
|
||||
+ create
|
||||
|
||||
Terraform will perform the following actions:
|
||||
|
||||
# azuread_application.default will be created
|
||||
+ resource "azuread_application" "default" {
|
||||
+ application_id = (known after apply)
|
||||
+ homepage = (known after apply)
|
||||
+ id = (known after apply)
|
||||
+ identifier_uris = (known after apply)
|
||||
+ name = "demo-tfquickstart-dev"
|
||||
+ object_id = (known after apply)
|
||||
+ public_client = (known after apply)
|
||||
+ reply_urls = (known after apply)
|
||||
+ type = "webapp/api"
|
||||
|
||||
+ oauth2_permissions {
|
||||
+ admin_consent_description = (known after apply)
|
||||
+ admin_consent_display_name = (known after apply)
|
||||
+ id = (known after apply)
|
||||
+ is_enabled = (known after apply)
|
||||
+ type = (known after apply)
|
||||
+ user_consent_description = (known after apply)
|
||||
+ user_consent_display_name = (known after apply)
|
||||
+ value = (known after apply)
|
||||
}
|
||||
}
|
||||
|
||||
# azuread_service_principal.default will be created
|
||||
+ resource "azuread_service_principal" "default" {
|
||||
+ application_id = (known after apply)
|
||||
+ display_name = (known after apply)
|
||||
+ id = (known after apply)
|
||||
+ object_id = (known after apply)
|
||||
|
||||
+ oauth2_permissions {
|
||||
+ admin_consent_description = (known after apply)
|
||||
+ admin_consent_display_name = (known after apply)
|
||||
+ id = (known after apply)
|
||||
+ is_enabled = (known after apply)
|
||||
+ type = (known after apply)
|
||||
+ user_consent_description = (known after apply)
|
||||
+ user_consent_display_name = (known after apply)
|
||||
+ value = (known after apply)
|
||||
}
|
||||
}
|
||||
|
||||
# azuread_service_principal_password.default will be created
|
||||
+ resource "azuread_service_principal_password" "default" {
|
||||
+ end_date = "2099-01-01T01:00:00Z"
|
||||
+ id = (known after apply)
|
||||
+ key_id = (known after apply)
|
||||
+ service_principal_id = (known after apply)
|
||||
+ start_date = (known after apply)
|
||||
+ value = (sensitive value)
|
||||
}
|
||||
|
||||
# azurerm_application_gateway.gateway will be created
|
||||
+ resource "azurerm_application_gateway" "gateway" {
|
||||
+ disabled_ssl_protocols = (known after apply)
|
||||
+ id = (known after apply)
|
||||
+ location = "westus2"
|
||||
+ name = "mtcden-demo-tfquickstart-dev-gateway"
|
||||
+ resource_group_name = "demo-tfquickstart-dev-rg"
|
||||
+ tags = (known after apply)
|
||||
|
||||
+ backend_address_pool {
|
||||
+ fqdn_list = (known after apply)
|
||||
+ fqdns = (known after apply)
|
||||
+ id = (known after apply)
|
||||
+ ip_address_list = (known after apply)
|
||||
+ ip_addresses = [
|
||||
+ "10.2.0.10",
|
||||
]
|
||||
+ name = "demo-tfquickstart-gateway-bepool"
|
||||
}
|
||||
|
||||
+ backend_http_settings {
|
||||
+ cookie_based_affinity = "Disabled"
|
||||
+ id = (known after apply)
|
||||
+ name = "demo-tfquickstart-gateway-http"
|
||||
+ pick_host_name_from_backend_address = false
|
||||
+ port = 80
|
||||
+ probe_id = (known after apply)
|
||||
+ probe_name = "demo-tfquickstart-gateway-probe"
|
||||
+ protocol = "http"
|
||||
+ request_timeout = 1
|
||||
}
|
||||
|
||||
+ frontend_ip_configuration {
|
||||
+ id = (known after apply)
|
||||
+ name = "demo-tfquickstart-gateway-feip"
|
||||
+ private_ip_address = (known after apply)
|
||||
+ private_ip_address_allocation = (known after apply)
|
||||
+ public_ip_address_id = (known after apply)
|
||||
+ subnet_id = (known after apply)
|
||||
}
|
||||
|
||||
+ frontend_port {
|
||||
+ id = (known after apply)
|
||||
+ name = "demo-tfquickstart-gateway-feport-http"
|
||||
+ port = 80
|
||||
}
|
||||
+ frontend_port {
|
||||
+ id = (known after apply)
|
||||
+ name = "demo-tfquickstart-gateway-feport-https"
|
||||
+ port = 443
|
||||
}
|
||||
|
||||
+ gateway_ip_configuration {
|
||||
+ id = (known after apply)
|
||||
+ name = "demo-tfquickstart-gateway-ipconfig"
|
||||
+ subnet_id = (known after apply)
|
||||
}
|
||||
|
||||
+ http_listener {
|
||||
+ frontend_ip_configuration_id = (known after apply)
|
||||
+ frontend_ip_configuration_name = "demo-tfquickstart-gateway-feip"
|
||||
+ frontend_port_id = (known after apply)
|
||||
+ frontend_port_name = "demo-tfquickstart-gateway-feport-http"
|
||||
+ id = (known after apply)
|
||||
+ name = "demo-tfquickstart-gateway-lstn-http"
|
||||
+ protocol = "http"
|
||||
+ ssl_certificate_id = (known after apply)
|
||||
}
|
||||
|
||||
+ identity {
|
||||
+ identity_ids = (known after apply)
|
||||
+ type = (known after apply)
|
||||
}
|
||||
|
||||
+ probe {
|
||||
+ host = "10.2.0.10"
|
||||
+ id = (known after apply)
|
||||
+ interval = 30
|
||||
+ minimum_servers = 0
|
||||
+ name = "demo-tfquickstart-gateway-probe"
|
||||
+ path = "/nginx-health"
|
||||
+ pick_host_name_from_backend_http_settings = false
|
||||
+ protocol = "http"
|
||||
+ timeout = 30
|
||||
+ unhealthy_threshold = 3
|
||||
|
||||
+ match {
|
||||
+ body = (known after apply)
|
||||
+ status_code = (known after apply)
|
||||
}
|
||||
}
|
||||
|
||||
+ request_routing_rule {
|
||||
+ backend_address_pool_id = (known after apply)
|
||||
+ backend_http_settings_id = (known after apply)
|
||||
+ http_listener_id = (known after apply)
|
||||
+ http_listener_name = "demo-tfquickstart-gateway-lstn-http"
|
||||
+ id = (known after apply)
|
||||
+ name = "demo-tfquickstart-gateway-router-http"
|
||||
+ redirect_configuration_id = (known after apply)
|
||||
+ rewrite_rule_set_id = (known after apply)
|
||||
+ rule_type = "PathBasedRouting"
|
||||
+ url_path_map_id = (known after apply)
|
||||
+ url_path_map_name = "demo-tfquickstart-gateway-urlpath"
|
||||
}
|
||||
|
||||
+ sku {
|
||||
+ capacity = 1
|
||||
+ name = "WAF_v2"
|
||||
+ tier = "WAF_v2"
|
||||
}
|
||||
|
||||
+ ssl_policy {
|
||||
+ cipher_suites = (known after apply)
|
||||
+ disabled_protocols = (known after apply)
|
||||
+ min_protocol_version = (known after apply)
|
||||
+ policy_name = (known after apply)
|
||||
+ policy_type = (known after apply)
|
||||
}
|
||||
|
||||
+ url_path_map {
|
||||
+ default_backend_address_pool_id = (known after apply)
|
||||
+ default_backend_address_pool_name = "demo-tfquickstart-gateway-bepool"
|
||||
+ default_backend_http_settings_id = (known after apply)
|
||||
+ default_backend_http_settings_name = "demo-tfquickstart-gateway-http"
|
||||
+ default_redirect_configuration_id = (known after apply)
|
||||
+ default_rewrite_rule_set_id = (known after apply)
|
||||
+ id = (known after apply)
|
||||
+ name = "demo-tfquickstart-gateway-urlpath"
|
||||
|
||||
+ path_rule {
|
||||
+ backend_address_pool_id = (known after apply)
|
||||
+ backend_address_pool_name = "demo-tfquickstart-gateway-bepool"
|
||||
+ backend_http_settings_id = (known after apply)
|
||||
+ backend_http_settings_name = "demo-tfquickstart-gateway-http"
|
||||
+ id = (known after apply)
|
||||
+ name = "demo-tfquickstart-gateway-urlrule"
|
||||
+ paths = [
|
||||
+ "/*",
|
||||
]
|
||||
+ redirect_configuration_id = (known after apply)
|
||||
+ rewrite_rule_set_id = (known after apply)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# azurerm_application_insights.default will be created
|
||||
+ resource "azurerm_application_insights" "default" {
|
||||
+ app_id = (known after apply)
|
||||
+ application_type = "Web"
|
||||
+ id = (known after apply)
|
||||
+ instrumentation_key = (sensitive value)
|
||||
+ location = "westus2"
|
||||
+ name = "demo-tfquickstart-dev-ai"
|
||||
+ resource_group_name = "demo-tfquickstart-dev-rg"
|
||||
+ tags = (known after apply)
|
||||
}
|
||||
|
||||
# azurerm_kubernetes_cluster.default will be created
|
||||
+ resource "azurerm_kubernetes_cluster" "default" {
|
||||
+ dns_prefix = "mtcden-demo-tfquickstart-aks-dev"
|
||||
+ enable_pod_security_policy = (known after apply)
|
||||
+ fqdn = (known after apply)
|
||||
+ id = (known after apply)
|
||||
+ kube_admin_config = (known after apply)
|
||||
+ kube_admin_config_raw = (sensitive value)
|
||||
+ kube_config = (known after apply)
|
||||
+ kube_config_raw = (sensitive value)
|
||||
+ kubernetes_version = (known after apply)
|
||||
+ location = "westus2"
|
||||
+ name = "demo-tfquickstart-aks"
|
||||
+ node_resource_group = (known after apply)
|
||||
+ resource_group_name = "demo-tfquickstart-dev-rg"
|
||||
+ tags = (known after apply)
|
||||
|
||||
+ addon_profile {
|
||||
|
||||
+ oms_agent {
|
||||
+ enabled = true
|
||||
+ log_analytics_workspace_id = (known after apply)
|
||||
}
|
||||
}
|
||||
|
||||
+ agent_pool_profile {
|
||||
+ count = 3
|
||||
+ dns_prefix = (known after apply)
|
||||
+ fqdn = (known after apply)
|
||||
+ max_pods = (known after apply)
|
||||
+ name = "default"
|
||||
+ os_disk_size_gb = 30
|
||||
+ os_type = "Linux"
|
||||
+ type = "AvailabilitySet"
|
||||
+ vm_size = "Standard_D1_v2"
|
||||
+ vnet_subnet_id = (known after apply)
|
||||
}
|
||||
|
||||
+ network_profile {
|
||||
+ dns_service_ip = (known after apply)
|
||||
+ docker_bridge_cidr = (known after apply)
|
||||
+ load_balancer_sku = "basic"
|
||||
+ network_plugin = "azure"
|
||||
+ network_policy = (known after apply)
|
||||
+ pod_cidr = (known after apply)
|
||||
+ service_cidr = (known after apply)
|
||||
}
|
||||
|
||||
+ role_based_access_control {
|
||||
+ enabled = true
|
||||
}
|
||||
|
||||
+ service_principal {
|
||||
+ client_id = (known after apply)
|
||||
+ client_secret = (sensitive value)
|
||||
}
|
||||
}
|
||||
|
||||
# azurerm_log_analytics_solution.default will be created
|
||||
+ resource "azurerm_log_analytics_solution" "default" {
|
||||
+ id = (known after apply)
|
||||
+ location = "westus2"
|
||||
+ resource_group_name = "demo-tfquickstart-dev-rg"
|
||||
+ solution_name = "ContainerInsights"
|
||||
+ workspace_name = "demo-tfquickstart-dev-law"
|
||||
+ workspace_resource_id = (known after apply)
|
||||
|
||||
+ plan {
|
||||
+ name = (known after apply)
|
||||
+ product = "OMSGallery/ContainerInsights"
|
||||
+ publisher = "Microsoft"
|
||||
}
|
||||
}
|
||||
|
||||
# azurerm_log_analytics_workspace.default will be created
|
||||
+ resource "azurerm_log_analytics_workspace" "default" {
|
||||
+ id = (known after apply)
|
||||
+ location = "westus2"
|
||||
+ name = "demo-tfquickstart-dev-law"
|
||||
+ portal_url = (known after apply)
|
||||
+ primary_shared_key = (sensitive value)
|
||||
+ resource_group_name = "demo-tfquickstart-dev-rg"
|
||||
+ retention_in_days = 30
|
||||
+ secondary_shared_key = (sensitive value)
|
||||
+ sku = "PerGB2018"
|
||||
+ tags = (known after apply)
|
||||
+ workspace_id = (known after apply)
|
||||
}
|
||||
|
||||
# azurerm_network_security_group.aks will be created
|
||||
+ resource "azurerm_network_security_group" "aks" {
|
||||
+ id = (known after apply)
|
||||
+ location = "westus2"
|
||||
+ name = "demo-tfquickstart-aks-nsg"
|
||||
+ resource_group_name = "demo-tfquickstart-dev-rg"
|
||||
+ security_rule = (known after apply)
|
||||
+ tags = (known after apply)
|
||||
}
|
||||
|
||||
# azurerm_network_security_group.gateway will be created
|
||||
+ resource "azurerm_network_security_group" "gateway" {
|
||||
+ id = (known after apply)
|
||||
+ location = "westus2"
|
||||
+ name = "demo-tfquickstart-gateway-nsg"
|
||||
+ resource_group_name = "demo-tfquickstart-dev-rg"
|
||||
+ security_rule = (known after apply)
|
||||
+ tags = (known after apply)
|
||||
}
|
||||
|
||||
# azurerm_network_security_group.ingress will be created
|
||||
+ resource "azurerm_network_security_group" "ingress" {
|
||||
+ id = (known after apply)
|
||||
+ location = "westus2"
|
||||
+ name = "demo-tfquickstart-ingress-nsg"
|
||||
+ resource_group_name = "demo-tfquickstart-dev-rg"
|
||||
+ security_rule = (known after apply)
|
||||
+ tags = (known after apply)
|
||||
}
|
||||
|
||||
# azurerm_public_ip.gateway will be created
|
||||
+ resource "azurerm_public_ip" "gateway" {
|
||||
+ allocation_method = "Static"
|
||||
+ domain_name_label = "mtcden-demo-tfquickstart-dev-gateway"
|
||||
+ fqdn = (known after apply)
|
||||
+ id = (known after apply)
|
||||
+ idle_timeout_in_minutes = 4
|
||||
+ ip_address = (known after apply)
|
||||
+ ip_version = "IPv4"
|
||||
+ location = "westus2"
|
||||
+ name = "mtcden-demo-tfquickstart-dev-gateway-ip"
|
||||
+ public_ip_address_allocation = (known after apply)
|
||||
+ resource_group_name = "demo-tfquickstart-dev-rg"
|
||||
+ sku = "Standard"
|
||||
+ tags = (known after apply)
|
||||
}
|
||||
|
||||
# azurerm_resource_group.default will be created
|
||||
+ resource "azurerm_resource_group" "default" {
|
||||
+ id = (known after apply)
|
||||
+ location = "westus2"
|
||||
+ name = "demo-tfquickstart-dev-rg"
|
||||
+ tags = (known after apply)
|
||||
}
|
||||
|
||||
# azurerm_role_assignment.default will be created
|
||||
+ resource "azurerm_role_assignment" "default" {
|
||||
+ id = (known after apply)
|
||||
+ name = (known after apply)
|
||||
+ principal_id = (known after apply)
|
||||
+ principal_type = (known after apply)
|
||||
+ role_definition_id = (known after apply)
|
||||
+ role_definition_name = "Network Contributor"
|
||||
+ scope = "/subscriptions/b0e04a4a-a321-4b66-b8fd-13715262ba3c/resourceGroups/demo-tfquickstart-dev-rg"
|
||||
+ skip_service_principal_aad_check = (known after apply)
|
||||
}
|
||||
|
||||
# azurerm_subnet.aks will be created
|
||||
+ resource "azurerm_subnet" "aks" {
|
||||
+ address_prefix = "10.1.0.0/16"
|
||||
+ id = (known after apply)
|
||||
+ ip_configurations = (known after apply)
|
||||
+ name = "demo-tfquickstart-aks-subnet"
|
||||
+ resource_group_name = "demo-tfquickstart-dev-rg"
|
||||
+ virtual_network_name = "demo-tfquickstart-vnet"
|
||||
}
|
||||
|
||||
# azurerm_subnet.gateway will be created
|
||||
+ resource "azurerm_subnet" "gateway" {
|
||||
+ address_prefix = "10.2.1.0/24"
|
||||
+ id = (known after apply)
|
||||
+ ip_configurations = (known after apply)
|
||||
+ name = "demo-tfquickstart-gateway-subnet"
|
||||
+ resource_group_name = "demo-tfquickstart-dev-rg"
|
||||
+ virtual_network_name = "demo-tfquickstart-vnet"
|
||||
}
|
||||
|
||||
# azurerm_subnet.ingress will be created
|
||||
+ resource "azurerm_subnet" "ingress" {
|
||||
+ address_prefix = "10.2.0.0/24"
|
||||
+ id = (known after apply)
|
||||
+ ip_configurations = (known after apply)
|
||||
+ name = "demo-tfquickstart-ingress-subnet"
|
||||
+ resource_group_name = "demo-tfquickstart-dev-rg"
|
||||
+ virtual_network_name = "demo-tfquickstart-vnet"
|
||||
}
|
||||
|
||||
# azurerm_subnet_network_security_group_association.aks will be created
|
||||
+ resource "azurerm_subnet_network_security_group_association" "aks" {
|
||||
+ id = (known after apply)
|
||||
+ network_security_group_id = (known after apply)
|
||||
+ subnet_id = (known after apply)
|
||||
}
|
||||
|
||||
# azurerm_subnet_network_security_group_association.gateway will be created
|
||||
+ resource "azurerm_subnet_network_security_group_association" "gateway" {
|
||||
+ id = (known after apply)
|
||||
+ network_security_group_id = (known after apply)
|
||||
+ subnet_id = (known after apply)
|
||||
}
|
||||
|
||||
# azurerm_subnet_network_security_group_association.ingress will be created
|
||||
+ resource "azurerm_subnet_network_security_group_association" "ingress" {
|
||||
+ id = (known after apply)
|
||||
+ network_security_group_id = (known after apply)
|
||||
+ subnet_id = (known after apply)
|
||||
}
|
||||
|
||||
# azurerm_virtual_network.default will be created
|
||||
+ resource "azurerm_virtual_network" "default" {
|
||||
+ address_space = [
|
||||
+ "10.0.0.0/8",
|
||||
]
|
||||
+ id = (known after apply)
|
||||
+ location = "westus2"
|
||||
+ name = "demo-tfquickstart-vnet"
|
||||
+ resource_group_name = "demo-tfquickstart-dev-rg"
|
||||
+ tags = (known after apply)
|
||||
|
||||
+ subnet {
|
||||
+ address_prefix = (known after apply)
|
||||
+ id = (known after apply)
|
||||
+ name = (known after apply)
|
||||
+ security_group = (known after apply)
|
||||
}
|
||||
}
|
||||
|
||||
# helm_release.ghost will be created
|
||||
+ resource "helm_release" "ghost" {
|
||||
+ chart = "bitnami/ghost"
|
||||
+ disable_webhooks = false
|
||||
+ force_update = false
|
||||
+ id = (known after apply)
|
||||
+ metadata = (known after apply)
|
||||
+ name = "ghost-blog"
|
||||
+ namespace = "default"
|
||||
+ recreate_pods = false
|
||||
+ reuse = false
|
||||
+ reuse_values = false
|
||||
+ status = "DEPLOYED"
|
||||
+ timeout = 300
|
||||
+ verify = false
|
||||
+ version = (known after apply)
|
||||
+ wait = true
|
||||
}
|
||||
|
||||
# helm_release.ingress will be created
|
||||
+ resource "helm_release" "ingress" {
|
||||
+ chart = "stable/nginx-ingress"
|
||||
+ disable_webhooks = false
|
||||
+ force_update = false
|
||||
+ id = (known after apply)
|
||||
+ metadata = (known after apply)
|
||||
+ name = "nginx-ingress"
|
||||
+ namespace = "kube-system"
|
||||
+ recreate_pods = false
|
||||
+ reuse = false
|
||||
+ reuse_values = false
|
||||
+ status = "DEPLOYED"
|
||||
+ timeout = 300
|
||||
+ values = [
|
||||
+ "controller:\r\n replicaCount: 2\r\n service:\r\n loadBalancerIP: 10.2.0.10\r\n annotations:\r\n service.beta.kubernetes.io/azure-load-balancer-internal: \"true\"\r\n service.beta.kubernetes.io/azure-load-balancer-internal-subnet: \"demo-tfquickstart-ingress-subnet\"\r\n",
|
||||
]
|
||||
+ verify = false
|
||||
+ version = "1.24.7"
|
||||
+ wait = true
|
||||
}
|
||||
|
||||
# kubernetes_cluster_role_binding.dashboard will be created
|
||||
+ resource "kubernetes_cluster_role_binding" "dashboard" {
|
||||
+ id = (known after apply)
|
||||
|
||||
+ metadata {
|
||||
+ generation = (known after apply)
|
||||
+ name = "kubernetes-dashboard"
|
||||
+ resource_version = (known after apply)
|
||||
+ self_link = (known after apply)
|
||||
+ uid = (known after apply)
|
||||
}
|
||||
|
||||
+ role_ref {
|
||||
+ api_group = "rbac.authorization.k8s.io"
|
||||
+ kind = "ClusterRole"
|
||||
+ name = "cluster-admin"
|
||||
}
|
||||
|
||||
+ subject {
|
||||
+ api_group = (known after apply)
|
||||
+ kind = "ServiceAccount"
|
||||
+ name = "kubernetes-dashboard"
|
||||
+ namespace = "kube-system"
|
||||
}
|
||||
}
|
||||
|
||||
# kubernetes_cluster_role_binding.default will be created
|
||||
+ resource "kubernetes_cluster_role_binding" "default" {
|
||||
+ id = (known after apply)
|
||||
|
||||
+ metadata {
|
||||
+ generation = (known after apply)
|
||||
+ name = "default"
|
||||
+ resource_version = (known after apply)
|
||||
+ self_link = (known after apply)
|
||||
+ uid = (known after apply)
|
||||
}
|
||||
|
||||
+ role_ref {
|
||||
+ api_group = "rbac.authorization.k8s.io"
|
||||
+ kind = "ClusterRole"
|
||||
+ name = "cluster-admin"
|
||||
}
|
||||
|
||||
+ subject {
|
||||
+ api_group = (known after apply)
|
||||
+ kind = "ServiceAccount"
|
||||
+ name = "default"
|
||||
+ namespace = "default"
|
||||
}
|
||||
}
|
||||
|
||||
# kubernetes_cluster_role_binding.tiller will be created
|
||||
+ resource "kubernetes_cluster_role_binding" "tiller" {
|
||||
+ id = (known after apply)
|
||||
|
||||
+ metadata {
|
||||
+ generation = (known after apply)
|
||||
+ name = "tiller"
|
||||
+ resource_version = (known after apply)
|
||||
+ self_link = (known after apply)
|
||||
+ uid = (known after apply)
|
||||
}
|
||||
|
||||
+ role_ref {
|
||||
+ api_group = "rbac.authorization.k8s.io"
|
||||
+ kind = "ClusterRole"
|
||||
+ name = "cluster-admin"
|
||||
}
|
||||
|
||||
+ subject {
|
||||
+ api_group = (known after apply)
|
||||
+ kind = "ServiceAccount"
|
||||
+ name = "tiller"
|
||||
+ namespace = "kube-system"
|
||||
}
|
||||
}
|
||||
|
||||
# kubernetes_service_account.tiller will be created
|
||||
+ resource "kubernetes_service_account" "tiller" {
|
||||
+ default_secret_name = (known after apply)
|
||||
+ id = (known after apply)
|
||||
|
||||
+ metadata {
|
||||
+ generation = (known after apply)
|
||||
+ name = "tiller"
|
||||
+ namespace = "kube-system"
|
||||
+ resource_version = (known after apply)
|
||||
+ self_link = (known after apply)
|
||||
+ uid = (known after apply)
|
||||
}
|
||||
}
|
||||
|
||||
# random_string.password will be created
|
||||
+ resource "random_string" "password" {
|
||||
+ id = (known after apply)
|
||||
+ length = 32
|
||||
+ lower = true
|
||||
+ min_lower = 0
|
||||
+ min_numeric = 0
|
||||
+ min_special = 0
|
||||
+ min_upper = 0
|
||||
+ number = true
|
||||
+ result = (known after apply)
|
||||
+ special = true
|
||||
+ upper = true
|
||||
}
|
||||
|
||||
Plan: 28 to add, 0 to change, 0 to destroy.
|
||||
|
||||
------------------------------------------------------------------------
|
||||
```
|
||||
|
||||
\* Example shown with [Bash](https://www.gnu.org/software/bash/). For [Powershell](https://docs.microsoft.com/en-us/powershell/) replace backslashes with backticks.
|
||||
85
quickstart/301-aks-enterprise/variables.tf
Normal file
85
quickstart/301-aks-enterprise/variables.tf
Normal file
@ -0,0 +1,85 @@
|
||||
// Naming
|
||||
variable "name" {
|
||||
type = "string"
|
||||
description = "Location of the azure resource group."
|
||||
default = "demo-tfquickstart"
|
||||
}
|
||||
|
||||
variable "environment" {
|
||||
type = "string"
|
||||
description = "Name of the deployment environment"
|
||||
default = "dev"
|
||||
}
|
||||
|
||||
// Resource information
|
||||
|
||||
variable "location" {
|
||||
type = "string"
|
||||
description = "Location of the azure resource group."
|
||||
default = "WestUS2"
|
||||
}
|
||||
|
||||
// Node type information
|
||||
|
||||
variable "node_count" {
|
||||
type = "string"
|
||||
description = "The number of K8S nodes to provision."
|
||||
default = 3
|
||||
}
|
||||
|
||||
variable "node_type" {
|
||||
type = "string"
|
||||
description = "The size of each node."
|
||||
default = "Standard_D1_v2"
|
||||
}
|
||||
|
||||
variable "node_os" {
|
||||
type = "string"
|
||||
description = "Windows or Linux"
|
||||
default = "Linux"
|
||||
}
|
||||
|
||||
variable "dns_prefix" {
|
||||
type = "string"
|
||||
description = "DNS Prefix"
|
||||
default = "mtcden"
|
||||
}
|
||||
|
||||
// Network information
|
||||
|
||||
variable "vnet_address_space" {
|
||||
type = "string"
|
||||
description = "Address space for the vnet"
|
||||
default = "10.0.0.0/8"
|
||||
}
|
||||
|
||||
variable "vnet_aks_subnet_space" {
|
||||
type = "string"
|
||||
description = "Address space for the AKS subnet"
|
||||
default = "10.1.0.0/16"
|
||||
}
|
||||
|
||||
variable "vnet_ingress_subnet_space" {
|
||||
type = "string"
|
||||
description = "Address space for the gateway subnet"
|
||||
default = "10.2.0.0/24"
|
||||
}
|
||||
|
||||
variable "vnet_gateway_subnet_space" {
|
||||
type = "string"
|
||||
description = "Address space for the gateway subnet"
|
||||
default = "10.2.1.0/24"
|
||||
}
|
||||
|
||||
variable "ingress_load_balancer_ip" {
|
||||
type = "string"
|
||||
description = "Address for the ingress controller load balancer"
|
||||
default = "10.2.0.10"
|
||||
}
|
||||
|
||||
|
||||
variable "gateway_instance_count" {
|
||||
type = "string"
|
||||
description = "The number of application gateways to deploy"
|
||||
default = "1"
|
||||
}
|
||||
@ -12,7 +12,7 @@ This project welcomes contributions and suggestions from the internal Microsoft
|
||||
|
||||
This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments.
|
||||
|
||||
## QuickStarts
|
||||
## Quickstarts
|
||||
|
||||
#### Beginner
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user