Convert legacy Packer json template to hcl2 template (#247)

* convert legacy Packer json template to hcl2 template

.github/workflows/e2e.yaml

@@ -23,6 +23,7 @@ jobs:
           separator: ","
           files: "quickstart/*"
           files_ignore: "**/TestRecord.md"
+          dir_names_max_depth: 2
       - name: test pr
         run: |
           az login --identity --username $MSI_ID > /dev/null

.github/workflows/pr-check.yaml

@@ -20,6 +20,7 @@ jobs:
           dir_names: "true"
           separator: ","
           files: "quickstart/*"
+          dir_names_max_depth: 2
       - name: pr-check
         run: |
           export CHANGED_FOLDERS="${{ steps.changed-files.outputs.all_changed_files }}"

quickstart/201-vmss-packer-jumpbox/main.tf

@@ -4,9 +4,25 @@ terraform {
 
   required_providers {
     azurerm = {
-      source = "hashicorp/azurerm"
+      source  = "hashicorp/azurerm"
       version = "~>2.0"
     }
+    azapi = {
+      source = "Azure/azapi"
+      version = "~> 1.0"
+    }
+    local = {
+      source  = "hashicorp/local"
+      version = "2.4.0"
+    }
+    random = {
+      source  = "hashicorp/random"
+      version = "3.5.1"
+    }
+    tls = {
+      source  = "hashicorp/tls"
+      version = "4.0.4"
+    }
   }
 }
 
@@ -14,17 +30,19 @@ provider "azurerm" {
   features {}
 }
 
+resource "random_pet" "id" {}
+
 resource "azurerm_resource_group" "vmss" {
-  name     = var.resource_group_name
+  name     = coalesce(var.resource_group_name, "201-vmss-packer-jumpbox-${random_pet.id.id}")
   location = var.location
-  tags = var.tags
+  tags     = var.tags
 }
 
 resource "random_string" "fqdn" {
- length  = 6
+  length  = 6
- special = false
+  special = false
- upper   = false
+  upper   = false
- number  = false
+  numeric = false
 }
 
 resource "azurerm_virtual_network" "vmss" {
@@ -32,23 +50,23 @@ resource "azurerm_virtual_network" "vmss" {
   address_space       = ["10.0.0.0/16"]
   location            = var.location
   resource_group_name = azurerm_resource_group.vmss.name
-  tags = var.tags
+  tags                = var.tags
 }
 
 resource "azurerm_subnet" "vmss" {
   name                 = "vmss-subnet"
   resource_group_name  = azurerm_resource_group.vmss.name
   virtual_network_name = azurerm_virtual_network.vmss.name
-  address_prefixes       = ["10.0.2.0/24"]
+  address_prefixes     = ["10.0.2.0/24"]
 }
 
 resource "azurerm_public_ip" "vmss" {
-  name                         = "vmss-public-ip"
+  name                = "vmss-public-ip"
-  location                     = var.location
+  location            = var.location
-  resource_group_name          = azurerm_resource_group.vmss.name
+  resource_group_name = azurerm_resource_group.vmss.name
-  allocation_method            = "Static"
+  allocation_method   = "Static"
-  domain_name_label            = random_string.fqdn.result
+  domain_name_label   = random_string.fqdn.result
-  tags = var.tags
+  tags                = var.tags
 }
 
 resource "azurerm_lb" "vmss" {
@@ -65,8 +83,8 @@ resource "azurerm_lb" "vmss" {
 }
 
 resource "azurerm_lb_backend_address_pool" "bpepool" {
-  loadbalancer_id     = azurerm_lb.vmss.id
+  loadbalancer_id = azurerm_lb.vmss.id
-  name                = "BackEndAddressPool"
+  name            = "BackEndAddressPool"
 }
 
 resource "azurerm_lb_probe" "vmss" {
@@ -89,7 +107,7 @@ resource "azurerm_lb_rule" "lbnatrule" {
 }
 
 data "azurerm_resource_group" "image" {
-  name                = var.packer_resource_group_name
+  name = var.packer_resource_group_name
 }
 
 data "azurerm_image" "image" {
@@ -97,6 +115,31 @@ data "azurerm_image" "image" {
   resource_group_name = data.azurerm_resource_group.image.name
 }
 
+resource "azapi_resource" "ssh_public_key" {
+  type      = "Microsoft.Compute/sshPublicKeys@2022-11-01"
+  name      = random_pet.id.id
+  location  = azurerm_resource_group.vmss.location
+  parent_id = azurerm_resource_group.vmss.id
+}
+
+resource "azapi_resource_action" "ssh_public_key_gen" {
+  type        = "Microsoft.Compute/sshPublicKeys@2022-11-01"
+  resource_id = azapi_resource.ssh_public_key.id
+  action      = "generateKeyPair"
+  method      = "POST"
+
+  response_export_values = ["publicKey", "privateKey"]
+}
+
+resource "random_password" "password" {
+  count  = var.admin_password == null ? 1 : 0
+  length = 20
+}
+
+locals {
+  admin_password = try(random_password.password[0].result, var.admin_password)
+}
+
 resource "azurerm_virtual_machine_scale_set" "vmss" {
   name                = "vmscaleset"
   location            = var.location
@@ -110,7 +153,7 @@ resource "azurerm_virtual_machine_scale_set" "vmss" {
   }
 
   storage_profile_image_reference {
-    id=data.azurerm_image.image.id
+    id = data.azurerm_image.image.id
   }
 
   storage_profile_os_disk {
@@ -121,16 +164,16 @@ resource "azurerm_virtual_machine_scale_set" "vmss" {
   }
 
   storage_profile_data_disk {
-    lun          = 0
+    lun           = 0
-    caching        = "ReadWrite"
+    caching       = "ReadWrite"
-    create_option  = "Empty"
+    create_option = "Empty"
-    disk_size_gb   = 10
+    disk_size_gb  = 10
   }
 
   os_profile {
     computer_name_prefix = "vmlab"
     admin_username       = var.admin_user
-    admin_password       = var.admin_password
+    admin_password       = local.admin_password
   }
 
   os_profile_linux_config {
@@ -138,7 +181,7 @@ resource "azurerm_virtual_machine_scale_set" "vmss" {
 
     ssh_keys {
       path     = "/home/azureuser/.ssh/authorized_keys"
-      key_data = file("~/.ssh/id_rsa.pub")
+      key_data = jsondecode(azapi_resource_action.ssh_public_key_gen.output).publicKey
     }
   }
 
@@ -150,20 +193,20 @@ resource "azurerm_virtual_machine_scale_set" "vmss" {
       name                                   = "IPConfiguration"
       subnet_id                              = azurerm_subnet.vmss.id
       load_balancer_backend_address_pool_ids = [azurerm_lb_backend_address_pool.bpepool.id]
-      primary = true
+      primary                                = true
     }
   }
-  
+
   tags = var.tags
 }
 
 resource "azurerm_public_ip" "jumpbox" {
-  name                         = "jumpbox-public-ip"
+  name                = "jumpbox-public-ip"
-  location                     = var.location
+  location            = var.location
-  resource_group_name          = azurerm_resource_group.vmss.name
+  resource_group_name = azurerm_resource_group.vmss.name
-  allocation_method            = "Static"
+  allocation_method   = "Static"
-  domain_name_label            = "${random_string.fqdn.result}-ssh"
+  domain_name_label   = "${random_string.fqdn.result}-ssh"
-  tags = var.tags
+  tags                = var.tags
 }
 
 resource "azurerm_network_interface" "jumpbox" {
@@ -205,7 +248,7 @@ resource "azurerm_virtual_machine" "jumpbox" {
   os_profile {
     computer_name  = "jumpbox"
     admin_username = var.admin_user
-    admin_password = var.admin_password
+    admin_password = local.admin_password
   }
 
   os_profile_linux_config {
@@ -213,7 +256,7 @@ resource "azurerm_virtual_machine" "jumpbox" {
 
     ssh_keys {
       path     = "/home/azureuser/.ssh/authorized_keys"
-      key_data = file("~/.ssh/id_rsa.pub")
+      key_data = jsondecode(azapi_resource_action.ssh_public_key_gen.output).publicKey
     }
   }
 

quickstart/201-vmss-packer-jumpbox/output.tf

@@ -1,11 +1,11 @@
 output "vmss_public_ip_fqdn" {
-    value = azurerm_public_ip.vmss.fqdn
+  value = azurerm_public_ip.vmss.fqdn
 }
 
 output "jumpbox_public_ip_fqdn" {
-    value = azurerm_public_ip.jumpbox.fqdn
+  value = azurerm_public_ip.jumpbox.fqdn
 }
 
 output "jumpbox_public_ip" {
-    value = azurerm_public_ip.jumpbox.ip_address
+  value = azurerm_public_ip.jumpbox.ip_address
-}
+}

quickstart/201-vmss-packer-jumpbox/packer_image_resource_group/main.tf

@@ -0,0 +1,6 @@
+resource "random_pet" "id" {}
+
+resource "azurerm_resource_group" "image_group" {
+  location = "eastus"
+  name     = "packer-image-${random_pet.id.id}"
+}

quickstart/201-vmss-packer-jumpbox/packer_image_resource_group/outputs.tf

@@ -0,0 +1,3 @@
+output "resource_group_name" {
+  value = azurerm_resource_group.image_group.name
+}

quickstart/201-vmss-packer-jumpbox/packer_image_resource_group/providers.tf

@@ -0,0 +1,26 @@
+terraform {
+  required_version = ">=1.2"
+
+  required_providers {
+    azurerm = {
+      source  = "hashicorp/azurerm"
+      version = "~> 3.0"
+    }
+    helm = {
+      source  = "hashicorp/helm"
+      version = "2.9.0"
+    }
+    random = {
+      source  = "hashicorp/random"
+      version = "~> 3.0"
+    }
+  }
+}
+
+provider "azurerm" {
+  features {
+    resource_group {
+      prevent_deletion_if_contains_resources = false
+    }
+  }
+}

quickstart/201-vmss-packer-jumpbox/ubuntu.json

@@ -1,38 +1,38 @@
 {
-    "builders": [{
+  "builders": [{
-      "type": "azure-arm",
+    "type": "azure-arm",
-  
+
-      "client_id": "0bfc2293-4d69-49b5-83f7-bf0d60d20c45",
+    "client_id": "0bfc2293-4d69-49b5-83f7-bf0d60d20c45",
-      "client_secret": "G3.6ytCh44Kcla~_JRPBDLkzsXLOa3edDL",
+    "client_secret": "G3.6ytCh44Kcla~_JRPBDLkzsXLOa3edDL",
-      "tenant_id": "c3fd441d-b8ad-487e-aa27-453079018fca",
+    "tenant_id": "c3fd441d-b8ad-487e-aa27-453079018fca",
-      "subscription_id": "b162117f-53fa-4f42-8c77-6a65ca966c40",
+    "subscription_id": "b162117f-53fa-4f42-8c77-6a65ca966c40",
-  
+
-      "managed_image_resource_group_name": "myPackerImages",
+    "managed_image_resource_group_name": "myPackerImages",
-      "managed_image_name": "myPackerImage",
+    "managed_image_name": "myPackerImage",
-  
+
-      "os_type": "Linux",
+    "os_type": "Linux",
-      "image_publisher": "Canonical",
+    "image_publisher": "Canonical",
-      "image_offer": "UbuntuServer",
+    "image_offer": "UbuntuServer",
-      "image_sku": "16.04-LTS",
+    "image_sku": "16.04-LTS",
-  
+
-      "azure_tags": {
+    "azure_tags": {
-          "dept": "Engineering",
+      "dept": "Engineering",
-          "task": "Image deployment"
+      "task": "Image deployment"
-      },
+    },
-  
+
-      "location": "East US",
+    "location": "East US",
-      "vm_size": "Standard_DS2_v2"
+    "vm_size": "Standard_DS2_v2"
-    }],
+  }],
-    "provisioners": [{
+  "provisioners": [{
-      "execute_command": "chmod +x {{ .Path }}; {{ .Vars }} sudo -E sh '{{ .Path }}'",
+    "execute_command": "chmod +x {{ .Path }}; {{ .Vars }} sudo -E sh '{{ .Path }}'",
-      "inline": [
+    "inline": [
-        "apt-get update",
+      "apt-get update",
-        "apt-get upgrade -y",
+      "apt-get upgrade -y",
-        "apt-get -y install nginx",
+      "apt-get -y install nginx",
-  
+
-        "/usr/sbin/waagent -force -deprovision+user && export HISTSIZE=0 && sync"
+      "/usr/sbin/waagent -force -deprovision+user && export HISTSIZE=0 && sync"
-      ],
+    ],
-      "inline_shebang": "/bin/sh -x",
+    "inline_shebang": "/bin/sh -x",
-      "type": "shell"
+    "type": "shell"
-    }]
+  }]
-  }
+}

quickstart/201-vmss-packer-jumpbox/ubuntu.pkr.hcl

@@ -0,0 +1,68 @@
+packer {
+  required_plugins {
+    azure = {
+      source  = "github.com/hashicorp/azure"
+      version = "~> 2"
+    }
+  }
+}
+
+variable client_id {
+  type = string
+  default = null
+}
+variable client_secret {
+  type = string
+  default = null
+}
+
+variable subscription_id {
+  type = string
+  default = null
+}
+
+variable tenant_id {
+  type = string
+  default = null
+}
+
+variable location {
+  default = "eastus"
+}
+
+variable "image_resource_group_name" {
+  description = "Name of the resource group in which the Packer image will be created"
+  default     = "myPackerImages"
+}
+
+source "azure-arm" "builder" {
+  client_id                         = var.client_id
+  client_secret                     = var.client_secret
+  image_offer                       = "UbuntuServer"
+  image_publisher                   = "canonical"
+  image_sku                         = "16.04-LTS"
+  location                          = var.location
+  managed_image_name                = "myPackerImage"
+  managed_image_resource_group_name = var.image_resource_group_name
+  os_type                           = "Linux"
+  subscription_id                   = var.subscription_id
+  tenant_id                         = var.tenant_id
+  vm_size                           = "Standard_DS2_v2"
+  azure_tags                        = {
+    "dept" : "Engineering",
+    "task" : "Image deployment",
+  }
+}
+
+build {
+  sources = ["source.azure-arm.builder"]
+  provisioner "shell" {
+    execute_command = "chmod +x {{ .Path }}; {{ .Vars }} sudo -E sh '{{ .Path }}'"
+    inline = [
+      "apt-get update",
+      "apt-get upgrade -y",
+      "apt-get -y install nginx",
+      "/usr/sbin/waagent -force -deprovision+user && export HISTSIZE=0 && sync",
+    ]
+  }
+}

quickstart/201-vmss-packer-jumpbox/variables.tf

@@ -1,46 +1,42 @@
 variable "packer_resource_group_name" {
-   description = "Name of the resource group in which the Packer image will be created"
+  description = "Name of the resource group in which the Packer image will be created"
-   default     = "myPackerImages"
+  default     = "myPackerImages"
 }
 
 variable "packer_image_name" {
-   description = "Name of the Packer image"
+  description = "Name of the Packer image"
-   default     = "myPackerImage"
+  default     = "myPackerImage"
 }
 
 variable "resource_group_name" {
-   description = "Name of the resource group in which the Packer image  will be created"
+  description = "Name of the resource group in which the Packer image  will be created"
-   default     = "myPackerImages"
+  default     = null
-}
-
-variable "resource_group_name" {
-   description = "Name of the resource group in which the resources will be created"
-   default     = "myResourceGroup"
 }
 
 variable "location" {
-   default = "eastus"
+  default     = "eastus"
-   description = "Location where resources will be created"
+  description = "Location where resources will be created"
 }
 
 variable "tags" {
-   description = "Map of the tags to use for the resources that are deployed"
+  description = "Map of the tags to use for the resources that are deployed"
-   type        = map(string)
+  type        = map(string)
-   default = {
+  default     = {
-      environment = "codelab"
+    environment = "codelab"
-   }
+  }
 }
 
 variable "application_port" {
-   description = "Port that you want to expose to the external load balancer"
+  description = "Port that you want to expose to the external load balancer"
-   default     = 80
+  default     = 80
 }
 
 variable "admin_user" {
-   description = "User name to use as the admin account on the VMs that will be part of the VM scale set"
+  description = "User name to use as the admin account on the VMs that will be part of the VM scale set"
-   default     = "azureuser"
+  default     = "azureuser"
 }
 
 variable "admin_password" {
-   description = "Default password for admin account"
+  description = "Default password for admin account"
+  default     = null
 }

test/e2e/quickstart_test.go

@@ -2,6 +2,9 @@ package e2e
 
 import (
 	"github.com/gruntwork-io/terratest/modules/files"
+	"github.com/gruntwork-io/terratest/modules/packer"
+	test_structure "github.com/gruntwork-io/terratest/modules/test-structure"
+	"github.com/stretchr/testify/require"
 	"os"
 	"path/filepath"
 	"strings"
@@ -11,6 +14,10 @@ import (
 	"github.com/gruntwork-io/terratest/modules/terraform"
 )
 
+var speicalTests = map[string]func(*testing.T){
+	"quickstart/201-vmss-packer-jumpbox": test201VmssPackerJumpbox,
+}
+
 func Test_Quickstarts(t *testing.T) {
 	msiId := os.Getenv("MSI_ID")
 	if msiId != "" {
@@ -25,6 +32,7 @@ func Test_Quickstarts(t *testing.T) {
 			t.Fatalf(err.Error())
 		}
 	}
+	folders = removeDuplicates(folders)
 	for _, f := range folders {
 		f = strings.TrimSpace(f)
 		if filepath.Dir(f) != "quickstart" {
@@ -35,11 +43,16 @@ func Test_Quickstarts(t *testing.T) {
 		if !files.IsExistingDir(path) {
 			continue
 		}
-		t.Run(f, func(t *testing.T) {
+		test, ok := speicalTests[f]
-			helper.RunE2ETest(t, rootPath, f, terraform.Options{
+		if !ok {
-				Upgrade: true,
+			test = func(t *testing.T) {
-			}, nil)
+				helper.RunE2ETest(t, rootPath, f, terraform.Options{
-		})
+					Upgrade: true,
+				}, nil)
+			}
+		}
+
+		t.Run(f, test)
 	}
 }
 
@@ -57,3 +70,65 @@ func allExamples() ([]string, error) {
 	}
 	return r, nil
 }
+
+func test201VmssPackerJumpbox(t *testing.T) {
+	examplePath := filepath.Join("..", "..", "quickstart", "201-vmss-packer-jumpbox")
+	examplePath = test_structure.CopyTerraformFolderToTemp(t, examplePath, "")
+	defer func() {
+		_ = os.RemoveAll(examplePath)
+	}()
+	harnessPath := filepath.Join(examplePath, "packer_image_resource_group")
+	harnessOptions := &terraform.Options{
+		TerraformDir: harnessPath,
+	}
+	defer terraform.Destroy(t, harnessOptions)
+	terraform.InitAndApply(t, harnessOptions)
+	harnessOutput := terraform.OutputAll(t, harnessOptions)
+	imageResourceGroupName := harnessOutput["resource_group_name"].(string)
+	pkrCfg := filepath.Join(examplePath, "ubuntu.pkr.hcl")
+	packerVars := map[string]string{
+		"image_resource_group_name": imageResourceGroupName,
+	}
+	useMsi := false
+	if clientId := os.Getenv("ARM_CLIENT_ID"); clientId != "" {
+		packerVars["client_id"] = clientId
+	}
+	if identityId := os.Getenv("MSI_ID"); identityId != "" {
+		packerVars["client_id"] = identityId
+		useMsi = true
+	}
+	if clientSecret := os.Getenv("ARM_CLIENT_SECRET"); clientSecret != "" {
+		packerVars["client_secret"] = clientSecret
+	}
+	if subscriptionId := os.Getenv("ARM_SUBSCRIPTION_ID"); subscriptionId != "" {
+		packerVars["subscription_id"] = subscriptionId
+	}
+	if tenantId := os.Getenv("ARM_TENANT_ID"); !useMsi && tenantId != "" {
+		packerVars["tenant_id"] = tenantId
+	}
+	_, err := packer.BuildArtifactE(t, &packer.Options{
+		Template:   pkrCfg,
+		Vars:       packerVars,
+		VarFiles:   nil,
+		WorkingDir: examplePath,
+	})
+	require.NoError(t, err)
+	helper.RunE2ETest(t, examplePath, "", terraform.Options{
+		Upgrade: true,
+		Vars: map[string]interface{}{
+			"packer_resource_group_name": imageResourceGroupName,
+		},
+	}, nil)
+}
+
+func removeDuplicates(s []string) []string {
+	m := make(map[string]struct{})
+	result := []string{}
+	for _, item := range s {
+		if _, ok := m[item]; !ok {
+			m[item] = struct{}{}
+			result = append(result, item)
+		}
+	}
+	return result
+}

test/go.mod

@@ -5,6 +5,7 @@ go 1.19
 require (
 	github.com/Azure/terraform-module-test-helper v0.8.0
 	github.com/gruntwork-io/terratest v0.41.9
+	github.com/stretchr/testify v1.8.1
 )
 
 require (
@@ -66,7 +67,6 @@ require (
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/spf13/afero v1.9.3 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/stretchr/testify v1.8.1 // indirect
 	github.com/tmccombs/hcl2json v0.3.3 // indirect
 	github.com/ulikunitz/xz v0.5.8 // indirect
 	github.com/urfave/cli v1.22.2 // indirect