add ci (#140)

.github/workflows/e2e.yaml

@@ -0,0 +1,37 @@
+name: E2E Test Check
+on:
+  pull_request:
+    types: ['opened', 'synchronize']
+    paths:
+      - '.github/**'
+      - '.github/workflows/**'
+      - 'quickstart/**'
+
+jobs:
+  e2e-check:
+    runs-on: [self-hosted, 1ES.Pool=terraform-azurerm-doc]
+    environment:
+      name: acctests
+    steps:
+      - name: checkout
+        uses: actions/checkout@v3
+      - name: Get changed files
+        id: changed-files
+        uses: tj-actions/changed-files@v34
+        with:
+          dir_names: "true"
+          separator: ","
+          files: "quickstart/*"
+      - name: test pr
+        run: |
+          az login --identity --username $MSI_ID > /dev/null
+          export ARM_SUBSCRIPTION_ID=$(az login --identity --username $MSI_ID | jq -r '.[0] | .id')
+          export ARM_TENANT_ID=$(az login --identity --username $MSI_ID | jq -r '.[0] | .tenantId')
+          export CHANGED_FOLDERS="${{ steps.changed-files.outputs.all_changed_files }}"
+          docker run --rm -v $(pwd):/src -w /src/test --network=host -e MSI_ID -e ARM_SUBSCRIPTION_ID -e ARM_TENANT_ID -e ARM_USE_MSI=true -e CHANGED_FOLDERS mcr.microsoft.com/azterraform:latest sh -c "go mod tidy && go test -timeout=360m -v ./e2e"
+      - uses: actions/upload-artifact@v3
+        with:
+          name: TestRecord-${{ github.event.number }}
+          retention-days: 60
+          path: |
+            quickstart/**/TestRecord.md.tmp

.github/workflows/pr-check.yaml

@@ -0,0 +1,29 @@
+name: Pre Pull Request Check
+on:
+  pull_request:
+    types: [ 'opened', 'synchronize' ]
+    paths:
+      - '.github/**'
+      - '.github/workflows/**'
+      - 'quickstart/**'
+
+jobs:
+  prepr-check:
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout
+        uses: actions/checkout@v3
+      - name: Get changed files
+        id: changed-files
+        uses: tj-actions/changed-files@v34
+        with:
+          dir_names: "true"
+          separator: ","
+          files: "quickstart/*"
+      - name: pr-check
+        run: |
+          export CHANGED_FOLDERS="${{ steps.changed-files.outputs.all_changed_files }}"
+          if [ -z "${{ github.event.number }}" ]; then
+            CHANGED_FOLDERS=$(find ./quickstart -maxdepth 1 -mindepth 1 -type d | tr '\n' ',')
+          fi
+          docker run --rm -v $(pwd):/src -w /src -e CHANGED_FOLDERS mcr.microsoft.com/azterraform:latest make pr-check

.github/workflows/pr-merged.yaml

@@ -0,0 +1,39 @@
+name: Main Branch Push
+on:
+  push:
+    branches:
+      - main
+      - master
+    paths:
+      - 'quickstart/**'
+jobs:
+  main-branch-push:
+    runs-on: ubuntu-latest
+    steps:
+      - name: get-default-branch
+        run: |
+          branch=$(curl -s "https://api.github.com/repos/$GITHUB_REPOSITORY" | jq -r '.default_branch')
+          echo "default_branch=$branch" >> $GITHUB_ENV
+      - name: checkout
+        uses: actions/checkout@v3
+      - uses: 8BitJonny/gh-get-current-pr@2.1.0
+        id: PR
+      - name: Download artifact
+        id: download-artifact
+        uses: dawidd6/action-download-artifact@v2
+        with:
+          github_token: ${{secrets.GITHUB_TOKEN}}
+          workflow: e2e.yaml
+          pr: ${{ steps.PR.outputs.number }}
+          name: TestRecord-${{ steps.PR.outputs.number }}
+          path: TestRecord
+          if_no_artifact_found: ignore
+      - name: Update
+        run: |
+          sh scripts/update-test-record.sh
+      - name: Commit & Push changes
+        uses: actions-js/push@master
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          message: 'Update TestRecord'
+          branch: ${{ env.default_branch }}

.github/workflows/weekly-e2e.yaml

@@ -0,0 +1,35 @@
+name: Weekly E2E Test Check
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 0 * * 0'
+
+jobs:
+  full-e2e-check:
+    runs-on: [self-hosted, 1ES.Pool=terraform-azurerm-doc]
+    timeout-minutes: 1440
+    environment:
+      name: crontests
+    steps:
+      - name: checkout
+        uses: actions/checkout@v3
+      - name: test all examples
+        continue-on-error: true
+        timeout-minutes: 1440
+        run: |
+          az login --identity --username $MSI_ID > /dev/null
+          export ARM_SUBSCRIPTION_ID=$(az login --identity --username $MSI_ID | jq -r '.[0] | .id')
+          export ARM_TENANT_ID=$(az login --identity --username $MSI_ID | jq -r '.[0] | .tenantId')
+          docker run --rm -v $(pwd):/src -w /src/test -e MSI_ID -e ARM_SUBSCRIPTION_ID -e ARM_TENANT_ID -e ARM_USE_MSI=true -e CHANGED_FOLDERS mcr.microsoft.com/azterraform sh -c "go mod tidy && go test -timeout=1440m -v ./e2e"
+      - name: Update
+        run: |
+          docker run --rm -v $(pwd):/src -w /src mcr.microsoft.com/azterraform sh scripts/update-test-record.sh
+          cd .git
+          sudo chmod -R a+rwX .
+          sudo find . -type d -exec chmod g+s '{}' +
+      - name: Commit & Push changes
+        uses: actions-js/push@master
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          message: 'Update TestRecord'
+          branch: ${{ env.default_branch }}

.gitignore

@@ -10,3 +10,10 @@ _repo.*/
 .ionide/
 .openpublishing.buildcore.ps1.vscode/
 *.DS_Store
+
+.terraform
+.terraform.lock.hcl
+*.tmp
+go.sum
+
+TestRecord

GNUmakefile

@@ -0,0 +1,90 @@
+fmt:
+	@echo "==> Fixing source code with gofmt..."
+	# This logic should match the search logic in scripts/gofmtcheck.sh
+	find . -name '*.go' | grep -v vendor | xargs gofmt -s -w
+
+fumpt:
+	@echo "==> Fixing source code with Gofumpt..."
+	# This logic should match the search logic in scripts/gofmtcheck.sh
+	find . -name '*.go' | grep -v vendor | xargs gofumpt -w
+
+gosec:
+	@echo "==> Checking go code with gosec..."
+	cd test && gosec -tests ./...
+
+tffmt:
+	@echo "==> Formatting terraform code..."
+	terraform fmt -recursive
+
+tffmtcheck:
+	@sh "$(CURDIR)/scripts/terraform-fmt.sh"
+
+tfvalidatecheck:
+	@sh "$(CURDIR)/scripts/terraform-validate.sh"
+
+terrafmtcheck:
+	@sh "$(CURDIR)/scripts/terrafmt-check.sh"
+
+gofmtcheck:
+	@sh "$(CURDIR)/scripts/gofmtcheck.sh"
+	@sh "$(CURDIR)/scripts/fumptcheck.sh"
+
+golint:
+	@sh "$(CURDIR)/scripts/run-golangci-lint.sh"
+
+tflint:
+	@sh "$(CURDIR)/scripts/run-tflint.sh"
+
+lint: golint tflint gosec
+
+checkovcheck:
+	@sh "$(CURDIR)/scripts/checkovcheck.sh"
+
+checkovplancheck:
+	@sh "$(CURDIR)/scripts/checkovplancheck.sh"
+
+fmtcheck: tfvalidatecheck # tffmtcheck terrafmtcheck
+
+pr-check: fmtcheck
+
+unit-test:
+	@sh "$(CURDIR)/scripts/run-unit-test.sh"
+
+e2e-test:
+	@sh "$(CURDIR)/scripts/run-e2e-test.sh"
+
+version-upgrade-test:
+	@sh "$(CURDIR)/scripts/version-upgrade-test.sh"
+
+terrafmt:
+	@echo "==> Fixing test and document terraform blocks code with terrafmt..."
+	@find . -name '*.md' -o -name "*.go" | grep -v -e '.github' -e '.terraform' -e 'vendor' | while read f; do terrafmt fmt -f $$f; done
+
+pre-commit: tffmt terrafmt depsensure fmt fumpt generate
+
+depsensure:
+	@sh "$(CURDIR)/scripts/deps-ensure.sh"
+
+depscheck:
+	@sh "$(CURDIR)/scripts/deps-check.sh"
+
+generate:
+	@echo "--> Generating doc"
+	@rm -f .terraform.lock.hcl
+	@terraform-docs markdown table --output-file README.md --output-mode inject ./
+	@markdown-table-formatter README.md
+
+gencheck:
+	@echo "==> Generating..."
+	@cp README.md README-generated.md
+	@terraform-docs markdown table --output-file README-generated.md --output-mode inject ./
+	@markdown-table-formatter README-generated.md
+	@echo "==> Comparing generated code to committed code..."
+	@diff -q README.md README-generated.md || \
+    		(echo; echo "Unexpected difference in generated document. Run 'make pre-commit' to update the generated document and commit."; exit 1)
+
+test: fmtcheck
+	@TEST=$(TEST) ./scripts/run-gradually-deprecated.sh
+	@TEST=$(TEST) ./scripts/run-test.sh
+
+.PHONY: fmt fmtcheck pr-check

scripts/terrafmt-check.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+echo "==> Checking documentation terraform blocks are formatted..."
+files=$(find . -type f -name "*.md" -o -name "*.go" | grep -v -e ".github" -e "-terraform" -e "vendor" -e ".terraform")
+error=false
+for f in $files; do
+  terrafmt diff -c -q "$f" || error=true
+done
+if ${error}; then
+  echo "------------------------------------------------"
+  echo ""
+  echo "The preceding files contain terraform blocks that are not correctly formatted or contain errors."
+  echo "You can fix this by running make tools and then terrafmt on them."
+  echo ""
+  echo "to easily fix all terraform blocks:"
+  echo "$ make terrafmt"
+  echo ""
+  exit 1
+fi
+exit 0

scripts/terraform-fmt.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+echo "==> Checking terraform codes are formatted..."
+error=false
+terraform fmt -check -recursive || error=true
+if ${error}; then
+  echo "------------------------------------------------"
+  echo ""
+  echo "The preceding files contain terraform codes that are not correctly formatted or contain errors."
+  echo "You can fix this by running make tools and then tffmt on them."
+  echo ""
+  echo "to easily fix all terraform codes:"
+  echo "$ make tffmt"
+  echo ""
+  exit 1
+fi
+exit 0

scripts/terraform-validate.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+error=false
+
+folders=$CHANGED_FOLDERS
+for f in ${folders//,/ }
+do
+	f=$(echo $f | xargs echo -n)
+  (echo "===> Terraform validating in" /src/$f && cd /src/$f && rm -f .terraform.lock.hcl && rm -rf .terraform && terraform init -upgrade && terraform validate -json | jq -e .valid) || error=true
+  if ${error}; then
+     echo "------------------------------------------------"
+     echo ""
+     echo "Some Terraform codes contain errors."
+     echo "$(cd /src/$f && terraform validate -json)"
+     echo ""
+     exit 1
+  fi
+done
+
+exit 0

scripts/update-test-record.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+if [ ! -d "TestRecord" ]; then
+  echo "No TestRecord found, exit"
+  exit 0
+fi
+
+cd TestRecord
+
+folders=$(find ./ -maxdepth 1 -mindepth 1 -type d)
+for f in $folders; do
+  d=${f#"./"}
+
+  if [ ! -f ../quickstart/$d/TestRecord.md ]; then
+    touch ../quickstart/$d/TestRecord.md
+  fi
+
+  cat ../quickstart/$d/TestRecord.md >> $d/TestRecord.md.tmp
+  cat $d/TestRecord.md.tmp > ../quickstart/$d/TestRecord.md
+done
+
+cd ..
+git add **/TestRecord.md

test/e2e/quickstart_test.go

@@ -0,0 +1,55 @@
+package e2e
+
+import (
+	"github.com/gruntwork-io/terratest/modules/files"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	helper "github.com/Azure/terraform-module-test-helper"
+	"github.com/gruntwork-io/terratest/modules/terraform"
+)
+
+func Test_Quickstarts(t *testing.T) {
+	input := os.Getenv("CHANGED_FOLDERS")
+	folders := strings.Split(input, ",")
+	if input == "" {
+		var err error
+		folders, err = allExamples()
+		if err != nil {
+			t.Fatalf(err.Error())
+		}
+	}
+	for _, f := range folders {
+		f = strings.TrimSpace(f)
+		if filepath.Dir(f) != "quickstart" {
+			continue
+		}
+		rootPath := filepath.Join("..", "..")
+		path := filepath.Join(rootPath, f)
+		if !files.IsExistingDir(path) {
+			continue
+		}
+		t.Run(f, func(t *testing.T) {
+			helper.RunE2ETest(t, rootPath, f, terraform.Options{
+				Upgrade: true,
+			}, nil)
+		})
+	}
+}
+
+func allExamples() ([]string, error) {
+	examples, err := os.ReadDir("../../quickstart")
+	if err != nil {
+		return nil, err
+	}
+	var r []string
+	for _, f := range examples {
+		if !f.IsDir() {
+			continue
+		}
+		r = append(r, filepath.Join("quickstart", f.Name()))
+	}
+	return r, nil
+}

test/go.mod

@@ -0,0 +1,103 @@
+module github.com/Azure/terraform
+
+go 1.19
+
+require (
+	github.com/Azure/terraform-module-test-helper v0.8.0
+	github.com/gruntwork-io/terratest v0.41.9
+)
+
+require (
+	cloud.google.com/go v0.83.0 // indirect
+	cloud.google.com/go/storage v1.14.0 // indirect
+	github.com/agext/levenshtein v1.2.3 // indirect
+	github.com/ahmetb/go-linq/v3 v3.2.0 // indirect
+	github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect
+	github.com/aws/aws-sdk-go v1.40.56 // indirect
+	github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect
+	github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c // indirect
+	github.com/go-errors/errors v1.0.2-0.20180813162953-d98b870cc4e0 // indirect
+	github.com/go-logr/logr v0.2.0 // indirect
+	github.com/go-sql-driver/mysql v1.4.1 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/golang/snappy v0.0.3 // indirect
+	github.com/google/go-cmp v0.5.8 // indirect
+	github.com/google/go-github/v42 v42.0.0 // indirect
+	github.com/google/go-querystring v1.1.0 // indirect
+	github.com/google/gofuzz v1.1.0 // indirect
+	github.com/google/uuid v1.2.0 // indirect
+	github.com/googleapis/gax-go/v2 v2.0.5 // indirect
+	github.com/googleapis/gnostic v0.4.1 // indirect
+	github.com/gruntwork-io/go-commons v0.8.0 // indirect
+	github.com/hashicorp/errwrap v1.0.0 // indirect
+	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
+	github.com/hashicorp/go-getter v1.6.1 // indirect
+	github.com/hashicorp/go-getter/v2 v2.1.1 // indirect
+	github.com/hashicorp/go-multierror v1.1.0 // indirect
+	github.com/hashicorp/go-safetemp v1.0.0 // indirect
+	github.com/hashicorp/go-version v1.5.0 // indirect
+	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/hashicorp/hcl/v2 v2.15.0 // indirect
+	github.com/hashicorp/terraform-config-inspect v0.0.0-20211115214459-90acf1ca460f // indirect
+	github.com/hashicorp/terraform-json v0.14.0 // indirect
+	github.com/imdario/mergo v0.3.11 // indirect
+	github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
+	github.com/json-iterator/go v1.1.11 // indirect
+	github.com/jstemmer/go-junit-report v0.9.1 // indirect
+	github.com/klauspost/compress v1.13.0 // indirect
+	github.com/lonegunmanb/tfmodredirector v0.1.0 // indirect
+	github.com/magodo/hclgrep v0.0.0-20220303061548-1b2b24c7caf6 // indirect
+	github.com/mattn/go-zglob v0.0.2-0.20190814121620-e3c945676326 // indirect
+	github.com/minamijoyo/hcledit v0.2.6 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/mitchellh/go-testing-interface v1.0.0 // indirect
+	github.com/mitchellh/go-wordwrap v1.0.1 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/pquerna/otp v1.2.0 // indirect
+	github.com/r3labs/diff/v3 v3.0.1 // indirect
+	github.com/russross/blackfriday/v2 v2.1.0 // indirect
+	github.com/spf13/afero v1.9.3 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/stretchr/testify v1.8.1 // indirect
+	github.com/tmccombs/hcl2json v0.3.3 // indirect
+	github.com/ulikunitz/xz v0.5.8 // indirect
+	github.com/urfave/cli v1.22.2 // indirect
+	github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect
+	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
+	github.com/zclconf/go-cty v1.12.1 // indirect
+	go.opencensus.io v0.23.0 // indirect
+	golang.org/x/crypto v0.0.0-20220517005047-85d78b3ac167 // indirect
+	golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect
+	golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect
+	golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 // indirect
+	golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c // indirect
+	golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c // indirect
+	golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 // indirect
+	golang.org/x/text v0.3.7 // indirect
+	golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e // indirect
+	golang.org/x/tools v0.1.10 // indirect
+	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
+	google.golang.org/api v0.47.0 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c // indirect
+	google.golang.org/grpc v1.38.0 // indirect
+	google.golang.org/protobuf v1.26.0 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	k8s.io/api v0.20.6 // indirect
+	k8s.io/apimachinery v0.20.6 // indirect
+	k8s.io/client-go v0.20.6 // indirect
+	k8s.io/klog/v2 v2.4.0 // indirect
+	k8s.io/utils v0.0.0-20201110183641-67b214c5f920 // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.0.3 // indirect
+	sigs.k8s.io/yaml v1.2.0 // indirect
+)