hcornet
506716e703
Some checks failed
Deployment Verification / deploy-and-test (push) Failing after 29s
985 lines
28 KiB
Go
985 lines
28 KiB
Go
package main
|
|
|
|
// Docker
|
|
import (
|
|
"archive/tar"
|
|
|
|
"github.com/shuffle/shuffle-shared"
|
|
|
|
//"bufio"
|
|
"path/filepath"
|
|
//"strconv"
|
|
|
|
"bytes"
|
|
"context"
|
|
"encoding/base64"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
|
|
//"github.com/docker/docker"
|
|
"github.com/docker/docker/api/types"
|
|
//"github.com/docker/docker/api/types/container"
|
|
"github.com/docker/docker/client"
|
|
newdockerclient "github.com/fsouza/go-dockerclient"
|
|
"github.com/go-git/go-billy/v5"
|
|
|
|
//network "github.com/docker/docker/api/types/network"
|
|
//natting "github.com/docker/go-connections/nat"
|
|
|
|
"io"
|
|
"io/ioutil"
|
|
"log"
|
|
"net/http"
|
|
"os"
|
|
"strings"
|
|
|
|
batchv1 "k8s.io/api/batch/v1"
|
|
corev1 "k8s.io/api/core/v1"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/client-go/kubernetes"
|
|
"k8s.io/client-go/rest"
|
|
"time"
|
|
// "k8s.io/client-go/tools/clientcmd"
|
|
// "k8s.io/client-go/util/homedir"
|
|
)
|
|
|
|
// Parses a directory with a Dockerfile into a tar for Docker images..
|
|
func getParsedTar(tw *tar.Writer, baseDir, extra string) error {
|
|
return filepath.Walk(baseDir, func(file string, fi os.FileInfo, err error) error {
|
|
if file == baseDir {
|
|
return nil
|
|
}
|
|
|
|
//log.Printf("File: %s", file)
|
|
//log.Printf("Fileinfo: %#v", fi)
|
|
switch mode := fi.Mode(); {
|
|
case mode.IsDir():
|
|
// do directory recursion
|
|
//log.Printf("DIR: %s", file)
|
|
|
|
// Append "src" as extra here
|
|
filenamesplit := strings.Split(file, "/")
|
|
filename := fmt.Sprintf("%s%s/", extra, filenamesplit[len(filenamesplit)-1])
|
|
|
|
tmpExtra := fmt.Sprintf(filename)
|
|
//log.Printf("TmpExtra: %s", tmpExtra)
|
|
err = getParsedTar(tw, file, tmpExtra)
|
|
if err != nil {
|
|
log.Printf("Directory parse issue: %s", err)
|
|
return err
|
|
}
|
|
case mode.IsRegular():
|
|
// do file stuff
|
|
//log.Printf("FILE: %s", file)
|
|
|
|
fileReader, err := os.Open(file)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Read the actual Dockerfile
|
|
readFile, err := ioutil.ReadAll(fileReader)
|
|
if err != nil {
|
|
log.Printf("Not file: %s", err)
|
|
return err
|
|
}
|
|
|
|
filenamesplit := strings.Split(file, "/")
|
|
filename := fmt.Sprintf("%s%s", extra, filenamesplit[len(filenamesplit)-1])
|
|
//log.Printf("Filename: %s", filename)
|
|
tarHeader := &tar.Header{
|
|
Name: filename,
|
|
Size: int64(len(readFile)),
|
|
}
|
|
|
|
//Writes the header described for the TAR file
|
|
err = tw.WriteHeader(tarHeader)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Writes the dockerfile data to the TAR file
|
|
_, err = tw.Write(readFile)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
})
|
|
}
|
|
|
|
// Custom TAR builder in memory for Docker images
|
|
func getParsedTarMemory(fs billy.Filesystem, tw *tar.Writer, baseDir, extra string) error {
|
|
// This one has to use baseDir + Extra
|
|
newBase := fmt.Sprintf("%s%s", baseDir, extra)
|
|
dir, err := fs.ReadDir(newBase)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
for _, file := range dir {
|
|
// Folder?
|
|
switch mode := file.Mode(); {
|
|
case mode.IsDir():
|
|
filename := file.Name()
|
|
filenamesplit := strings.Split(filename, "/")
|
|
|
|
tmpExtra := fmt.Sprintf("%s%s/", extra, filenamesplit[len(filenamesplit)-1])
|
|
//log.Printf("EXTRA: %s", tmpExtra)
|
|
err = getParsedTarMemory(fs, tw, baseDir, tmpExtra)
|
|
if err != nil {
|
|
log.Printf("Directory parse issue: %s", err)
|
|
return err
|
|
}
|
|
case mode.IsRegular():
|
|
filenamesplit := strings.Split(file.Name(), "/")
|
|
filename := fmt.Sprintf("%s%s", extra, filenamesplit[len(filenamesplit)-1])
|
|
// Newbase
|
|
path := fmt.Sprintf("%s%s", newBase, file.Name())
|
|
|
|
fileReader, err := fs.Open(path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
//log.Printf("FILENAME: %s", filename)
|
|
readFile, err := ioutil.ReadAll(fileReader)
|
|
if err != nil {
|
|
log.Printf("Not file: %s", err)
|
|
return err
|
|
}
|
|
|
|
// Fixes issues with older versions of Docker and reference formats
|
|
// Specific to Shuffle rn. Could expand.
|
|
// FIXME: Seems like the issue was with multi-stage builds
|
|
/*
|
|
if filename == "Dockerfile" {
|
|
log.Printf("Should search and replace in readfile.")
|
|
|
|
referenceCheck := "FROM frikky/shuffle:"
|
|
if strings.Contains(string(readFile), referenceCheck) {
|
|
log.Printf("SHOULD SEARCH & REPLACE!")
|
|
newReference := fmt.Sprintf("FROM registry.hub.docker.com/frikky/shuffle:")
|
|
readFile = []byte(strings.Replace(string(readFile), referenceCheck, newReference, -1))
|
|
}
|
|
}
|
|
*/
|
|
|
|
//log.Printf("Filename: %s", filename)
|
|
// FIXME - might need the folder from EXTRA here
|
|
// Name has to be e.g. just "requirements.txt"
|
|
tarHeader := &tar.Header{
|
|
Name: filename,
|
|
Size: int64(len(readFile)),
|
|
}
|
|
|
|
//Writes the header described for the TAR file
|
|
err = tw.WriteHeader(tarHeader)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Writes the dockerfile data to the TAR file
|
|
_, err = tw.Write(readFile)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
/*
|
|
// Fixes App SDK issues.. meh
|
|
func fixTags(tags []string) []string {
|
|
checkTag := "frikky/shuffle"
|
|
newTags := []string{}
|
|
for _, tag := range tags {
|
|
if strings.HasPrefix(tag, checkTags) {
|
|
newTags.append(newTags, fmt.Sprintf("registry.hub.docker.com/%s", tag))
|
|
}
|
|
|
|
newTags.append(tag)
|
|
}
|
|
}
|
|
*/
|
|
|
|
// Custom Docker image builder wrapper in memory
|
|
func buildImageMemory(fs billy.Filesystem, tags []string, dockerfileFolder string, downloadIfFail bool) error {
|
|
ctx := context.Background()
|
|
client, err := client.NewEnvClient()
|
|
if err != nil {
|
|
log.Printf("Unable to create docker client: %s", err)
|
|
return err
|
|
}
|
|
|
|
buf := new(bytes.Buffer)
|
|
tw := tar.NewWriter(buf)
|
|
defer tw.Close()
|
|
|
|
log.Printf("[INFO] Setting up memory build structure for folder: %s", dockerfileFolder)
|
|
err = getParsedTarMemory(fs, tw, dockerfileFolder, "")
|
|
if err != nil {
|
|
log.Printf("Tar issue: %s", err)
|
|
return err
|
|
}
|
|
|
|
dockerFileTarReader := bytes.NewReader(buf.Bytes())
|
|
|
|
// Dockerfile is inside the TAR itself. Not local context
|
|
// docker build --build-arg http_proxy=http://my.proxy.url
|
|
// Attempt at setting name according to #359: https://github.com/frikky/Shuffle/issues/359
|
|
labels := map[string]string{}
|
|
//target := ""
|
|
//if len(tags) > 0 {
|
|
// if strings.Contains(tags[0], ":") {
|
|
// version := strings.Split(tags[0], ":")
|
|
// if len(version) == 2 {
|
|
// target = fmt.Sprintf("shuffle-build-%s", version[1])
|
|
// tags = append(tags, target)
|
|
// labels["name"] = target
|
|
// }
|
|
// }
|
|
//}
|
|
|
|
buildOptions := types.ImageBuildOptions{
|
|
Remove: true,
|
|
Tags: tags,
|
|
BuildArgs: map[string]*string{},
|
|
Labels: labels,
|
|
}
|
|
// NetworkMode: "host",
|
|
|
|
httpProxy := os.Getenv("HTTP_PROXY")
|
|
if len(httpProxy) > 0 {
|
|
buildOptions.BuildArgs["HTTP_PROXY"] = &httpProxy
|
|
}
|
|
httpsProxy := os.Getenv("HTTPS_PROXY")
|
|
if len(httpProxy) > 0 {
|
|
buildOptions.BuildArgs["HTTPS_PROXY"] = &httpsProxy
|
|
}
|
|
|
|
// Build the actual image
|
|
log.Printf(`[INFO] Building %s with proxy "%s". Tags: "%s". This may take up to a few minutes.`, dockerfileFolder, httpsProxy, strings.Join(tags, ","))
|
|
imageBuildResponse, err := client.ImageBuild(
|
|
ctx,
|
|
dockerFileTarReader,
|
|
buildOptions,
|
|
)
|
|
|
|
//log.Printf("RESPONSE: %#v", imageBuildResponse)
|
|
//log.Printf("Response: %#v", imageBuildResponse.Body)
|
|
//log.Printf("[DEBUG] IMAGERESPONSE: %#v", imageBuildResponse.Body)
|
|
|
|
if imageBuildResponse.Body != nil {
|
|
defer imageBuildResponse.Body.Close()
|
|
buildBuf := new(strings.Builder)
|
|
_, newerr := io.Copy(buildBuf, imageBuildResponse.Body)
|
|
if newerr != nil {
|
|
log.Printf("[WARNING] Failed reading Docker build STDOUT: %s", newerr)
|
|
} else {
|
|
log.Printf("[INFO] STRING: %s", buildBuf.String())
|
|
if strings.Contains(buildBuf.String(), "errorDetail") {
|
|
log.Printf("[ERROR] Docker build:\n%s\nERROR ABOVE: Trying to pull tags from: %s", buildBuf.String(), strings.Join(tags, "\n"))
|
|
|
|
// Handles pulling of the same image if applicable
|
|
// This fixes some issues with older versions of Docker which can't build
|
|
// on their own ( <17.05 )
|
|
pullOptions := types.ImagePullOptions{}
|
|
downloaded := false
|
|
for _, image := range tags {
|
|
// Is this ok? Not sure. Tags shouldn't be controlled here prolly.
|
|
image = strings.ToLower(image)
|
|
|
|
newImage := fmt.Sprintf("%s/%s", registryName, image)
|
|
log.Printf("[INFO] Pulling image %s", newImage)
|
|
reader, err := client.ImagePull(ctx, newImage, pullOptions)
|
|
if err != nil {
|
|
log.Printf("[ERROR] Failed getting image %s: %s", newImage, err)
|
|
continue
|
|
}
|
|
|
|
// Attempt to retag the image to not contain registry...
|
|
|
|
//newBuf := buildBuf
|
|
downloaded = true
|
|
io.Copy(os.Stdout, reader)
|
|
log.Printf("[INFO] Successfully downloaded and built %s", newImage)
|
|
}
|
|
|
|
if !downloaded {
|
|
|
|
return errors.New(fmt.Sprintf("Failed to build / download images %s", strings.Join(tags, ",")))
|
|
}
|
|
//baseDockerName
|
|
}
|
|
}
|
|
}
|
|
|
|
if err != nil {
|
|
// Read the STDOUT from the build process
|
|
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func getK8sClient() (*kubernetes.Clientset, error) {
|
|
config, err := rest.InClusterConfig()
|
|
if err != nil {
|
|
return nil, fmt.Errorf("[ERROR] failed to get in-cluster config: %v", err)
|
|
}
|
|
|
|
clientset, err := kubernetes.NewForConfig(config)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("[ERROR] failed to create Kubernetes client: %v", err)
|
|
}
|
|
|
|
return clientset, nil
|
|
}
|
|
|
|
func deleteJob(client *kubernetes.Clientset, jobName, namespace string) error {
|
|
deletePolicy := metav1.DeletePropagationForeground
|
|
return client.BatchV1().Jobs(namespace).Delete(context.TODO(), jobName, metav1.DeleteOptions{
|
|
PropagationPolicy: &deletePolicy,
|
|
})
|
|
}
|
|
|
|
func buildImage(tags []string, dockerfileFolder string) error {
|
|
|
|
isKubernetes := false
|
|
if os.Getenv("IS_KUBERNETES") == "true" {
|
|
isKubernetes = true
|
|
}
|
|
|
|
if isKubernetes {
|
|
// log.Printf("K8S ###################")
|
|
// log.Print("dockerfileFolder: ", dockerfileFolder)
|
|
// log.Print("tags: ", tags)
|
|
// log.Print("only tag: ", tags[1])
|
|
|
|
registryName := ""
|
|
if len(os.Getenv("REGISTRY_URL")) > 0 {
|
|
registryName = os.Getenv("REGISTRY_URL")
|
|
}
|
|
|
|
log.Printf("[INFO] registry name: %s", registryName)
|
|
|
|
contextDir := strings.Replace(dockerfileFolder, "Dockerfile", "", -1)
|
|
contextDir = "/app/" + contextDir
|
|
log.Print("contextDir: ", contextDir)
|
|
dockerFile := "./Dockerfile"
|
|
|
|
client, err := getK8sClient()
|
|
if err != nil {
|
|
fmt.Printf("Unable to authencticate : %v\n", err)
|
|
return err
|
|
}
|
|
|
|
BackendPodLabel := "io.kompose.service=backend"
|
|
|
|
backendPodList, podListErr := client.CoreV1().Pods("shuffle").List(context.TODO(), metav1.ListOptions{
|
|
LabelSelector: BackendPodLabel,
|
|
})
|
|
|
|
if podListErr != nil || len(backendPodList.Items) == 0 {
|
|
fmt.Println("Error getting backend pod or no pod found:", podListErr)
|
|
return podListErr
|
|
}
|
|
|
|
backendNodeName := backendPodList.Items[0].Spec.NodeName
|
|
log.Printf("[INFO] Backend running on: %s", backendNodeName)
|
|
|
|
job := &batchv1.Job{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "shuffle-app-builder",
|
|
},
|
|
Spec: batchv1.JobSpec{
|
|
Template: corev1.PodTemplateSpec{
|
|
Spec: corev1.PodSpec{
|
|
Containers: []corev1.Container{
|
|
{
|
|
Name: "kaniko",
|
|
Image: "gcr.io/kaniko-project/executor:latest",
|
|
Args: []string{
|
|
"--verbosity=debug",
|
|
"--dockerfile=" + dockerFile,
|
|
"--context=dir://" + contextDir,
|
|
"--skip-tls-verify",
|
|
"--destination=" + registryName + "/" + tags[1],
|
|
},
|
|
VolumeMounts: []corev1.VolumeMount{
|
|
{
|
|
Name: "kaniko-workspace",
|
|
MountPath: "/app/generated",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
NodeSelector: map[string]string{
|
|
"node": backendNodeName,
|
|
},
|
|
RestartPolicy: corev1.RestartPolicyNever,
|
|
Volumes: []corev1.Volume{
|
|
{
|
|
Name: "kaniko-workspace",
|
|
VolumeSource: corev1.VolumeSource{
|
|
PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
|
|
ClaimName: "backend-apps-claim",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
createdJob, err := client.BatchV1().Jobs("shuffle").Create(context.TODO(), job, metav1.CreateOptions{})
|
|
if err != nil {
|
|
log.Printf("Failed to start image builder job: %s", err)
|
|
return err
|
|
}
|
|
|
|
timeout := time.After(5 * time.Minute)
|
|
tick := time.Tick(5 * time.Second)
|
|
|
|
for {
|
|
select {
|
|
case <-timeout:
|
|
return fmt.Errorf("job didn't complete within the expected time")
|
|
case <-tick:
|
|
currentJob, err := client.BatchV1().Jobs("shuffle").Get(context.TODO(), createdJob.Name, metav1.GetOptions{})
|
|
if err != nil {
|
|
return fmt.Errorf("[ERROR] failed to fetch %s status: %v", createdJob.Name, err)
|
|
}
|
|
|
|
if currentJob.Status.Succeeded > 0 {
|
|
log.Printf("[INFO] Job %s completed successfully!", createdJob.Name)
|
|
log.Printf("[INFO] Cleaning up the job %s", createdJob.Name)
|
|
err := deleteJob(client, createdJob.Name, "shuffle")
|
|
if err != nil {
|
|
return fmt.Errorf("[ERROR] failed deleting job %s with error: %s", createdJob.Name, err)
|
|
}
|
|
log.Println("Job deleted successfully!")
|
|
return nil
|
|
} else if currentJob.Status.Failed > 0 {
|
|
log.Printf("[ERROR] %s job failed with error: %s", createdJob.Name, err)
|
|
err := deleteJob(client, createdJob.Name, "shuffle")
|
|
if err != nil {
|
|
return fmt.Errorf("[ERROR] failed deleting job %s with error: %s", createdJob.Name, err)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
} else {
|
|
|
|
ctx := context.Background()
|
|
client, err := client.NewEnvClient()
|
|
if err != nil {
|
|
log.Printf("Unable to create docker client: %s", err)
|
|
return err
|
|
}
|
|
|
|
log.Printf("[INFO] Docker Tags: %s", tags)
|
|
dockerfileSplit := strings.Split(dockerfileFolder, "/")
|
|
|
|
// Create a buffer
|
|
buf := new(bytes.Buffer)
|
|
tw := tar.NewWriter(buf)
|
|
defer tw.Close()
|
|
baseDir := strings.Join(dockerfileSplit[0:len(dockerfileSplit)-1], "/")
|
|
|
|
// Builds the entire folder into buf
|
|
err = getParsedTar(tw, baseDir, "")
|
|
if err != nil {
|
|
log.Printf("Tar issue: %s", err)
|
|
}
|
|
|
|
dockerFileTarReader := bytes.NewReader(buf.Bytes())
|
|
buildOptions := types.ImageBuildOptions{
|
|
Remove: true,
|
|
Tags: tags,
|
|
BuildArgs: map[string]*string{},
|
|
}
|
|
//NetworkMode: "host",
|
|
|
|
httpProxy := os.Getenv("HTTP_PROXY")
|
|
if len(httpProxy) > 0 {
|
|
buildOptions.BuildArgs["HTTP_PROXY"] = &httpProxy
|
|
}
|
|
httpsProxy := os.Getenv("HTTPS_PROXY")
|
|
if len(httpProxy) > 0 {
|
|
buildOptions.BuildArgs["https_proxy"] = &httpsProxy
|
|
}
|
|
|
|
// Build the actual image
|
|
imageBuildResponse, err := client.ImageBuild(
|
|
ctx,
|
|
dockerFileTarReader,
|
|
buildOptions,
|
|
)
|
|
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Read the STDOUT from the build process
|
|
defer imageBuildResponse.Body.Close()
|
|
buildBuf := new(strings.Builder)
|
|
_, err = io.Copy(buildBuf, imageBuildResponse.Body)
|
|
if err != nil {
|
|
return err
|
|
} else {
|
|
if strings.Contains(buildBuf.String(), "errorDetail") {
|
|
log.Printf("[ERROR] Docker build:\n%s\nERROR ABOVE: Trying to pull tags from: %s", buildBuf.String(), strings.Join(tags, "\n"))
|
|
return errors.New(fmt.Sprintf("Failed building %s. Check backend logs for details. Most likely means you have an old version of Docker.", strings.Join(tags, ",")))
|
|
}
|
|
}
|
|
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Checks if an image exists
|
|
func imageCheckBuilder(images []string) error {
|
|
//log.Printf("[FIXME] ImageNames to check: %#v", images)
|
|
return nil
|
|
|
|
ctx := context.Background()
|
|
client, err := client.NewEnvClient()
|
|
if err != nil {
|
|
log.Printf("Unable to create docker client: %s", err)
|
|
return err
|
|
}
|
|
|
|
allImages, err := client.ImageList(ctx, types.ImageListOptions{
|
|
All: true,
|
|
})
|
|
|
|
if err != nil {
|
|
log.Printf("[ERROR] Failed creating imagelist: %s", err)
|
|
return err
|
|
}
|
|
|
|
filteredImages := []types.ImageSummary{}
|
|
for _, image := range allImages {
|
|
found := false
|
|
for _, repoTag := range image.RepoTags {
|
|
if strings.Contains(repoTag, baseDockerName) {
|
|
found = true
|
|
break
|
|
}
|
|
}
|
|
|
|
if found {
|
|
filteredImages = append(filteredImages, image)
|
|
}
|
|
}
|
|
|
|
// FIXME: Continue fixing apps here
|
|
// https://github.com/frikky/Shuffle/issues/135
|
|
// 1. Find if app exists
|
|
// 2. Create app if it doesn't
|
|
//log.Printf("Apps: %#v", filteredImages)
|
|
|
|
return nil
|
|
}
|
|
|
|
// https://stackoverflow.com/questions/23935141/how-to-copy-docker-images-from-one-host-to-another-without-using-a-repository
|
|
func getDockerImage(resp http.ResponseWriter, request *http.Request) {
|
|
cors := shuffle.HandleCors(resp, request)
|
|
if cors {
|
|
return
|
|
}
|
|
|
|
// Just here to verify that the user is logged in
|
|
//_, err := shuffle.HandleApiAuthentication(resp, request)
|
|
//if err != nil {
|
|
// log.Printf("[WARNING] Api authentication failed in DOWNLOAD IMAGE: %s", err)
|
|
// resp.WriteHeader(401)
|
|
// resp.Write([]byte(`{"success": false}`))
|
|
// return
|
|
//}
|
|
|
|
body, err := ioutil.ReadAll(request.Body)
|
|
if err != nil {
|
|
resp.WriteHeader(401)
|
|
resp.Write([]byte(`{"success": false, "reason": "Failed reading body"}`))
|
|
return
|
|
}
|
|
|
|
// This has to be done in a weird way because Datastore doesn't
|
|
// support map[string]interface and similar (openapi3.Swagger)
|
|
var version shuffle.DockerRequestCheck
|
|
err = json.Unmarshal(body, &version)
|
|
if err != nil {
|
|
resp.WriteHeader(422)
|
|
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed JSON marshalling: %s"}`, err)))
|
|
return
|
|
}
|
|
|
|
//log.Printf("[DEBUG] Image to load: %s", version.Name)
|
|
dockercli, err := client.NewEnvClient()
|
|
if err != nil {
|
|
log.Printf("[WARNING] Unable to create docker client: %s", err)
|
|
resp.WriteHeader(422)
|
|
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed JSON marshalling: %s"}`, err)))
|
|
return
|
|
}
|
|
|
|
ctx := context.Background()
|
|
images, err := dockercli.ImageList(ctx, types.ImageListOptions{
|
|
All: true,
|
|
})
|
|
|
|
img := types.ImageSummary{}
|
|
tagFound := ""
|
|
|
|
img2 := types.ImageSummary{}
|
|
tagFound2 := ""
|
|
|
|
alternativeNameSplit := strings.Split(version.Name, "/")
|
|
alternativeName := version.Name
|
|
if len(alternativeNameSplit) == 3 {
|
|
alternativeName = strings.Join(alternativeNameSplit[1:3], "/")
|
|
}
|
|
|
|
log.Printf("[INFO] Trying to download image: %s. Alt: %s", version.Name, alternativeName)
|
|
|
|
for _, image := range images {
|
|
for _, tag := range image.RepoTags {
|
|
//log.Printf("[DEBUG] Tag: %s", tag)
|
|
if strings.ToLower(tag) == strings.ToLower(version.Name) {
|
|
img = image
|
|
tagFound = tag
|
|
break
|
|
}
|
|
|
|
if strings.ToLower(tag) == strings.ToLower(alternativeName) {
|
|
img2 = image
|
|
tagFound2 = tag
|
|
}
|
|
}
|
|
}
|
|
|
|
pullOptions := types.ImagePullOptions{}
|
|
if len(img.ID) == 0 {
|
|
_, err := dockercli.ImagePull(context.Background(), version.Name, pullOptions)
|
|
if err == nil {
|
|
tagFound = version.Name
|
|
img.ID = version.Name
|
|
img2.ID = version.Name
|
|
|
|
dockercli.ImageTag(ctx, version.Name, alternativeName)
|
|
}
|
|
}
|
|
|
|
if len(img2.ID) == 0 {
|
|
_, err := dockercli.ImagePull(context.Background(), alternativeName, pullOptions)
|
|
if err == nil {
|
|
tagFound = alternativeName
|
|
img.ID = alternativeName
|
|
img2.ID = alternativeName
|
|
|
|
dockercli.ImageTag(ctx, alternativeName, version.Name)
|
|
}
|
|
}
|
|
|
|
// REBUILDS THE APP
|
|
if len(img.ID) == 0 {
|
|
if len(img2.ID) == 0 {
|
|
workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 0, 0)
|
|
log.Printf("[INFO] Getting workflowapps for a rebuild. Got %d with err %#v", len(workflowapps), err)
|
|
if err == nil {
|
|
imageName := ""
|
|
imageVersion := ""
|
|
newNameSplit := strings.Split(version.Name, ":")
|
|
if len(newNameSplit) == 2 {
|
|
//log.Printf("[DEBUG] Found name %#v", newNameSplit)
|
|
|
|
findVersionSplit := strings.Split(newNameSplit[1], "_")
|
|
//log.Printf("[DEBUG] Found another split %#v", findVersionSplit)
|
|
if len(findVersionSplit) == 2 {
|
|
imageVersion = findVersionSplit[len(findVersionSplit)-1]
|
|
imageName = findVersionSplit[0]
|
|
} else if len(findVersionSplit) >= 2 {
|
|
imageVersion = findVersionSplit[len(findVersionSplit)-1]
|
|
imageName = strings.Join(findVersionSplit[0:len(findVersionSplit)-1], "_")
|
|
} else {
|
|
log.Printf("[DEBUG] Couldn't parse appname & version for %#v", findVersionSplit)
|
|
}
|
|
}
|
|
|
|
if len(imageName) > 0 && len(imageVersion) > 0 {
|
|
foundApp := shuffle.WorkflowApp{}
|
|
imageName = strings.ToLower(imageName)
|
|
imageVersion = strings.ToLower(imageVersion)
|
|
log.Printf("[DEBUG] Docker Looking for appname %s with version %s", imageName, imageVersion)
|
|
|
|
for _, app := range workflowapps {
|
|
if strings.ToLower(strings.Replace(app.Name, " ", "_", -1)) == imageName && app.AppVersion == imageVersion {
|
|
if app.Generated {
|
|
log.Printf("[DEBUG] Found matching app %s:%s - %s", imageName, imageVersion, app.ID)
|
|
foundApp = app
|
|
break
|
|
} else {
|
|
log.Printf("[WARNING] Trying to rebuild app that isn't generated - not allowed. Looking further.")
|
|
}
|
|
|
|
//break
|
|
}
|
|
}
|
|
|
|
if len(foundApp.ID) > 0 {
|
|
openApiApp, err := shuffle.GetOpenApiDatastore(ctx, foundApp.ID)
|
|
if err != nil {
|
|
log.Printf("[ERROR] Failed getting OpenAPI app %s to database: %s", foundApp.ID, err)
|
|
} else {
|
|
log.Printf("[DEBUG] Found OpenAPI app for %s as generated - now building!", version.Name)
|
|
user := shuffle.User{}
|
|
|
|
//img = version.Name
|
|
if len(alternativeName) > 0 {
|
|
tagFound = alternativeName
|
|
} else {
|
|
tagFound = version.Name
|
|
}
|
|
|
|
buildSwaggerApp(resp, []byte(openApiApp.Body), user, false)
|
|
}
|
|
}
|
|
}
|
|
} else {
|
|
log.Printf("[WARNING] Couldn't find an image with registry name %s and %s", version.Name, alternativeName)
|
|
resp.WriteHeader(401)
|
|
resp.Write([]byte(fmt.Sprintf(`{"success": false, "message": "Couldn't find image %s"}`, version.Name)))
|
|
return
|
|
}
|
|
}
|
|
|
|
if len(tagFound) == 0 && len(tagFound2) > 0 {
|
|
img = img2
|
|
tagFound = tagFound2
|
|
}
|
|
}
|
|
|
|
//log.Printf("[INFO] Img found (%s): %#v", tagFound, img)
|
|
//log.Printf("[INFO] Img found to be downloaded by client: %s", tagFound)
|
|
|
|
newClient, err := newdockerclient.NewClientFromEnv()
|
|
if err != nil {
|
|
log.Printf("[ERROR] Failed setting up docker env: %#v", newClient)
|
|
resp.WriteHeader(401)
|
|
resp.Write([]byte(fmt.Sprintf(`{"success": false, "message": "Couldn't make docker client"}`)))
|
|
return
|
|
}
|
|
|
|
////https://github.com/fsouza/go-dockerclient/issues/600
|
|
//defer fileReader.Close()
|
|
opts := newdockerclient.ExportImageOptions{
|
|
Name: tagFound,
|
|
OutputStream: resp,
|
|
}
|
|
|
|
if err := newClient.ExportImage(opts); err != nil {
|
|
log.Printf("[ERROR] FAILED to save image to file: %s", err)
|
|
resp.WriteHeader(401)
|
|
resp.Write([]byte(fmt.Sprintf(`{"success": false, "message": "Couldn't export image"}`)))
|
|
return
|
|
}
|
|
|
|
//resp.WriteHeader(200)
|
|
}
|
|
|
|
// Downloads and activates an app from shuffler.io if possible
|
|
func handleRemoteDownloadApp(resp http.ResponseWriter, ctx context.Context, user shuffle.User, appId string) {
|
|
url := fmt.Sprintf("https://shuffler.io/api/v1/apps/%s/config", appId)
|
|
log.Printf("Downloading API from %s", url)
|
|
req, err := http.NewRequest(
|
|
"GET",
|
|
url,
|
|
nil,
|
|
)
|
|
|
|
if err != nil {
|
|
log.Printf("[ERROR] Failed auto-downloading app %s: %s", appId, err)
|
|
resp.WriteHeader(401)
|
|
resp.Write([]byte(`{"success": false, "reason": "App doesn't exist"}`))
|
|
return
|
|
}
|
|
|
|
httpClient := &http.Client{}
|
|
newresp, err := httpClient.Do(req)
|
|
if err != nil {
|
|
log.Printf("[ERROR] Failed running auto-download request for %s: %s", appId, err)
|
|
resp.WriteHeader(401)
|
|
resp.Write([]byte(`{"success": false, "reason": "App doesn't exist"}`))
|
|
return
|
|
}
|
|
|
|
respBody, err := ioutil.ReadAll(newresp.Body)
|
|
if err != nil {
|
|
log.Printf("[ERROR] Failed setting respbody for workflow download: %s", err)
|
|
resp.WriteHeader(401)
|
|
resp.Write([]byte(`{"success": false, "reason": "App doesn't exist"}`))
|
|
return
|
|
}
|
|
|
|
if len(respBody) > 0 {
|
|
type tmpapp struct {
|
|
Success bool `json:"success"`
|
|
OpenAPI string `json:"openapi"`
|
|
}
|
|
|
|
app := tmpapp{}
|
|
err := json.Unmarshal(respBody, &app)
|
|
if err != nil || app.Success == false || len(app.OpenAPI) == 0 {
|
|
log.Printf("[ERROR] Failed app unmarshal during auto-download. Success: %#v. Applength: %d: %s", app.Success, len(app.OpenAPI), err)
|
|
resp.WriteHeader(401)
|
|
resp.Write([]byte(`{"success": false, "reason": "App doesn't exist"}`))
|
|
return
|
|
}
|
|
|
|
key, err := base64.StdEncoding.DecodeString(app.OpenAPI)
|
|
if err != nil {
|
|
log.Printf("[ERROR] Failed auto-setting OpenAPI app: %s", err)
|
|
resp.WriteHeader(401)
|
|
resp.Write([]byte(`{"success": false, "reason": "App doesn't exist"}`))
|
|
return
|
|
}
|
|
|
|
cacheKey := fmt.Sprintf("workflowapps-sorted-100")
|
|
shuffle.DeleteCache(ctx, cacheKey)
|
|
cacheKey = fmt.Sprintf("workflowapps-sorted-500")
|
|
shuffle.DeleteCache(ctx, cacheKey)
|
|
cacheKey = fmt.Sprintf("workflowapps-sorted-1000")
|
|
shuffle.DeleteCache(ctx, cacheKey)
|
|
|
|
newapp := shuffle.ParsedOpenApi{}
|
|
err = json.Unmarshal(key, &newapp)
|
|
if err != nil {
|
|
log.Printf("[ERROR] Failed openapi unmarshal during auto-download: %s", app.Success, len(app.OpenAPI), err)
|
|
resp.WriteHeader(401)
|
|
resp.Write([]byte(`{"success": false, "reason": "App doesn't exist"}`))
|
|
return
|
|
}
|
|
|
|
err = json.Unmarshal(key, &newapp)
|
|
if err != nil {
|
|
log.Printf("[ERROR] Failed openapi unmarshal during auto-download: %s", app.Success, len(app.OpenAPI), err)
|
|
resp.WriteHeader(401)
|
|
resp.Write([]byte(`{"success": false, "reason": "App doesn't exist"}`))
|
|
return
|
|
}
|
|
|
|
buildSwaggerApp(resp, []byte(newapp.Body), user, true)
|
|
return
|
|
}
|
|
}
|
|
|
|
func activateWorkflowAppDocker(resp http.ResponseWriter, request *http.Request) {
|
|
cors := shuffle.HandleCors(resp, request)
|
|
if cors {
|
|
return
|
|
}
|
|
|
|
user, err := shuffle.HandleApiAuthentication(resp, request)
|
|
if err != nil {
|
|
log.Printf("[WARNING] Api authentication failed in get active apps: %s", err)
|
|
resp.WriteHeader(401)
|
|
resp.Write([]byte(`{"success": false}`))
|
|
return
|
|
}
|
|
|
|
if user.Role == "org-reader" {
|
|
log.Printf("[WARNING] Org-reader doesn't have access to activate workflow app (shared): %s (%s)", user.Username, user.Id)
|
|
resp.WriteHeader(401)
|
|
resp.Write([]byte(`{"success": false, "reason": "Read only user"}`))
|
|
return
|
|
}
|
|
|
|
ctx := context.Background()
|
|
location := strings.Split(request.URL.String(), "/")
|
|
var fileId string
|
|
if location[1] == "api" {
|
|
if len(location) <= 4 {
|
|
resp.WriteHeader(401)
|
|
resp.Write([]byte(`{"success": false}`))
|
|
return
|
|
}
|
|
|
|
fileId = location[4]
|
|
}
|
|
|
|
app, err := shuffle.GetApp(ctx, fileId, user, false)
|
|
if err != nil {
|
|
appName := request.URL.Query().Get("app_name")
|
|
appVersion := request.URL.Query().Get("app_version")
|
|
|
|
if len(appName) > 0 && len(appVersion) > 0 {
|
|
apps, err := shuffle.FindWorkflowAppByName(ctx, appName)
|
|
//log.Printf("[INFO] Found %d apps for %s", len(apps), appName)
|
|
if err != nil || len(apps) == 0 {
|
|
log.Printf("[WARNING] Error getting app %s (app config). Starting remote download.: %s", appName, err)
|
|
|
|
handleRemoteDownloadApp(resp, ctx, user, fileId)
|
|
return
|
|
}
|
|
|
|
selectedApp := shuffle.WorkflowApp{}
|
|
for _, app := range apps {
|
|
if !app.Sharing && !app.Public {
|
|
continue
|
|
}
|
|
|
|
if app.Name == appName {
|
|
selectedApp = app
|
|
}
|
|
|
|
if app.Name == appName && app.AppVersion == appVersion {
|
|
selectedApp = app
|
|
}
|
|
}
|
|
|
|
app = &selectedApp
|
|
} else {
|
|
log.Printf("[WARNING] Error getting app with ID %s (app config): %s. Starting remote download(2)", fileId, err)
|
|
handleRemoteDownloadApp(resp, ctx, user, fileId)
|
|
return
|
|
//resp.WriteHeader(401)
|
|
//resp.Write([]byte(`{"success": false, "reason": "App doesn't exist"}`))
|
|
//return
|
|
}
|
|
}
|
|
|
|
// Just making sure it's being built properly
|
|
if app == nil {
|
|
log.Printf("[WARNING] App is nil. This shouldn't happen. Starting remote download(3)")
|
|
handleRemoteDownloadApp(resp, ctx, user, fileId)
|
|
return
|
|
}
|
|
|
|
// Check the app.. hmm
|
|
openApiApp, err := shuffle.GetOpenApiDatastore(ctx, app.ID)
|
|
if err != nil {
|
|
log.Printf("[WARNING] Error getting app %s (openapi config): %s", app.ID, err)
|
|
resp.WriteHeader(401)
|
|
resp.Write([]byte(`{"success": false, "reason": "Couldn't find app OpenAPI"}`))
|
|
return
|
|
}
|
|
|
|
log.Printf("[INFO] User %s (%s) is activating %s. Public: %t, Shared: %t", user.Username, user.Id, app.Name, app.Public, app.Sharing)
|
|
buildSwaggerApp(resp, []byte(openApiApp.Body), user, true)
|
|
|
|
//app.Active = true
|
|
//app.Generated = true
|
|
//app, err := shuffle.SetApp(ctx, app)
|
|
|
|
//resp.WriteHeader(200)
|
|
//resp.Write([]byte(`{"success": true}`))
|
|
}
|