Tips-Of-Mine/soc-fortress
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

first sync
Some checks failed
Deployment Verification / deploy-and-test (push) Failing after 29s
Details

Browse Source
This commit is contained in:
Hubert Cornet
2025-03-04 07:59:21 +01:00
parent 9cdcf486b6
commit 506716e703
1450 changed files with 577316 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files

499
shuffle/backend/app_gen/openapi/testGCP.go Normal file
View File

@ -0,0 +1,499 @@
package main
/*
Code used to generate apps from OpenAPI JSON data
Any function ending with GCP doesn't use local fileIO, but rather
google cloud storage, and also has a normal filesystem version of the
same code (doesn't required client as first argument).
This code is used in the backend to generate apps on the fly for users.
All new code is appended to backend/go-app/codegen.go
*/
import (
"context"
"crypto/md5"
"encoding/hex"
"errors"
"fmt"
"io"
"io/ioutil"
"log"
"os"
"strings"
"cloud.google.com/go/storage"
"github.com/getkin/kin-openapi/openapi3"
"gopkg.in/yaml.v2"
)
var bucketName = "shuffler.appspot.com"
type WorkflowApp struct {
Name string `json:"name" yaml:"name" required:true datastore:"name"`
IsValid bool `json:"is_valid" yaml:"is_valid" required:true datastore:"is_valid"`
ID string `json:"id" yaml:"id,omitempty" required:false datastore:"id"`
Link string `json:"link" yaml:"link" required:false datastore:"link,noindex"`
AppVersion string `json:"app_version" yaml:"app_version" required:true datastore:"app_version"`
Description string `json:"description" datastore:"description" required:false yaml:"description"`
Environment string `json:"environment" datastore:"environment" required:true yaml:"environment"`
SmallImage string `json:"small_image" datastore:"small_image,noindex" required:false yaml:"small_image"`
LargeImage string `json:"large_image" datastore:"large_image,noindex" yaml:"large_image" required:false`
ContactInfo struct {
Name string `json:"name" datastore:"name" yaml:"name"`
Url string `json:"url" datastore:"url" yaml:"url"`
} `json:"contact_info" datastore:"contact_info" yaml:"contact_info" required:false`
Actions []WorkflowAppAction `json:"actions" yaml:"actions" required:true datastore:"actions"`
Authentication Authentication `json:"authentication" yaml:"authentication" required:false datastore:"authentication"`
}
type AuthenticationParams struct {
Description string `json:"description" datastore:"description" yaml:"description"`
ID string `json:"id" datastore:"id" yaml:"id"`
Name string `json:"name" datastore:"name" yaml:"name"`
Example string `json:"example" datastore:"example" yaml:"example"s`
Value string `json:"value,omitempty" datastore:"value" yaml:"value"`
Multiline bool `json:"multiline" datastore:"multiline" yaml:"multiline"`
Required bool `json:"required" datastore:"required" yaml:"required"`
}
type Authentication struct {
Required bool `json:"required" datastore:"required" yaml:"required" `
Parameters []AuthenticationParams `json:"parameters" datastore:"parameters" yaml:"parameters"`
}
type AuthenticationStore struct {
Key string `json:"key" datastore:"key"`
Value string `json:"value" datastore:"value"`
}
type WorkflowAppActionParameter struct {
Description string `json:"description" datastore:"description" yaml:"description"`
ID string `json:"id" datastore:"id" yaml:"id,omitempty"`
Name string `json:"name" datastore:"name" yaml:"name"`
Example string `json:"example" datastore:"example" yaml:"example"`
Value string `json:"value" datastore:"value" yaml:"value,omitempty"`
Multiline bool `json:"multiline" datastore:"multiline" yaml:"multiline"`
ActionField string `json:"action_field" datastore:"action_field" yaml:"actionfield,omitempty"`
Variant string `json:"variant" datastore:"variant" yaml:"variant,omitempty"`
Required bool `json:"required" datastore:"required" yaml:"required"`
Schema SchemaDefinition `json:"schema" datastore:"schema" yaml:"schema"`
}
type SchemaDefinition struct {
Type string `json:"type" datastore:"type"`
}
type WorkflowAppAction struct {
Description string `json:"description" datastore:"description"`
ID string `json:"id" datastore:"id" yaml:"id,omitempty"`
Name string `json:"name" datastore:"name"`
NodeType string `json:"node_type" datastore:"node_type"`
Environment string `json:"environment" datastore:"environment"`
Authentication []AuthenticationStore `json:"authentication" datastore:"authentication" yaml:"authentication,omitempty"`
Parameters []WorkflowAppActionParameter `json:"parameters" datastore: "parameters"`
Returns struct {
Description string `json:"description" datastore:"returns" yaml:"description,omitempty"`
ID string `json:"id" datastore:"id" yaml:"id,omitempty"`
Schema SchemaDefinition `json:"schema" datastore:"schema" yaml:"schema"`
} `json:"returns" datastore:"returns"`
}
func copyFile(fromfile, tofile string) error {
from, err := os.Open(fromfile)
if err != nil {
return err
}
defer from.Close()
to, err := os.OpenFile(tofile, os.O_RDWR|os.O_CREATE, 0666)
if err != nil {
return err
}
defer to.Close()
_, err = io.Copy(to, from)
if err != nil {
return err
}
return nil
}
func buildStructureGCP(client *storage.Client, swagger *openapi3.Swagger, curHash string) (string, error) {
ctx := context.Background()
// 1. Have baseline in bucket/generated_apps/baseline
// 2. Copy the baseline to a new folder with identifier name
basePath := "generated_apps"
identifier := fmt.Sprintf("%s-%s", swagger.Info.Title, curHash)
appPath := fmt.Sprintf("%s/%s", basePath, identifier)
fileNames := []string{"Dockerfile", "requirements.txt"}
for _, file := range fileNames {
src := client.Bucket(bucketName).Object(fmt.Sprintf("%s/baseline/%s", basePath, file))
dst := client.Bucket(bucketName).Object(fmt.Sprintf("%s/%s", appPath, file))
if _, err := dst.CopierFrom(src).Run(ctx); err != nil {
return "", err
}
}
return appPath, nil
}
// Builds the base structure for the app that we're making
// Returns error if anything goes wrong. This has to work if
// the python code is supposed to be generated
func buildStructure(swagger *openapi3.Swagger, curHash string) (string, error) {
//log.Printf("%#v", swagger)
// adding md5 based on input data to not overwrite earlier data.
generatedPath := "generated"
identifier := fmt.Sprintf("%s-%s", swagger.Info.Title, curHash)
appPath := fmt.Sprintf("%s/%s", generatedPath, identifier)
os.MkdirAll(appPath, os.ModePerm)
os.Mkdir(fmt.Sprintf("%s/src", appPath), os.ModePerm)
err := copyFile("baseline/Dockerfile", fmt.Sprintf("%s/%s", appPath, "Dockerfile"))
if err != nil {
log.Println("Failed to move Dockerfile")
return appPath, err
}
err = copyFile("baseline/requirements.txt", fmt.Sprintf("%s/%s", appPath, "requirements.txt"))
if err != nil {
log.Println("Failed to move requrements.txt")
return appPath, err
}
return appPath, nil
}
func makePythoncode(name, url, method string, parameters, optionalQueries []string) string {
method = strings.ToLower(method)
queryString := ""
queryData := ""
// FIXME - this might break - need to check if ? or & should be set as query
parameterData := ""
if len(optionalQueries) > 0 {
queryString += ", "
for _, query := range optionalQueries {
queryString += fmt.Sprintf("%s=\"\"", query)
queryData += fmt.Sprintf(`
if %s:
url += f"&%s={%s}"`, query, query, query)
}
}
if len(parameters) > 0 {
parameterData = fmt.Sprintf(", %s", strings.Join(parameters, ", "))
}
// FIXME - add checks for query data etc
data := fmt.Sprintf(` async def %s_%s(self%s%s):
url=f"%s"
%s
return requests.%s(url).text
`, name, method, parameterData, queryString, url, queryData, method)
return data
}
func generateYaml(swagger *openapi3.Swagger) (WorkflowApp, []string, error) {
api := WorkflowApp{}
log.Printf("%#v", swagger.Info)
if len(swagger.Info.Title) == 0 {
return WorkflowApp{}, []string{}, errors.New("Swagger.Info.Title can't be empty.")
}
if len(swagger.Servers) == 0 {
return WorkflowApp{}, []string{}, errors.New("Swagger.Servers can't be empty. Add 'servers':[{'url':'hostname.com'}'")
}
api.Name = swagger.Info.Title
api.Description = swagger.Info.Description
api.IsValid = true
api.Link = swagger.Servers[0].URL // host does not exist lol
api.AppVersion = "1.0.0"
api.Environment = "cloud"
api.ID = ""
api.SmallImage = ""
api.LargeImage = ""
// This is the python code to be generated
// Could just as well be go at this point lol
pythonFunctions := []string{}
for actualPath, path := range swagger.Paths {
//log.Printf("%#v", path)
//log.Printf("%#v", actualPath)
// Find the path name and add it to makeCode() param
firstQuery := true
if path.Get != nil {
// What to do with this, hmm
functionName := strings.ReplaceAll(path.Get.Summary, " ", "_")
functionName = strings.ToLower(functionName)
action := WorkflowAppAction{
Description: path.Get.Description,
Name: path.Get.Summary,
NodeType: "action",
Environment: api.Environment,
Parameters: []WorkflowAppActionParameter{},
}
action.Returns.Schema.Type = "string"
baseUrl := fmt.Sprintf("%s%s", api.Link, actualPath)
//log.Println(path.Parameters)
// Parameters: []WorkflowAppActionParameter{},
// FIXME - add data for POST stuff
firstQuery = true
optionalQueries := []string{}
parameters := []string{}
optionalParameters := []WorkflowAppActionParameter{}
if len(path.Get.Parameters) > 0 {
for _, param := range path.Get.Parameters {
curParam := WorkflowAppActionParameter{
Name: param.Value.Name,
Description: param.Value.Description,
Multiline: false,
Required: param.Value.Required,
Schema: SchemaDefinition{
Type: param.Value.Schema.Value.Type,
},
}
if param.Value.Required {
action.Parameters = append(action.Parameters, curParam)
} else {
optionalParameters = append(optionalParameters, curParam)
}
if param.Value.In == "path" {
log.Printf("PATH!: %s", param.Value.Name)
parameters = append(parameters, param.Value.Name)
//baseUrl = fmt.Sprintf("%s%s", baseUrl)
} else if param.Value.In == "query" {
log.Printf("QUERY!: %s", param.Value.Name)
if !param.Value.Required {
optionalQueries = append(optionalQueries, param.Value.Name)
continue
}
parameters = append(parameters, param.Value.Name)
if firstQuery {
baseUrl = fmt.Sprintf("%s?%s={%s}", baseUrl, param.Value.Name, param.Value.Name)
firstQuery = false
} else {
baseUrl = fmt.Sprintf("%s&%s={%s}", baseUrl, param.Value.Name, param.Value.Name)
firstQuery = false
}
}
}
}
// ensuring that they end up last in the specification
// (order is ish important for optional params) - they need to be last.
for _, optionalParam := range optionalParameters {
action.Parameters = append(action.Parameters, optionalParam)
}
curCode := makePythoncode(functionName, baseUrl, "get", parameters, optionalQueries)
pythonFunctions = append(pythonFunctions, curCode)
api.Actions = append(api.Actions, action)
}
}
return api, pythonFunctions, nil
}
func verifyApi(api WorkflowApp) WorkflowApp {
if api.AppVersion == "" {
api.AppVersion = "1.0.0"
}
return api
}
func dumpPythonGCP(client *storage.Client, basePath, name, version string, pythonFunctions []string) error {
//log.Printf("%#v", api)
log.Printf(strings.Join(pythonFunctions, "\n"))
parsedCode := fmt.Sprintf(`import requests
import asyncio
import json
from walkoff_app_sdk.app_base import AppBase
class %s(AppBase):
"""
Autogenerated class by Shuffler
"""
__version__ = "%s"
app_name = "%s"
def __init__(self, redis, logger, console_logger=None):
self.verify = False
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
super().__init__(redis, logger, console_logger)
%s
if __name__ == "__main__":
asyncio.run(CarbonBlack.run(), debug=True)
`, name, version, name, strings.Join(pythonFunctions, "\n"))
// Create bucket handle
ctx := context.Background()
bucket := client.Bucket(bucketName)
obj := bucket.Object(fmt.Sprintf("%s/src/app.py", basePath))
w := obj.NewWriter(ctx)
if _, err := fmt.Fprintf(w, parsedCode); err != nil {
return err
}
// Close, just like writing a file.
if err := w.Close(); err != nil {
return err
}
return nil
}
func dumpPython(basePath, name, version string, pythonFunctions []string) error {
//log.Printf("%#v", api)
log.Printf(strings.Join(pythonFunctions, "\n"))
parsedCode := fmt.Sprintf(`import requests
import asyncio
import json
from walkoff_app_sdk.app_base import AppBase
class %s(AppBase):
"""
Autogenerated class by Shuffler
"""
__version__ = "%s"
app_name = "%s"
def __init__(self, redis, logger, console_logger=None):
self.verify = False
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
super().__init__(redis, logger, console_logger)
%s
if __name__ == "__main__":
asyncio.run(CarbonBlack.run(), debug=True)
`, name, version, name, strings.Join(pythonFunctions, "\n"))
err := ioutil.WriteFile(fmt.Sprintf("%s/src/app.py", basePath), []byte(parsedCode), os.ModePerm)
if err != nil {
return err
}
fmt.Println(parsedCode)
//log.Println(string(data))
return nil
}
func dumpApiGCP(client *storage.Client, basePath string, api WorkflowApp) error {
//log.Printf("%#v", api)
data, err := yaml.Marshal(api)
if err != nil {
log.Printf("Error with yaml marshal: %s", err)
return err
}
// Create bucket handle
ctx := context.Background()
bucket := client.Bucket(bucketName)
obj := bucket.Object(fmt.Sprintf("%s/app.yaml", basePath))
w := obj.NewWriter(ctx)
if _, err := fmt.Fprintf(w, string(data)); err != nil {
return err
}
// Close, just like writing a file.
if err := w.Close(); err != nil {
return err
}
//log.Println(string(data))
return nil
}
func dumpApi(basePath string, api WorkflowApp) error {
//log.Printf("%#v", api)
data, err := yaml.Marshal(api)
if err != nil {
log.Printf("Error with yaml marshal: %s", err)
return err
}
err = ioutil.WriteFile(fmt.Sprintf("%s/api.yaml", basePath), []byte(data), os.ModePerm)
if err != nil {
return err
}
//log.Println(string(data))
return nil
}
func main() {
data := []byte(`{"swagger":"3.0","info":{"title":"hi","description":"you","version":"1.0"},"servers":[{"url":"https://shuffler.io/api/v1"}],"host":"shuffler.io","basePath":"/api/v1","schemes":["https:"],"paths":{"/workflows":{"get":{"responses":{"default":{"description":"default","schema":{}}},"summary":"Get workflows","description":"Get workflows","parameters":[]}},"/workflows/{id}":{"get":{"responses":{"default":{"description":"default","schema":{}}},"summary":"Get workflow","description":"Get workflow","parameters":[{"in":"query","name":"forgetme","description":"Generated by shuffler.io OpenAPI","required":true,"schema":{"type":"string"}},{"in":"query","name":"anotherone","description":"Generated by shuffler.io OpenAPI","required":false,"schema":{"type":"string"}},{"in":"query","name":"hi","description":"Generated by shuffler.io OpenAPI","required":true,"schema":{"type":"string"}},{"in":"path","name":"id","description":"Generated by shuffler.io OpenAPI","required":true,"schema":{"type":"string"}}]}}},"securityDefinitions":{}}`)
ctx := context.Background()
client, err := storage.NewClient(ctx)
if err != nil {
log.Printf("Failed to create client: %v", err)
os.Exit(3)
}
hasher := md5.New()
hasher.Write(data)
newmd5 := hex.EncodeToString(hasher.Sum(nil))
swagger, err := openapi3.NewSwaggerLoader().LoadSwaggerFromData(data)
if err != nil {
log.Printf("Swagger validation error: %s", err)
os.Exit(3)
}
if strings.Contains(swagger.Info.Title, " ") {
strings.ReplaceAll(swagger.Info.Title, " ", "")
}
basePath, err := buildStructureGCP(client, swagger, newmd5)
if err != nil {
log.Printf("Failed to build base structure: %s", err)
os.Exit(3)
}
api, pythonfunctions, err := generateYaml(swagger)
if err != nil {
log.Printf("Failed building and generating yaml: %s", err)
os.Exit(3)
}
err = dumpApiGCP(client, basePath, api)
if err != nil {
log.Printf("Failed dumping yaml: %s", err)
os.Exit(3)
}
err = dumpPythonGCP(client, basePath, swagger.Info.Title, swagger.Info.Version, pythonfunctions)
if err != nil {
log.Printf("Failed dumping python: %s", err)
os.Exit(3)
}
}