Merge pull request #56 from trickest/error-handling

Improve error handling, CI mode, and some fixes

README.md

@@ -149,18 +149,19 @@ Use config.yaml file provided using **--config** flag to specify:
 trickest execute --workflow <workflow_or_tool_name> --space <space_name> --config <config_file_path> --set-name "New Name" [--watch]
 ```
 
-| Flag          | Type    | Default | Description                                                                                       |
-| ------------- | ------- | ------- | ------------------------------------------------------------------------------------------------- |
-| --config      | file    | /       | YAML file for run configuration                                                                   |
-| --workflow    | string  | /       | Workflow from the Store to be executed                                                            |
-| --file        | file    | /       | Workflow YAML file to execute                                                                     |
-| --max         | boolean | /       | Use maximum number of machines for workflow execution                                             |
-| --output      | string  | /       | A comma-separated list of nodes whose outputs should be downloaded when the execution is finished |
-| --output-all  | boolean | /       | Download all outputs when the execution is finished                                               |
-| --output-dir  | string  | .       | Path to the directory which should be used to store outputs                                       |
-| --show-params | boolean | /       | Show parameters in the workflow tree                                                              |
-| --watch       | boolean | /       | Option to track execution status in case workflow is in running state                             |
-| --set-name    | string  | /       | Sets the new workflow name and will copy the workflow to space and project supplied               |
+| Flag          | Type    | Default | Description                                                                                                                                 |
+| ------------- | ------- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------- |
+| --config      | file    | /       | YAML file for run configuration                                                                                                             |
+| --workflow    | string  | /       | Workflow from the Store to be executed                                                                                                      |
+| --file        | file    | /       | Workflow YAML file to execute                                                                                                               |
+| --max         | boolean | /       | Use maximum number of machines for workflow execution                                                                                       |
+| --output      | string  | /       | A comma-separated list of nodes whose outputs should be downloaded when the execution is finished                                           |
+| --output-all  | boolean | /       | Download all outputs when the execution is finished                                                                                         |
+| --output-dir  | string  | .       | Path to the directory which should be used to store outputs                                                                                 |
+| --show-params | boolean | /       | Show parameters in the workflow tree                                                                                                        |
+| --watch       | boolean | /       | Option to track execution status in case workflow is in running state                                                                       |
+| --set-name    | string  | /       | Sets the new workflow name and will copy the workflow to space and project supplied                                                         |
+| --ci          | boolean | false   | un in CI mode (in-progreess executions will be stopped when the CLI is forcefully stopped - if not set, you will be asked for confirmation) |
 
 Predefined config.yaml file content:
 ```

cmd/create/create.go

@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"github.com/spf13/cobra"
 	"io/ioutil"
 	"net/http"
 	"os"
@@ -13,6 +12,8 @@ import (
 	"trickest-cli/cmd/list"
 	"trickest-cli/types"
 	"trickest-cli/util"
+
+	"github.com/spf13/cobra"
 )
 
 var description string
@@ -135,17 +136,17 @@ func CreateProject(name string, description string, spaceName string) *types.Pro
 		os.Exit(0)
 	}
 
+	if resp.StatusCode != http.StatusCreated {
+		util.ProcessUnexpectedResponse(resp)
+	}
+
 	var bodyBytes []byte
 	bodyBytes, err = ioutil.ReadAll(resp.Body)
 	if err != nil {
 		fmt.Println("Error: Couldn't read response body!")
 		os.Exit(0)
 	}
 
-	if resp.StatusCode != http.StatusCreated {
-		util.ProcessUnexpectedResponse(resp)
-	}
-
 	fmt.Println("Project successfully created!")
 	var newProject types.Project
 	err = json.Unmarshal(bodyBytes, &newProject)

cmd/execute/execute.go

@@ -9,6 +9,7 @@ import (
 	"os"
 	"os/signal"
 	"path"
+	"regexp"
 	"sort"
 	"strconv"
 	"strings"
@@ -42,6 +43,7 @@ var (
 	downloadAllNodes  bool
 	outputsDirectory  string
 	outputNodesFlag   string
+	ci                bool
 )
 
 // ExecuteCmd represents the execute command
@@ -108,6 +110,7 @@ func init() {
 	ExecuteCmd.Flags().BoolVar(&downloadAllNodes, "output-all", false, "Download all outputs when the execution is finished")
 	ExecuteCmd.Flags().StringVar(&outputNodesFlag, "output", "", "A comma separated list of nodes which outputs should be downloaded when the execution is finished")
 	ExecuteCmd.Flags().StringVar(&outputsDirectory, "output-dir", "", "Path to directory which should be used to store outputs")
+	ExecuteCmd.Flags().BoolVar(&ci, "ci", false, "Run in CI mode (in-progreess executions will be stopped when the CLI is forcefully stopped - if not set, you will be asked for confirmation)")
 }
 
 func getToolScriptOrSplitterFromYAMLNode(node types.WorkflowYAMLNode) (*types.Tool, *types.Script, *types.Splitter) {
@@ -149,7 +152,7 @@ func setConnectedSplitters(version *types.WorkflowVersionDetailed, splitterIDs *
 		tempMap := make(map[string]string, 0)
 		splitterIDs = &tempMap
 		for _, node := range version.Data.Nodes {
-			if strings.HasPrefix(node.Name, "file-splitter") {
+			if strings.HasPrefix(node.Name, "file-splitter") || strings.HasPrefix(node.Name, "split-to-string") {
 				(*splitterIDs)[node.Name] = node.Name
 				continue
 			}
@@ -165,7 +168,8 @@ func setConnectedSplitters(version *types.WorkflowVersionDetailed, splitterIDs *
 				if strings.Contains(connection.Source.ID, nodeName) {
 					destinationNodeID := getNodeNameFromConnectionID(connection.Destination.ID)
 					_, exists := (*splitterIDs)[destinationNodeID]
-					if strings.HasPrefix(destinationNodeID, "file-splitter-") ||
+					isSplitter := strings.HasPrefix(destinationNodeID, "file-splitter-") || strings.HasPrefix(destinationNodeID, "split-to-string-")
+					if isSplitter ||
 						(strings.Contains(connection.Destination.ID, "folder") &&
 							version.Data.Nodes[destinationNodeID].Script != nil) || exists {
 						continue
@@ -527,7 +531,7 @@ func readWorkflowYAMLandCreateVersion(fileName string, workflowName string, obje
 							if toolInput.Command != "" {
 								newNode.Inputs[name].Command = &toolInput.Command
 							}
-							if strings.HasPrefix(val, "file-splitter-") {
+							if strings.HasPrefix(val, "file-splitter-") || strings.HasPrefix(val, "split-to-string-") {
 								workerConnected := true
 								newNode.Inputs[name].Value = "in/" + val + ":item"
 								newNode.Inputs[name].WorkerConnected = &workerConnected
@@ -567,7 +571,7 @@ func readWorkflowYAMLandCreateVersion(fileName string, workflowName string, obje
 								Command:     &toolInput.Command,
 								Description: &toolInput.Description,
 							}
-							if strings.HasPrefix(val, "file-splitter-") {
+							if strings.HasPrefix(val, "file-splitter-") || strings.HasPrefix(val, "split-to-string-") {
 								workerConnected := true
 								newNode.Inputs[name].Value = "in/" + val + ":item"
 								newNode.Inputs[name].WorkerConnected = &workerConnected
@@ -849,15 +853,20 @@ func WatchRun(runID, downloadPath string, nodesToDownload map[string]output.Node
 			_ = writer.Flush()
 			writer.Stop()
 
-			fmt.Println("The program will exit. Would you like to stop the remote execution? (Y/N)")
-			var answer string
-			for {
-				_, _ = fmt.Scan(&answer)
-				if strings.ToLower(answer) == "y" || strings.ToLower(answer) == "yes" {
-					stopRun(runID)
-					os.Exit(0)
-				} else if strings.ToLower(answer) == "n" || strings.ToLower(answer) == "no" {
-					os.Exit(0)
+			if ci {
+				stopRun(runID)
+				os.Exit(0)
+			} else {
+				fmt.Println("The program will exit. Would you like to stop the remote execution? (Y/N)")
+				var answer string
+				for {
+					_, _ = fmt.Scan(&answer)
+					if strings.ToLower(answer) == "y" || strings.ToLower(answer) == "yes" {
+						stopRun(runID)
+						os.Exit(0)
+					} else if strings.ToLower(answer) == "n" || strings.ToLower(answer) == "no" {
+						os.Exit(0)
+					}
 				}
 			}
 		}()
@@ -958,7 +967,7 @@ func PrintTrees(roots []*types.TreeNode, allNodes *map[string]*types.TreeNode, s
 		treeSplit := strings.Split(tree, "\n")
 		for _, line := range treeSplit {
 			if line != "" {
-				if strings.Contains(line, "(") {
+				if match, _ := regexp.MatchString(`\([-a-z0-9]+-[0-9]+\)`, line); match {
 					lineSplit := strings.Split(line, "(")
 					nodeName := strings.Trim(lineSplit[1], ")")
 					node := (*allNodes)[nodeName]
@@ -1012,7 +1021,7 @@ func printTree(node *types.TreeNode, branch *treeprint.Tree, allNodes *map[strin
 				switch v := input.Value.(type) {
 				case string:
 					if strings.HasPrefix(v, "in/") {
-						if strings.Contains(v, "/file-splitter-") {
+						if strings.Contains(v, "/file-splitter-") || strings.Contains(v, "/split-to-string-") {
 							v = strings.TrimPrefix(v, "/in")
 							v = strings.TrimSuffix(v, ":item")
 						} else {
@@ -1368,7 +1377,7 @@ func getNodeByName(name string, version *types.WorkflowVersionDetailed) *types.N
 		toolNodeFound := false
 		for id, n := range version.Data.Nodes {
 			if n.Meta.Label == name {
-				if n.Script != nil || strings.HasPrefix(id, "file-splitter") {
+				if n.Script != nil || strings.HasPrefix(id, "file-splitter") || strings.HasPrefix(id, "split-to-string") {
 					node = n
 					labelCnt++
 					ok = true
@@ -1472,7 +1481,8 @@ func readConfigInputs(config *map[string]interface{}, wfVersion *types.WorkflowV
 					} else {
 						node = getNodeByName(param, wfVersion)
 					}
-					if node.Script == nil && !strings.HasPrefix(node.Name, "file-splitter") &&
+					isSplitter := strings.HasPrefix(node.Name, "file-splitter") || strings.HasPrefix(node.Name, "split-to-string")
+					if node.Script == nil && !isSplitter &&
 						len(wfVersion.Data.Nodes) > 1 {
 						fmt.Println(param)
 						fmt.Println("Node is not a script or a file splitter, use tool.param-name syntax instead!")
@@ -1510,7 +1520,7 @@ func readConfigInputs(config *map[string]interface{}, wfVersion *types.WorkflowV
 						fmt.Println("      ...\n   ]")
 						os.Exit(0)
 					}
-					if strings.HasPrefix(node.ID, "file-splitter") {
+					if strings.HasPrefix(node.ID, "file-splitter") || strings.HasPrefix(node.ID, "split-to-string") {
 						fmt.Println(param)
 						fmt.Println("Node is a file splitter, use the following syntax:")
 						fmt.Println("<splitter name or ID>:")
@@ -1534,7 +1544,8 @@ func readConfigInputs(config *map[string]interface{}, wfVersion *types.WorkflowV
 				}
 				inputType = toolInput.Type
 			} else {
-				if node.Script == nil && !strings.HasPrefix(node.Name, "file-splitter") {
+				isSplitter := strings.HasPrefix(node.Name, "file-splitter") || strings.HasPrefix(node.Name, "split-to-string")
+				if node.Script == nil && !isSplitter {
 					oldParam, paramExists := node.Inputs[paramName]
 					paramExists = paramExists && oldParam.Value != nil
 					if !paramExists {
@@ -1594,7 +1605,8 @@ func readConfigInputs(config *map[string]interface{}, wfVersion *types.WorkflowV
 				newPNode.Name = "boolean-input-" + strconv.Itoa(booleanInputsCnt)
 				newPNode.Value = val
 			case map[string]interface{}:
-				if node == nil || (node.Script == nil && !strings.HasPrefix(node.Name, "file-splitter")) {
+				isSplitter := strings.HasPrefix(node.Name, "file-splitter") || strings.HasPrefix(node.Name, "split-to-string")
+				if node == nil || (node.Script == nil && !isSplitter) {
 					fmt.Println(param + ": ")
 					fmt.Println(val)
 					fmt.Println("Invalid input type! Object inputs are used for scripts and splitters.")
@@ -1763,7 +1775,7 @@ func addPrimitiveNodeFromConfig(wfVersion *types.WorkflowVersionDetailed, newPri
 	updateNeeded := false
 	for _, connection := range wfVersion.Data.Connections {
 		source := getNodeNameFromConnectionID(connection.Source.ID)
-		isSplitter := strings.HasPrefix(node.Name, "file-splitter")
+		isSplitter := strings.HasPrefix(node.Name, "file-splitter") || strings.HasPrefix(node.Name, "split-to-string")
 		if strings.HasSuffix(connection.Destination.ID, node.Name+"/"+paramName) ||
 			(isSplitter && strings.HasSuffix(connection.Destination.ID, node.Name+"/multiple/"+source)) ||
 			(node.Script != nil && (strings.HasSuffix(connection.Destination.ID,
@@ -1790,7 +1802,7 @@ func addPrimitiveNodeFromConfig(wfVersion *types.WorkflowVersionDetailed, newPri
 
 			savedPNode, alreadyExists := (*newPrimitiveNodes)[primitiveNode.Name]
 			if alreadyExists {
-				if node.Script != nil || strings.HasPrefix(node.Name, "file-splitter") {
+				if node.Script != nil || strings.HasPrefix(node.Name, "file-splitter") || strings.HasPrefix(node.Name, "split-to-string") {
 					continue
 				} else if savedPNode.Value != newPNode.Value {
 					processDifferentParamsForASinglePNode(*savedPNode, newPNode)
@@ -1812,7 +1824,7 @@ func addPrimitiveNodeFromConfig(wfVersion *types.WorkflowVersionDetailed, newPri
 							break
 						}
 					}
-				} else if strings.HasPrefix(node.Name, "file-splitter") {
+				} else if strings.HasPrefix(node.Name, "file-splitter") || strings.HasPrefix(node.Name, "split-to-string") {
 					for id, input := range node.Inputs {
 						if id == "multiple/"+newPNode.Name {
 							input.Value = "in/" + newPNode.Name + "/" + path.Base(newPNode.Value.(string))

cmd/export/export.go

@@ -2,8 +2,6 @@ package export
 
 import (
 	"fmt"
-	"github.com/spf13/cobra"
-	"gopkg.in/yaml.v3"
 	"os"
 	"path/filepath"
 	"sort"
@@ -12,6 +10,9 @@ import (
 	"trickest-cli/cmd/list"
 	"trickest-cli/types"
 	"trickest-cli/util"
+
+	"github.com/spf13/cobra"
+	"gopkg.in/yaml.v3"
 )
 
 type workflowExport struct {
@@ -102,7 +103,7 @@ func createYAML(workflow *types.Workflow, destinationPath string) {
 				}
 
 				inputValueStr := fmt.Sprintf("%v", input.Value)
-				if strings.HasPrefix(inputValueStr, "in/file-splitter-") {
+				if strings.HasPrefix(inputValueStr, "in/file-splitter-") || strings.HasPrefix(inputValueStr, "in/split-to-string-") {
 					// in/file-splitter-x:item
 					value := strings.Split(inputValueStr, "/")[1]
 					value = strings.Split(value, ":")[0]

cmd/output/output.go

@@ -3,9 +3,6 @@ package output
 import (
 	"encoding/json"
 	"fmt"
-	"github.com/schollz/progressbar/v3"
-	"github.com/spf13/cobra"
-	"gopkg.in/yaml.v3"
 	"io"
 	"io/ioutil"
 	"math"
@@ -14,10 +11,13 @@ import (
 	"path"
 	"strconv"
 	"strings"
-	"time"
 	"trickest-cli/cmd/list"
 	"trickest-cli/types"
 	"trickest-cli/util"
+
+	"github.com/schollz/progressbar/v3"
+	"github.com/spf13/cobra"
+	"gopkg.in/yaml.v3"
 )
 
 type NodeInfo struct {
@@ -197,7 +197,8 @@ func DownloadRunOutput(run *types.Run, nodes map[string]NodeInfo, version *types
 		}
 	}
 
-	runDir := "run-" + run.StartedDate.Format(time.RFC3339)
+	const layout = "2006-01-02T150405Z"
+	runDir := "run-" + run.StartedDate.Format(layout)
 	runDir = strings.TrimSuffix(runDir, "Z")
 	runDir = strings.Replace(runDir, "T", "-", 1)
 	runDir = path.Join(destinationPath, runDir)

util/util.go

@@ -3,14 +3,14 @@ package util
 import (
 	"encoding/json"
 	"fmt"
-	"github.com/hako/durafmt"
 	"io/ioutil"
 	"net/http"
 	"os"
-	"strconv"
 	"strings"
 	"time"
 	"trickest-cli/types"
+
+	"github.com/hako/durafmt"
 )
 
 type UnexpectedResponse map[string]interface{}
@@ -143,12 +143,17 @@ func GetHiveInfo() *types.Hive {
 }
 
 func ProcessUnexpectedResponse(resp *http.Response) {
-	fmt.Println(resp.Request.Method + " " + resp.Request.URL.Path + " " + strconv.Itoa(resp.StatusCode))
+	// fmt.Println(resp.Request.Method + " " + resp.Request.URL.Path + " " + strconv.Itoa(resp.StatusCode))
 	if resp.StatusCode >= http.StatusInternalServerError {
 		fmt.Println("Sorry, something went wrong!")
 		os.Exit(0)
 	}
 
+	if resp.StatusCode == http.StatusUnauthorized {
+		fmt.Println("Error: Unauthorized to perform this action.\nPlease, make sure that your token is correct and that you have access to this resource.")
+		os.Exit(1)
+	}
+
 	bodyBytes, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
 		fmt.Println("Error: Couldn't read unexpected response.")