Add debug logging option

And fix stdout so nothing except the schema prints (unless there is an error).

Also add debug logging for the reader package, and for validation rules.

```
Debug: found the following files in "/home/aisling/dev/HewlettPackard/terraschema/test/modules/simple":
        "/home/aisling/dev/HewlettPackard/terraschema/test/modules/simple/main.tf", with variable(s):
        "/home/aisling/dev/HewlettPackard/terraschema/test/modules/simple/variables.tf", with variable(s):
                name
                age
Warning: couldn't apply validation for "age" with condition "var.age > env.age": no translation rules are supported for this condition
Debug: condition located at "/home/aisling/dev/HewlettPackard/terraschema/test/modules/simple/variables.tf:14,21-38"
Debug: the following errors occurred:
        contains([...],var.input_parameter): condition is not a 'contains()' function
        var == "a" || var == "b": operator is not || or ==
        a <>= (variable or variable length) (&& ...): could not evaluate expression as a constant value: /home/aisling/dev/HewlettPackard/terraschema/test/modules/simple/variables.tf:14,31-34: Variables not allowed; Variables may not be used here.
        can(regex("...",var.input_parameter)): rule can only be applied to string types, not "number"
Schema written to "/home/aisling/dev/HewlettPackard/terraschema/schema.json"
```

cmd/cmd.go

@@ -21,6 +21,7 @@ var (
 	outputStdOut                 bool
 	inputPath                    string
 	outputPath                   string
+	debugOut                     bool
 )
 
 // rootCmd is the base command for terraschema
@@ -46,6 +47,7 @@ var rootCmd = &cobra.Command{
 //   - input: folder, default is .
 //   - allow-empty: if no variables are found, print empty schema and exit with 0
 //   - require-all: require all variables to be present in the schema, even if a default value is specified
+//   - debug: output logs to track variables retrieved from each file, and get more verbose logs from custom validation rules
 func Execute() error {
 	return rootCmd.Execute()
 }
@@ -73,14 +75,19 @@ func init() {
 	)
 
 	rootCmd.Flags().BoolVar(&overwrite, "overwrite", false,
-		"overwrite an existing schema file",
+		"allow overwriting an existing file",
 	)
 
 	rootCmd.Flags().BoolVar(&outputStdOut, "stdout", false,
-		"output schema content to stdout instead of a file and disable any other logging\n"+
+		"output JSON Schema content to stdout instead of a file and disable any other logging\n"+
 			"unless an error occurs. Overrides 'debug' and 'output.",
 	)
 
+	rootCmd.Flags().BoolVar(&debugOut, "debug", false,
+		"output debug logs, may useful for troubleshooting issues relating to translating\n"+
+			"validation rules. Does not work with --stdout",
+	)
+
 	rootCmd.SetFlagErrorFunc(func(cmd *cobra.Command, err error) error {
 		_ = rootCmd.Usage()
 
@@ -151,6 +158,8 @@ func runCommand(cmd *cobra.Command, args []string) error {
 		RequireAll:                requireAll,
 		AllowAdditionalProperties: !disallowAdditionalProperties,
 		AllowEmpty:                allowEmpty,
+		DebugOut:                  debugOut && !outputStdOut,
+		SuppressLogging:           outputStdOut,
 	})
 	if err != nil {
 		return fmt.Errorf("error creating schema: %w", err)

pkg/jsonschema/json-schema.go

@@ -14,29 +14,23 @@ type CreateSchemaOptions struct {
 	RequireAll                bool
 	AllowAdditionalProperties bool
 	AllowEmpty                bool
+	DebugOut                  bool
+	SuppressLogging           bool
 }
 
 func CreateSchema(path string, options CreateSchemaOptions) (map[string]any, error) {
 	schemaOut := make(map[string]any)
 
-	varMap, err := reader.GetVarMap(path)
+	varMap, err := reader.GetVarMap(path, options.DebugOut)
 	if err != nil {
-		if errors.Is(err, reader.ErrFilesNotFound) {
-			if options.AllowEmpty {
-				fmt.Printf("Info: no tf files were found in %q, creating empty schema\n", path)
-
-				return schemaOut, nil
+		if options.AllowEmpty && (errors.Is(err, reader.ErrFilesNotFound) || errors.Is(err, reader.ErrNoVariablesFound)) {
+			if !options.SuppressLogging {
+				fmt.Printf("Warning: directory %q: %v, creating empty schema file\n", path, err)
 			}
-		} else {
-			return schemaOut, fmt.Errorf("error reading tf files at %q: %w", path, err)
-		}
-	}
 
-	if len(varMap) == 0 {
-		if options.AllowEmpty {
 			return schemaOut, nil
 		} else {
-			return schemaOut, errors.New("no variables found in tf files")
+			return schemaOut, fmt.Errorf("error reading tf files at %q: %w", path, err)
 		}
 	}
 
@@ -92,8 +86,21 @@ func createNode(name string, v model.TranslatedVariable, options CreateSchemaOpt
 	if v.Variable.Validation != nil && v.ConditionAsString != nil {
 		err = parseConditionToNode(v.Variable.Validation.Condition, *v.ConditionAsString, name, &node)
 		// if an error occurs, log it and continue.
-		if err != nil {
-			fmt.Printf("couldn't apply validation for %q with condition %q. Error: %v\n", name, *v.ConditionAsString, err)
+		if err != nil && !options.SuppressLogging {
+			fmt.Printf("Warning: couldn't apply validation for %q with condition %q: %v\n",
+				name,
+				*v.ConditionAsString,
+				err,
+			)
+			// if the debug flag is set, print all the errors returned by each of the rules as they try to apply to the condition.
+			var validationError ValidationApplyError
+			if ok := errors.As(err, &validationError); ok && options.DebugOut {
+				fmt.Printf("Debug: condition located at %q\n", v.Variable.Validation.Condition.Range().String())
+				fmt.Println("Debug: the following errors occurred:")
+				for k, v := range validationError.ErrorMap {
+					fmt.Printf("\t%s: %v\n", k, v)
+				}
+			}
 		}
 	}
 

pkg/jsonschema/validation.go

@@ -9,7 +9,12 @@ import (
 
 type conditionMutator func(hcl.Expression, string, string) (map[string]any, error)
 
-var ErrConditionNotApplied = fmt.Errorf("condition could not be applied")
+var ErrConditionNotApplied = fmt.Errorf("no translation rules are supported for this condition")
+
+type ValidationApplyError struct {
+	error
+	ErrorMap map[string]error
+}
 
 func parseConditionToNode(ex hcl.Expression, _ string, name string, m *map[string]any) error {
 	if m == nil {
@@ -20,12 +25,14 @@ func parseConditionToNode(ex hcl.Expression, _ string, name string, m *map[strin
 		return fmt.Errorf("cannot apply validation, type is not defined for %#v", *m)
 	}
 	functions := map[string]conditionMutator{
-		"contains([...],var.input_parameter)":             contains,
-		"var == \"a\" || var == \"b\"":                    isOneOf,
-		"a <= (variable or variable length) < b (&& ...)": comparison,
-		"can(regex(\"...\",var.input_parameter))":         canRegex,
+		"contains([...],var.input_parameter)":          contains,
+		"var == \"a\" || var == \"b\"":                 isOneOf,
+		"a <>= (variable or variable length) (&& ...)": comparison,
+		"can(regex(\"...\",var.input_parameter))":      canRegex,
 	}
-	for _, fn := range functions {
+
+	errorMap := make(map[string]error)
+	for fnName, fn := range functions {
 		updatedNode, err := fn(ex, name, t)
 		if err == nil {
 			// apply updated node to m:
@@ -35,9 +42,10 @@ func parseConditionToNode(ex hcl.Expression, _ string, name string, m *map[strin
 
 			return nil
 		}
+		errorMap[fnName] = err
 	}
 
-	return ErrConditionNotApplied
+	return ValidationApplyError{ErrConditionNotApplied, errorMap}
 }
 
 func isOneOf(ex hcl.Expression, name string, _ string) (map[string]any, error) {
@@ -57,7 +65,7 @@ func isOneOf(ex hcl.Expression, name string, _ string) (map[string]any, error) {
 func contains(ex hcl.Expression, name string, _ string) (map[string]any, error) {
 	args, ok := argumentsOfCall(ex, "contains", 2)
 	if !ok {
-		return nil, fmt.Errorf("condition is not a contains function")
+		return nil, fmt.Errorf("condition is not a 'contains()' function")
 	}
 
 	l, d := hcl.ExprList(args[0])
@@ -89,7 +97,7 @@ func comparison(ex hcl.Expression, name string, t string) (map[string]any, error
 		"string": true,
 	}
 	if !allowedTypes[t] {
-		return nil, fmt.Errorf("rule can only be applied to object, array, number or string types")
+		return nil, fmt.Errorf("rule can only be applied to object, array, number or string types, not %q", t)
 	}
 
 	node := map[string]any{"type": t}
@@ -103,17 +111,17 @@ func comparison(ex hcl.Expression, name string, t string) (map[string]any, error
 
 func canRegex(ex hcl.Expression, name string, t string) (map[string]any, error) {
 	if t != "string" {
-		return nil, fmt.Errorf("rule can only be applied to string types")
+		return nil, fmt.Errorf("rule can only be applied to string types, not %q", t)
 	}
 
 	canArgs, ok := argumentsOfCall(ex, "can", 1)
 	if !ok {
-		return nil, fmt.Errorf("condition is not a can function")
+		return nil, fmt.Errorf("condition is not a 'can()' function")
 	}
 
 	regexArgs, ok := argumentsOfCall(canArgs[0], "regex", 2)
 	if !ok {
-		return nil, fmt.Errorf("condition is not a can(regex) function")
+		return nil, fmt.Errorf("condition is not a 'can(regex())' function")
 	}
 	if !isExpressionVarName(regexArgs[1], name) {
 		return nil, fmt.Errorf("second argument is not a direct reference to the input variable")

pkg/jsonschema/validation_util.go

@@ -96,9 +96,11 @@ func parseComparisonExpression(ex *hclsyntax.BinaryOpExpr, name string, node *ma
 			return fmt.Errorf("could not flip sign")
 		}
 	}
-	val, d := ex.RHS.Value(nil)
+	// parse the right hand side as a constant numeric value. Can't reference other variables since that would require
+	// making a more complex JSON schema (eg with "if" and "then" properties).
+	val, d := ex.RHS.Value(&hcl.EvalContext{})
 	if d.HasErrors() {
-		return fmt.Errorf("could not evaluate expression: %w", d)
+		return fmt.Errorf("could not evaluate expression as a constant value: %w", d)
 	}
 	var num float64
 	err := gocty.FromCtyValue(val, &num)
@@ -119,7 +121,7 @@ func parseComparisonExpression(ex *hclsyntax.BinaryOpExpr, name string, node *ma
 	if condition1 || condition2 {
 		return performOp(ex.Op, node, num, nodeType)
 	} else {
-		return fmt.Errorf("variable name not found")
+		return fmt.Errorf("operation not supported for type %q op %v", nodeType, ex.Op)
 	}
 }
 
@@ -270,5 +272,5 @@ func parseEqualityExpression(ex *hclsyntax.BinaryOpExpr, name string, enum *[]an
 		return nil
 	}
 
-	return fmt.Errorf("variable name not found")
+	return fmt.Errorf("variable name not found in expression")
 }

pkg/jsonschema/value_test.go

@@ -37,7 +37,7 @@ func TestExpressionToJSONObject_Default(t *testing.T) {
 
 			defaults := make(map[string]any)
 
-			varMap, err := reader.GetVarMap(filepath.Join(tfPath, name))
+			varMap, err := reader.GetVarMap(filepath.Join(tfPath, name), true)
 			if err != nil && !errors.Is(err, reader.ErrFilesNotFound) {
 				t.Errorf("error reading tf files: %v", err)
 			}

pkg/reader/reader.go

@@ -21,14 +21,17 @@ var fileSchema = &hcl.BodySchema{
 	},
 }
 
-var ErrFilesNotFound = fmt.Errorf("no .tf files found in directory")
+var (
+	ErrFilesNotFound    = fmt.Errorf("no .tf files found")
+	ErrNoVariablesFound = fmt.Errorf("tf files don't contain any variables")
+)
 
 // GetVarMap reads all .tf files in a directory and returns a map of variable names to their translated values.
 // For the purpose of this application, all that matters is the model.VariableBlock contained in this, which
 // contains a direct unmarshal of the block itself using the hcl package. The rest of the information is for
 // debugging purposes, and to simplify the process of deciding if a variable is 'required' later. Note: in 'strict'
 // mode, all variables are required, regardless of whether they have a default value or not.
-func GetVarMap(path string) (map[string]model.TranslatedVariable, error) {
+func GetVarMap(path string, debugOut bool) (map[string]model.TranslatedVariable, error) {
 	// read all tf files in directory
 	files, err := filepath.Glob(filepath.Join(path, "*.tf"))
 	if err != nil {
@@ -38,10 +41,18 @@ func GetVarMap(path string) (map[string]model.TranslatedVariable, error) {
 		return nil, ErrFilesNotFound
 	}
 
+	if debugOut {
+		fmt.Printf("Debug: found the following files in %q:\n", path)
+	}
+
 	parser := hclparse.NewParser()
 
 	varMap := make(map[string]model.TranslatedVariable)
 	for _, fileName := range files {
+		if debugOut {
+			fmt.Printf("\t%q, with variable(s):\n", fileName)
+		}
+
 		file, d := parser.ParseHCLFile(fileName)
 		if d.HasErrors() {
 			return nil, d
@@ -57,9 +68,17 @@ func GetVarMap(path string) (map[string]model.TranslatedVariable, error) {
 				return nil, fmt.Errorf("error getting parsing %q: %w", name, err)
 			}
 			varMap[name] = translated
+
+			if debugOut {
+				fmt.Printf("\t\t%s\n", name)
+			}
 		}
 	}
 
+	if len(varMap) == 0 {
+		return nil, ErrNoVariablesFound
+	}
+
 	return varMap, nil
 }
 

pkg/reader/reader_test.go

@@ -21,7 +21,7 @@ func TestGetVarMap_Required(t *testing.T) {
 		name := testCases[i]
 		t.Run(name, func(t *testing.T) {
 			t.Parallel()
-			varMap, err := GetVarMap(filepath.Join(tfPath, name))
+			varMap, err := GetVarMap(filepath.Join(tfPath, name), true)
 			if err != nil && !errors.Is(err, ErrFilesNotFound) {
 				t.Errorf("Error reading tf files: %v", err)
 			}

pkg/reader/type-constraint_test.go

@@ -30,7 +30,7 @@ func TestGetTypeConstraint(t *testing.T) {
 			expected, err := os.ReadFile(filepath.Join(expectedPath, name, "type-constraints.json"))
 			require.NoError(t, err)
 
-			varMap, err := GetVarMap(filepath.Join(tfPath, name))
+			varMap, err := GetVarMap(filepath.Join(tfPath, name), true)
 			if err != nil && !errors.Is(err, ErrFilesNotFound) {
 				t.Errorf("Error reading tf files: %v", err)
 			}