feat: Support raising builtin errors in the test runner

It can be useful to know if Rego unit tests fail due to logic errors, or
if a builtin raised an error, such as parsing JSON input that was typo'd.

Signed-off-by: James Alseth <james@jalseth.me>

Author: jalseth

tester/runner.go

@@ -118,6 +118,7 @@ type Runner struct {
 	cover                 topdown.QueryTracer
 	trace                 bool
 	enablePrintStatements bool
+	raiseBuiltinErrors    bool
 	runtime               *ast.Term
 	timeout               time.Duration
 	modules               map[string]*ast.Module
@@ -140,6 +141,13 @@ func (r *Runner) SetCompiler(compiler *ast.Compiler) *Runner {
 	return r
 }
 
+// RaiseBuiltinErrors sets the runner to raise errors encountered by builtins
+// such as parsing input.
+func (r *Runner) RaiseBuiltinErrors(enabled bool) *Runner {
+	r.raiseBuiltinErrors = enabled
+	return r
+}
+
 type Builtin struct {
 	Decl *ast.Builtin
 	Func func(*rego.Rego)
@@ -461,7 +469,7 @@ func (r *Runner) runTest(ctx context.Context, txn storage.Transaction, mod *ast.
 	}
 
 	printbuf := bytes.NewBuffer(nil)
-
+	var builtinErrors []topdown.Error
 	rg := rego.New(
 		rego.Store(r.store),
 		rego.Transaction(txn),
@@ -471,6 +479,7 @@ func (r *Runner) runTest(ctx context.Context, txn storage.Transaction, mod *ast.
 		rego.Runtime(r.runtime),
 		rego.Target(r.target),
 		rego.PrintHook(topdown.NewPrintHook(printbuf)),
+		rego.BuiltinErrorList(&builtinErrors),
 	)
 
 	// Register custom builtins on rego instance
@@ -483,15 +492,24 @@ func (r *Runner) runTest(ctx context.Context, txn storage.Transaction, mod *ast.
 	dt := time.Since(t0)
 
 	var trace []*topdown.Event
-
 	if bufferTracer != nil {
 		trace = *bufferTracer
 	}
 
 	tr := newResult(rule.Loc(), mod.Package.Path.String(), rule.Head.Ref().String(), dt, trace, printbuf.Bytes())
-	tr.Error = err
-	var stop bool
 
+	// If there was an error other than errors from builtins, prefer that error.
+	if err != nil {
+		tr.Error = err
+	} else if r.raiseBuiltinErrors && len(builtinErrors) > 0 {
+		if len(builtinErrors) == 1 {
+			tr.Error = &builtinErrors[0]
+		} else {
+			tr.Error = fmt.Errorf("%v", builtinErrors)
+		}
+	}
+
+	var stop bool
 	if err != nil {
 		if topdown.IsCancel(err) || wasm_errors.IsCancel(err) {
 			stop = ctx.Err() != context.DeadlineExceeded

tester/runner_test.go

@@ -6,6 +6,7 @@ package tester_test
 
 import (
 	"context"
+	"fmt"
 	"reflect"
 	"testing"
 	"time"
@@ -609,3 +610,68 @@ func TestRunnerWithCustomBuiltin(t *testing.T) {
 		}
 	})
 }
+
+func TestRunnerWithBuiltinErrors(t *testing.T) {
+	const ruleTemplate = `package test
+	test_json_parsing {
+      x := json.unmarshal("%s")
+	  x.test == 123
+	}`
+
+	testCases := []struct {
+		desc          string
+		json          string
+		builtinErrors bool
+		wantErr       bool
+	}{
+		{
+			desc:          "Valid JSON with flag enabled does not raise an error",
+			json:          `{\"test\": 123}`,
+			builtinErrors: true,
+		},
+		{
+			desc:          "Invalid JSON with flag enabled raises an error",
+			json:          `test: 123`,
+			builtinErrors: true,
+			wantErr:       true,
+		},
+		{
+			desc: "Invalid JSON with flag disabled does not raise an error",
+			json: `test: 123`,
+		},
+	}
+
+	ctx := context.Background()
+
+	for _, tc := range testCases {
+		t.Run(tc.desc, func(t *testing.T) {
+			files := map[string]string{
+				"builtin_error_test.rego": fmt.Sprintf(ruleTemplate, tc.json),
+			}
+
+			test.WithTempFS(files, func(d string) {
+				paths := []string{d}
+				modules, store, err := tester.Load(paths, nil)
+				if err != nil {
+					t.Fatal(err)
+				}
+				txn := storage.NewTransactionOrDie(ctx, store)
+				runner := tester.
+					NewRunner().
+					SetStore(store).
+					SetModules(modules).
+					RaiseBuiltinErrors(tc.builtinErrors)
+
+				ch, err := runner.RunTests(ctx, txn)
+				if err != nil {
+					t.Fatal(err)
+				}
+				for result := range ch {
+					if gotErr := result.Error != nil; gotErr != tc.wantErr {
+						t.Errorf("wantErr = %v, gotErr = %v", tc.wantErr, gotErr)
+					}
+				}
+			})
+		})
+	}
+}