internal/encoding: don't interpret file as UTF8 with binary encoding

cmd/cue/cmd/script_test.go

@@ -238,6 +238,18 @@ func TestScript(t *testing.T) {
 					ts.Check(err)
 				}
 			},
+			// strconv-unquote treats each argument as a go quoted string, unquotes it and prints
+			// them, separated by newlines, to stdout
+			"strconv-unquote": func(ts *testscript.TestScript, neg bool, args []string) {
+				if neg {
+					ts.Fatalf("usage: strconv-unquote args...")
+				}
+				for _, quoted := range args {
+					s, err := strconv.Unquote(quoted)
+					ts.Check(err)
+					fmt.Fprintln(ts.Stdout(), s)
+				}
+			},
 		},
 		Setup: func(e *testscript.Env) error {
 			// If a testscript loads CUE packages but forgot to set up a cue.mod,

cmd/cue/cmd/testdata/script/embed.txtar

@@ -4,6 +4,14 @@ env CUE_EXPERIMENT=embed=0
 cmp stderr out/noembed
 env CUE_EXPERIMENT=
 
+# Create test files for type=binary. txtar can only hold valid UTF8 - we need to test arbitrary bytes as input so we use quoted strings for storage
+# Text file, encoded in UTF8-8 with leading BOM
+strconv-unquote '"\xef\xbb\xbfHello, 世界"'
+cp stdout test-utf8.binary
+# A byte sequence - should be invalid UTF-8
+strconv-unquote "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
+cp stdout test-nonutf8.binary
+
 exec cue eval
 cmp stdout out/eval
 
@@ -39,6 +47,12 @@ f: _ @embed(file="openapi.json", type=openapi)
 
 g: _ @embed(file="openapi.json") // test no auto mode!
 
+h: _ @embed(file="test-utf8.binary", type=binary)
+
+i: _ @embed(file="test-utf8.binary", type=text)
+
+j: _ @embed(file="test-nonutf8.binary", type=binary)
+
 special: {
 	// These are all valid.
 	underscoreFile: _ @embed(file="y/_test.json")
@@ -213,6 +227,18 @@ g: {
         }
     }
 }
+h: '''
+    \ufeffHello, 世界
+
+    '''
+i: """
+    Hello, 世界
+
+    """
+j: '''
+    \xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7
+
+    '''
 special: {
     underscoreFile: {
         z: 45
@@ -273,6 +299,18 @@ g: {
 		}
 	}
 }
+h: '''
+	\ufeffHello, 世界
+
+	'''
+i: """
+	Hello, 世界
+
+	"""
+j: '''
+	\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7
+
+	'''
 special: {
 	// These are all valid.
 	underscoreFile: z:  45

internal/encoding/encoding.go

@@ -262,7 +262,9 @@ func NewDecoder(ctx *cue.Context, f *build.File, cfg *Config) *Decoder {
 		i.err = err
 		i.expr = ast.NewString(string(b))
 	case build.Binary:
-		b, err := io.ReadAll(r)
+		// Binary files should not generally be treated as UTF-8. Don't use the
+		// [transform.Reader] created above but read directly from the file instead.
+		b, err := io.ReadAll(srcr)
 		i.err = err
 		s := literal.Bytes.WithTabIndent(1).Quote(string(b))
 		i.expr = ast.NewLit(token.STRING, s)