Fix bufio.Scanner: token too long

godotenv.go

@@ -99,16 +99,7 @@ func Read(filenames ...string) (envMap map[string]string, err error) {
 // Parse reads an env file from io.Reader, returning a map of keys and values.
 func Parse(r io.Reader) (envMap map[string]string, err error) {
 	envMap = make(map[string]string)
-
-	var lines []string
-	scanner := bufio.NewScanner(r)
-	for scanner.Scan() {
-		lines = append(lines, scanner.Text())
-	}
-
-	if err = scanner.Err(); err != nil {
-		return
-	}
+	lines, err := ReadAllLines(r)
 
 	for _, fullLine := range lines {
 		if !isIgnoredLine(fullLine) {
@@ -124,6 +115,35 @@ func Parse(r io.Reader) (envMap map[string]string, err error) {
 	return
 }
 
+// ReadAllLines will convert the Reader into an array of strings
+// one for each line in the original text.
+func ReadAllLines(r io.Reader) (lines []string, err error) {
+	bf := bufio.NewReader(r)
+	linePart, isPrefix, err := bf.ReadLine()
+
+	for err == nil {
+		lineStr := string(linePart)
+
+		// handling of lines larger than the default buffer size
+		for err == nil && isPrefix == true {
+			linePart, isPrefix, err = bf.ReadLine()
+			if err != nil {
+				break
+			}
+			lineStr += string(linePart)
+		}
+
+		// `err == nil` is used to skip lines which caused an error
+		if isPrefix == false && err == nil {
+			lines = append(lines, lineStr)
+		}
+
+		linePart, isPrefix, err = bf.ReadLine()
+	}
+
+	return
+}
+
 //Unmarshal reads an env file from a string, returning a map of keys and values.
 func Unmarshal(str string) (envMap map[string]string, err error) {
 	return Parse(strings.NewReader(str))

godotenv_test.go

@@ -98,6 +98,30 @@ func TestReadPlainEnv(t *testing.T) {
 	}
 }
 
+func TestReadPlainEnvWithLongLine(t *testing.T) {
+	envFileName := "fixtures/long.env"
+	expectedBigValue := strings.Repeat("abcd", 20000)
+	expectedValues := map[string]string{
+		"BIG_VALUE": expectedBigValue,
+		"SMALL_VALUE": "abcd",
+	}
+
+	envMap, err := Read(envFileName)
+	if err != nil {
+		t.Error("Error reading file -> " + err.Error() )
+	}
+
+	if len(envMap) != len(expectedValues) {
+		t.Error("Didn't get the right size map back -->")
+	}
+
+	for key, value := range expectedValues {
+		if envMap[key] != value {
+			t.Error("Read got one of the keys wrong")
+		}
+	}
+}
+
 func TestParse(t *testing.T) {
 	envMap, err := Parse(bytes.NewReader([]byte("ONE=1\nTWO='2'\nTHREE = \"3\"")))
 	expectedValues := map[string]string{