Fix "hugo new" EOF error with an archetype file without the final EOL

This rewrites `extractFrontMatterDelims` function to make it work with
an archetype file without the final EOL and adds more detailed error
messages and comments.

It also removes `matches` and `matches_quick` functions which aren't
called anywhere.

hugolib/page_test.go

@@ -447,7 +447,7 @@ func TestDegenerateInvalidFrontMatterShortDelim(t *testing.T) {
 		r   string
 		err string
 	}{
-		{INVALID_FRONT_MATTER_SHORT_DELIM_ENDING, "Unable to read frontmatter at filepos 45: EOF"},
+		{INVALID_FRONT_MATTER_SHORT_DELIM_ENDING, "unable to read frontmatter at filepos 45: EOF"},
 	}
 	for _, test := range tests {
 

parser/page.go

@@ -159,17 +159,13 @@ func isFrontMatterDelim(data []byte) bool {
 
 func determineDelims(firstLine []byte) (left, right []byte) {
 	switch len(firstLine) {
+	case 5:
+		fallthrough
 	case 4:
 		if firstLine[0] == YAML_LEAD[0] {
-			return []byte(YAML_DELIM_UNIX), []byte(YAML_DELIM_UNIX)
+			return []byte(YAML_DELIM), []byte(YAML_DELIM)
 		}
-		return []byte(TOML_DELIM_UNIX), []byte(TOML_DELIM_UNIX)
+		return []byte(TOML_DELIM), []byte(TOML_DELIM)
-
-	case 5:
-		if firstLine[0] == YAML_LEAD[0] {
-			return []byte(YAML_DELIM_DOS), []byte(YAML_DELIM_DOS)
-		}
-		return []byte(TOML_DELIM_DOS), []byte(TOML_DELIM_DOS)
 	case 3:
 		fallthrough
 	case 2:
@@ -181,104 +177,98 @@ func determineDelims(firstLine []byte) (left, right []byte) {
 	}
 }
 
+// extractFrontMatterDelims takes a frontmatter from the content bufio.Reader.
+// Begining white spaces of the bufio.Reader must be trimmed before call this
+// function.
 func extractFrontMatterDelims(r *bufio.Reader, left, right []byte) (fm FrontMatter, err error) {
 	var (
 		c         byte
-		level     int = 0
+		buf       bytes.Buffer
-		bytesRead int = 0
+		level     int  = 0
-		sameDelim     = bytes.Equal(left, right)
+		sameDelim bool = bytes.Equal(left, right)
 	)
 
-	wr := new(bytes.Buffer)
+	// Frontmatter must start with a delimiter. To check it first,
+	// pre-reads beginning delimiter length - 1 bytes from Reader
+	for i := 0; i < len(left)-1; i++ {
+		if c, err = r.ReadByte(); err != nil {
+			return nil, fmt.Errorf("unable to read frontmatter at filepos %d: %s", buf.Len(), err)
+		}
+		if err = buf.WriteByte(c); err != nil {
+			return nil, err
+		}
+	}
+
+	// Reads a character from Reader one by one and checks it matches the
+	// last character of one of delemiters to find the last character of
+	// frontmatter. If it matches, makes sure it contains the delimiter
+	// and if so, also checks it is followed by CR+LF or LF when YAML,
+	// TOML case. In JSON case, nested delimiters must be parsed and it
+	// is expected that the delimiter only contains one character.
 	for {
 		if c, err = r.ReadByte(); err != nil {
-			return nil, fmt.Errorf("Unable to read frontmatter at filepos %d: %s", bytesRead, err)
+			return nil, fmt.Errorf("unable to read frontmatter at filepos %d: %s", buf.Len(), err)
+		}
+		if err = buf.WriteByte(c); err != nil {
+			return nil, err
 		}
-		bytesRead += 1
 
 		switch c {
-		case left[0]:
+		case left[len(left)-1]:
-			var (
+			if sameDelim { // YAML, TOML case
-				buf       []byte = []byte{c}
+				if bytes.HasSuffix(buf.Bytes(), left) {
-				remaining []byte
+					c, err = r.ReadByte()
-			)
+					if err != nil {
-
+						// It is ok that the end delimiter ends with EOF
-			if remaining, err = r.Peek(len(left) - 1); err != nil {
+						if err != io.EOF || level != 1 {
-				return nil, err
+							return nil, fmt.Errorf("unable to read frontmatter at filepos %d: %s", buf.Len(), err)
-			}
+						}
-
+					} else {
-			buf = append(buf, remaining...)
+						switch c {
-
+						case '\n':
-			if bytes.Equal(buf, left) {
+							// ok
-				if sameDelim {
+						case '\r':
+							if err = buf.WriteByte(c); err != nil {
+								return nil, err
+							}
+							if c, err = r.ReadByte(); err != nil {
+								return nil, fmt.Errorf("unable to read frontmatter at filepos %d: %s", buf.Len(), err)
+							}
+							if c != '\n' {
+								return nil, fmt.Errorf("frontmatter delimiter must be followed by CR+LF or LF but those can't be found at filepos %d", buf.Len())
+							}
+						default:
+							return nil, fmt.Errorf("frontmatter delimiter must be followed by CR+LF or LF but those can't be found at filepos %d", buf.Len())
+						}
+						if err = buf.WriteByte(c); err != nil {
+							return nil, err
+						}
+					}
 					if level == 0 {
 						level = 1
 					} else {
 						level = 0
 					}
-				} else {
-					level += 1
 				}
+			} else { // JSON case
+				level++
 			}
-
+		case right[len(right)-1]: // JSON case only reaches here
-			if _, err = wr.Write([]byte{c}); err != nil {
+			level--
-				return nil, err
-			}
-
-			if level == 0 {
-				if _, err = r.Read(remaining); err != nil {
-					return nil, err
-				}
-				if _, err = wr.Write(remaining); err != nil {
-					return nil, err
-				}
-			}
-		case right[0]:
-			match, err := matches(r, wr, []byte{c}, right)
-			if err != nil {
-				return nil, err
-			}
-			if match {
-				level -= 1
-			}
-		default:
-			if err = wr.WriteByte(c); err != nil {
-				return nil, err
-			}
 		}
 
-		if level == 0 && !unicode.IsSpace(rune(c)) {
+		if level == 0 {
+			// Consumes white spaces immediately behind frontmatter
 			if err = chompWhitespace(r); err != nil {
 				if err != io.EOF {
 					return nil, err
 				}
 			}
-			return wr.Bytes(), nil
+			return buf.Bytes(), nil
 		}
 	}
 }
 
-func matches_quick(buf, expected []byte) (ok bool, err error) {
-	return bytes.Equal(expected, buf), nil
-}
-
-func matches(r *bufio.Reader, wr io.Writer, c, expected []byte) (ok bool, err error) {
-	if len(expected) == 1 {
-		if _, err = wr.Write(c); err != nil {
-			return
-		}
-		return bytes.Equal(c, expected), nil
-	}
-
-	buf := make([]byte, len(expected)-1)
-	if buf, err = r.Peek(len(expected) - 1); err != nil {
-		return
-	}
-
-	buf = append(c, buf...)
-	return bytes.Equal(expected, buf), nil
-}
-
 func extractContent(r io.Reader) (content Content, err error) {
 	wr := new(bytes.Buffer)
 	if _, err = wr.ReadFrom(r); err != nil {

parser/parse_frontmatter_test.go

@@ -54,7 +54,6 @@ func TestDegenerateCreatePageFrom(t *testing.T) {
 	}{
 		{CONTENT_MISSING_END_FM_DELIM},
 		{CONTENT_INCOMPLETE_END_FM_DELIM},
-		{CONTENT_FM_NO_DOC},
 	}
 
 	for _, test := range tests {
@@ -230,6 +229,7 @@ func TestExtractFrontMatter(t *testing.T) {
 		{"---\nfoobar\nbarfoo\nfizbaz\n", nil, false},
 		{"---\nblar\n-\n", nil, false},
 		{"---\nralb\n---\n", []byte("---\nralb\n---\n"), true},
+		{"---\neof\n---", []byte("---\neof\n---"), true},
 		{"---\nminc\n---\ncontent", []byte("---\nminc\n---\n"), true},
 		{"---\ncnim\n---\ncontent\n", []byte("---\ncnim\n---\n"), true},
 		{"---\ntitle: slug doc 2\nslug: slug-doc-2\n---\ncontent\n", []byte("---\ntitle: slug doc 2\nslug: slug-doc-2\n---\n"), true},
@@ -274,7 +274,6 @@ func TestExtractFrontMatterDelim(t *testing.T) {
 		{"", "", errExpected},
 		{"{", "", errExpected},
 		{"{}", "{}", noErrExpected},
-		{" {}", " {}", noErrExpected},
 		{"{} ", "{}", noErrExpected},
 		{"{ } ", "{ }", noErrExpected},
 		{"{ { }", "", errExpected},