langs/i18n: Upgrade to go-i18n v2

Fixes #5242

common/hreflect/helpers.go

@@ -22,6 +22,42 @@ import (
 	"github.com/gohugoio/hugo/common/types"
 )
 
+// TODO(bep) replace the private versions in /tpl with these.
+// IsInt returns whether the given kind is a number.
+func IsNumber(kind reflect.Kind) bool {
+	return IsInt(kind) || IsUint(kind) || IsFloat(kind)
+}
+
+// IsInt returns whether the given kind is an int.
+func IsInt(kind reflect.Kind) bool {
+	switch kind {
+	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+		return true
+	default:
+		return false
+	}
+}
+
+// IsUint returns whether the given kind is an uint.
+func IsUint(kind reflect.Kind) bool {
+	switch kind {
+	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
+		return true
+	default:
+		return false
+	}
+}
+
+// IsFloat returns whether the given kind is a float.
+func IsFloat(kind reflect.Kind) bool {
+	switch kind {
+	case reflect.Float32, reflect.Float64:
+		return true
+	default:
+		return false
+	}
+}
+
 // IsTruthful returns whether in represents a truthful value.
 // See IsTruthfulValue
 func IsTruthful(in interface{}) bool {

deps/deps.go

@@ -66,7 +66,7 @@ type Deps struct {
 	FileCaches filecache.Caches
 
 	// The translation func to use
-	Translate func(translationID string, args ...interface{}) string `json:"-"`
+	Translate func(translationID string, templateData interface{}) string `json:"-"`
 
 	// The language in use. TODO(bep) consolidate with site
 	Language *langs.Language

go.mod

@@ -35,7 +35,7 @@ require (
 	github.com/mitchellh/mapstructure v1.3.3
 	github.com/muesli/smartcrop v0.3.0
 	github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 // indirect
-	github.com/nicksnyder/go-i18n v1.10.1
+	github.com/nicksnyder/go-i18n/v2 v2.1.1
 	github.com/niklasfasching/go-org v1.3.2
 	github.com/olekukonko/tablewriter v0.0.4
 	github.com/pelletier/go-toml v1.6.0 // indirect

go.sum

@@ -352,6 +352,8 @@ github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6
 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8=
 github.com/nicksnyder/go-i18n v1.10.1 h1:isfg77E/aCD7+0lD/D00ebR2MV5vgeQ276WYyDaCRQc=
 github.com/nicksnyder/go-i18n v1.10.1/go.mod h1:e4Di5xjP9oTVrC6y3C7C0HoSYXjSbhh/dU0eUV32nB4=
+github.com/nicksnyder/go-i18n/v2 v2.1.1 h1:ATCOanRDlrfKVB4WHAdJnLEqZtDmKYsweqsOUYflnBU=
+github.com/nicksnyder/go-i18n/v2 v2.1.1/go.mod h1:d++QJC9ZVf7pa48qrsRWhMJ5pSHIPmS3OLqK1niyLxs=
 github.com/niklasfasching/go-org v1.3.2 h1:ZKTSd+GdJYkoZl1pBXLR/k7DRiRXnmB96TRiHmHdzwI=
 github.com/niklasfasching/go-org v1.3.2/go.mod h1:AsLD6X7djzRIz4/RFZu8vwRL0VGjUvGZCCH1Nz0VdrU=
 github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229/go.mod h1:0aYXnNPJ8l7uZxf45rWW1a/uME32OF0rhiYGNQ2oF2E=

langs/i18n/i18n.go

@@ -14,35 +14,40 @@
 package i18n
 
 import (
+	"reflect"
+	"strings"
+
+	"github.com/gohugoio/hugo/common/hreflect"
 	"github.com/gohugoio/hugo/common/loggers"
 	"github.com/gohugoio/hugo/config"
 	"github.com/gohugoio/hugo/helpers"
 
-	"github.com/nicksnyder/go-i18n/i18n/bundle"
+	"github.com/nicksnyder/go-i18n/v2/i18n"
-	"github.com/nicksnyder/go-i18n/i18n/translation"
 )
 
+type translateFunc func(translationID string, templateData interface{}) string
+
 var (
 	i18nWarningLogger = helpers.NewDistinctFeedbackLogger()
 )
 
 // Translator handles i18n translations.
 type Translator struct {
-	translateFuncs map[string]bundle.TranslateFunc
+	translateFuncs map[string]translateFunc
 	cfg            config.Provider
 	logger         *loggers.Logger
 }
 
 // NewTranslator creates a new Translator for the given language bundle and configuration.
-func NewTranslator(b *bundle.Bundle, cfg config.Provider, logger *loggers.Logger) Translator {
+func NewTranslator(b *i18n.Bundle, cfg config.Provider, logger *loggers.Logger) Translator {
-	t := Translator{cfg: cfg, logger: logger, translateFuncs: make(map[string]bundle.TranslateFunc)}
+	t := Translator{cfg: cfg, logger: logger, translateFuncs: make(map[string]translateFunc)}
 	t.initFuncs(b)
 	return t
 }
 
 // Func gets the translate func for the given language, or for the default
 // configured language if not found.
-func (t Translator) Func(lang string) bundle.TranslateFunc {
+func (t Translator) Func(lang string) translateFunc {
 	if f, ok := t.translateFuncs[lang]; ok {
 		return f
 	}
@@ -50,68 +55,57 @@ func (t Translator) Func(lang string) bundle.TranslateFunc {
 	if f, ok := t.translateFuncs[t.cfg.GetString("defaultContentLanguage")]; ok {
 		return f
 	}
+
 	t.logger.INFO.Println("i18n not initialized; if you need string translations, check that you have a bundle in /i18n that matches the site language or the default language.")
-	return func(translationID string, args ...interface{}) string {
+	return func(translationID string, args interface{}) string {
 		return ""
 	}
 
 }
 
-func (t Translator) initFuncs(bndl *bundle.Bundle) {
+func (t Translator) initFuncs(bndl *i18n.Bundle) {
-	defaultContentLanguage := t.cfg.GetString("defaultContentLanguage")
-
-	defaultT, err := bndl.Tfunc(defaultContentLanguage)
-	if err != nil {
-		t.logger.INFO.Printf("No translation bundle found for default language %q", defaultContentLanguage)
-	}
-
-	translations := bndl.Translations()
-
 	enableMissingTranslationPlaceholders := t.cfg.GetBool("enableMissingTranslationPlaceholders")
 	for _, lang := range bndl.LanguageTags() {
+
 		currentLang := lang
+		currentLangStr := currentLang.String()
+		currentLangKey := strings.TrimPrefix(currentLangStr, artificialLangTagPrefix)
+		localizer := i18n.NewLocalizer(bndl, currentLangStr)
 
-		t.translateFuncs[currentLang] = func(translationID string, args ...interface{}) string {
+		t.translateFuncs[currentLangKey] = func(translationID string, templateData interface{}) string {
-			tFunc, err := bndl.Tfunc(currentLang)
+
-			if err != nil {
+			if templateData != nil {
-				t.logger.WARN.Printf("could not load translations for language %q (%s), will use default content language.\n", lang, err)
+				tp := reflect.TypeOf(templateData)
+				if hreflect.IsNumber(tp.Kind()) {
+					// This was how go-i18n worked in v1.
+					templateData = map[string]interface{}{
+						"Count": templateData,
+					}
+				}
 			}
 
-			translated := tFunc(translationID, args...)
+			translated, translatedLang, err := localizer.LocalizeWithTag(&i18n.LocalizeConfig{
-			if translated != translationID {
+				MessageID:    translationID,
+				TemplateData: templateData,
+			})
+
+			if err == nil && currentLang == translatedLang {
 				return translated
 			}
-			// If there is no translation for translationID,
+
-			// then Tfunc returns translationID itself.
+			if _, ok := err.(*i18n.MessageNotFoundErr); !ok {
-			// But if user set same translationID and translation, we should check
+				t.logger.WARN.Printf("Failed to get translated string for language %q and ID %q: %s", currentLangStr, translationID, err)
-			// if it really untranslated:
-			if isIDTranslated(translations, currentLang, translationID) {
-				return translated
 			}
 
 			if t.cfg.GetBool("logI18nWarnings") {
-				i18nWarningLogger.Printf("i18n|MISSING_TRANSLATION|%s|%s", currentLang, translationID)
+				i18nWarningLogger.Printf("i18n|MISSING_TRANSLATION|%s|%s", currentLangStr, translationID)
 			}
+
 			if enableMissingTranslationPlaceholders {
 				return "[i18n] " + translationID
 			}
-			if defaultT != nil {
-				translated := defaultT(translationID, args...)
-				if translated != translationID {
-					return translated
-				}
-				if isIDTranslated(translations, defaultContentLanguage, translationID) {
-					return translated
-				}
-			}
-			return ""
-		}
-	}
-}
 
-// If the translation map contains translationID for specified currentLang,
+			return translated
-// then the translationID is actually translated.
+		}
-func isIDTranslated(translations map[string]map[string]translation.Translation, lang, id string) bool {
+	}
-	_, contains := translations[lang][id]
-	return contains
 }

langs/i18n/translationProvider.go

@@ -14,16 +14,19 @@
 package i18n
 
 import (
-	"errors"
+	"encoding/json"
 
 	"github.com/gohugoio/hugo/common/herrors"
+	"golang.org/x/text/language"
+	yaml "gopkg.in/yaml.v2"
+
+	"github.com/BurntSushi/toml"
+	"github.com/gohugoio/hugo/helpers"
+	"github.com/nicksnyder/go-i18n/v2/i18n"
 
 	"github.com/gohugoio/hugo/deps"
-	"github.com/gohugoio/hugo/helpers"
 	"github.com/gohugoio/hugo/hugofs"
 	"github.com/gohugoio/hugo/source"
-	"github.com/nicksnyder/go-i18n/i18n/bundle"
-	"github.com/nicksnyder/go-i18n/i18n/language"
 	_errors "github.com/pkg/errors"
 )
 
@@ -42,13 +45,10 @@ func NewTranslationProvider() *TranslationProvider {
 func (tp *TranslationProvider) Update(d *deps.Deps) error {
 	spec := source.NewSourceSpec(d.PathSpec, nil)
 
-	i18nBundle := bundle.New()
+	bundle := i18n.NewBundle(language.English)
-
+	bundle.RegisterUnmarshalFunc("toml", toml.Unmarshal)
-	en := language.GetPluralSpec("en")
+	bundle.RegisterUnmarshalFunc("yaml", yaml.Unmarshal)
-	if en == nil {
+	bundle.RegisterUnmarshalFunc("json", json.Unmarshal)
-		return errors.New("the English language has vanished like an old oak table")
-	}
-	var newLangs []string
 
 	// The source dirs are ordered so the most important comes first. Since this is a
 	// last key win situation, we have to reverse the iteration order.
@@ -56,33 +56,18 @@ func (tp *TranslationProvider) Update(d *deps.Deps) error {
 	for i := len(dirs) - 1; i >= 0; i-- {
 		dir := dirs[i]
 		src := spec.NewFilesystemFromFileMetaInfo(dir)
-
 		files, err := src.Files()
 		if err != nil {
 			return err
 		}
-
-		for _, r := range files {
-			currentSpec := language.GetPluralSpec(r.BaseFileName())
-			if currentSpec == nil {
-				// This may is a language code not supported by go-i18n, it may be
-				// Klingon or ... not even a fake language. Make sure it works.
-				newLangs = append(newLangs, r.BaseFileName())
-			}
-		}
-
-		if len(newLangs) > 0 {
-			language.RegisterPluralSpec(newLangs, en)
-		}
-
 		for _, file := range files {
-			if err := addTranslationFile(i18nBundle, file); err != nil {
+			if err := addTranslationFile(bundle, file); err != nil {
 				return err
 			}
 		}
 	}
 
-	tp.t = NewTranslator(i18nBundle, d.Cfg, d.Log)
+	tp.t = NewTranslator(bundle, d.Cfg, d.Log)
 
 	d.Translate = tp.t.Func(d.Language.Lang)
 
@@ -90,16 +75,29 @@ func (tp *TranslationProvider) Update(d *deps.Deps) error {
 
 }
 
-func addTranslationFile(bundle *bundle.Bundle, r source.File) error {
+const artificialLangTagPrefix = "art-x-"
+
+func addTranslationFile(bundle *i18n.Bundle, r source.File) error {
 	f, err := r.FileInfo().Meta().Open()
 	if err != nil {
 		return _errors.Wrapf(err, "failed to open translations file %q:", r.LogicalName())
 	}
-	err = bundle.ParseTranslationFileBytes(r.LogicalName(), helpers.ReaderToBytes(f))
+
+	b := helpers.ReaderToBytes(f)
 	f.Close()
+
+	name := r.LogicalName()
+	lang := helpers.Filename(name)
+	tag := language.Make(lang)
+	if tag == language.Und {
+		name = artificialLangTagPrefix + name
+	}
+
+	_, err = bundle.ParseMessageFileBytes(b, name)
 	if err != nil {
 		return errWithFileContext(_errors.Wrapf(err, "failed to load translations"), r)
 	}
+
 	return nil
 }
 

tpl/lang/lang.go

@@ -15,12 +15,13 @@
 package lang
 
 import (
-	"errors"
 	"fmt"
 	"math"
 	"strconv"
 	"strings"
 
+	"github.com/pkg/errors"
+
 	"github.com/gohugoio/hugo/deps"
 	"github.com/spf13/cast"
 )
@@ -39,12 +40,21 @@ type Namespace struct {
 
 // Translate returns a translated string for id.
 func (ns *Namespace) Translate(id interface{}, args ...interface{}) (string, error) {
+	var templateData interface{}
+
+	if len(args) > 0 {
+		if len(args) > 1 {
+			return "", errors.Errorf("wrong number of arguments, expecting at most 2, got %d", len(args)+1)
+		}
+		templateData = args[0]
+	}
+
 	sid, err := cast.ToStringE(id)
 	if err != nil {
 		return "", nil
 	}
 
-	return ns.deps.Translate(sid, args...), nil
+	return ns.deps.Translate(sid, templateData), nil
 }
 
 // NumFmt formats a number with the given precision using the