2019-01-02 06:33:26 -05:00
|
|
|
// Copyright 2019 The Hugo Authors. All rights reserved.
|
2015-12-02 05:36:18 -05:00
|
|
|
//
|
|
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
// you may not use this file except in compliance with the License.
|
|
|
|
// You may obtain a copy of the License at
|
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
//
|
|
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
// See the License for the specific language governing permissions and
|
|
|
|
// limitations under the License.
|
|
|
|
|
|
|
|
package commands
|
|
|
|
|
|
|
|
import (
|
|
|
|
"bytes"
|
|
|
|
"errors"
|
2018-10-20 11:38:49 -04:00
|
|
|
"fmt"
|
2015-12-02 05:36:18 -05:00
|
|
|
"io/ioutil"
|
|
|
|
"os"
|
|
|
|
"path/filepath"
|
|
|
|
"regexp"
|
|
|
|
"strconv"
|
|
|
|
"strings"
|
|
|
|
"time"
|
2018-11-19 11:33:54 -05:00
|
|
|
"unicode"
|
2015-12-02 05:36:18 -05:00
|
|
|
|
2019-09-10 05:26:34 -04:00
|
|
|
"github.com/gohugoio/hugo/parser/pageparser"
|
|
|
|
|
Add Hugo Modules
This commit implements Hugo Modules.
This is a broad subject, but some keywords include:
* A new `module` configuration section where you can import almost anything. You can configure both your own file mounts nd the file mounts of the modules you import. This is the new recommended way of configuring what you earlier put in `configDir`, `staticDir` etc. And it also allows you to mount folders in non-Hugo-projects, e.g. the `SCSS` folder in the Bootstrap GitHub project.
* A module consists of a set of mounts to the standard 7 component types in Hugo: `static`, `content`, `layouts`, `data`, `assets`, `i18n`, and `archetypes`. Yes, Theme Components can now include content, which should be very useful, especially in bigger multilingual projects.
* Modules not in your local file cache will be downloaded automatically and even "hot replaced" while the server is running.
* Hugo Modules supports and encourages semver versioned modules, and uses the minimal version selection algorithm to resolve versions.
* A new set of CLI commands are provided to manage all of this: `hugo mod init`, `hugo mod get`, `hugo mod graph`, `hugo mod tidy`, and `hugo mod vendor`.
All of the above is backed by Go Modules.
Fixes #5973
Fixes #5996
Fixes #6010
Fixes #5911
Fixes #5940
Fixes #6074
Fixes #6082
Fixes #6092
2019-05-03 03:16:58 -04:00
|
|
|
"github.com/gohugoio/hugo/common/hugio"
|
|
|
|
|
2018-10-20 05:16:18 -04:00
|
|
|
"github.com/gohugoio/hugo/parser/metadecoders"
|
|
|
|
|
2019-11-21 15:59:38 -05:00
|
|
|
"github.com/gohugoio/hugo/common/maps"
|
2017-06-13 12:42:45 -04:00
|
|
|
"github.com/gohugoio/hugo/helpers"
|
|
|
|
"github.com/gohugoio/hugo/hugofs"
|
|
|
|
"github.com/gohugoio/hugo/hugolib"
|
|
|
|
"github.com/gohugoio/hugo/parser"
|
2017-06-13 13:07:35 -04:00
|
|
|
"github.com/spf13/afero"
|
|
|
|
"github.com/spf13/cobra"
|
2015-12-02 05:36:18 -05:00
|
|
|
jww "github.com/spf13/jwalterweatherman"
|
|
|
|
)
|
|
|
|
|
2018-04-10 03:19:26 -04:00
|
|
|
var _ cmder = (*importCmd)(nil)
|
2018-04-09 14:37:17 -04:00
|
|
|
|
|
|
|
type importCmd struct {
|
2018-04-10 03:19:26 -04:00
|
|
|
*baseCmd
|
2015-12-02 05:42:53 -05:00
|
|
|
}
|
|
|
|
|
2018-04-09 14:37:17 -04:00
|
|
|
func newImportCmd() *importCmd {
|
|
|
|
cc := &importCmd{}
|
|
|
|
|
2018-04-10 03:19:26 -04:00
|
|
|
cc.baseCmd = newBaseCmd(&cobra.Command{
|
2018-04-09 14:37:17 -04:00
|
|
|
Use: "import",
|
|
|
|
Short: "Import your site from others.",
|
|
|
|
Long: `Import your site from other web site generators like Jekyll.
|
2015-12-02 05:42:53 -05:00
|
|
|
|
|
|
|
Import requires a subcommand, e.g. ` + "`hugo import jekyll jekyll_root_path target_path`.",
|
2018-04-09 14:37:17 -04:00
|
|
|
RunE: nil,
|
2018-04-10 03:19:26 -04:00
|
|
|
})
|
2015-12-02 05:42:53 -05:00
|
|
|
|
2018-04-09 14:37:17 -04:00
|
|
|
importJekyllCmd := &cobra.Command{
|
|
|
|
Use: "jekyll",
|
|
|
|
Short: "hugo import from Jekyll",
|
|
|
|
Long: `hugo import from Jekyll.
|
2015-12-02 05:36:18 -05:00
|
|
|
|
|
|
|
Import from Jekyll requires two paths, e.g. ` + "`hugo import jekyll jekyll_root_path target_path`.",
|
2018-04-09 14:37:17 -04:00
|
|
|
RunE: cc.importFromJekyll,
|
|
|
|
}
|
|
|
|
|
|
|
|
importJekyllCmd.Flags().Bool("force", false, "allow import into non-empty target directory")
|
|
|
|
|
|
|
|
cc.cmd.AddCommand(importJekyllCmd)
|
|
|
|
|
|
|
|
return cc
|
2015-12-02 05:36:18 -05:00
|
|
|
}
|
|
|
|
|
2018-04-09 14:37:17 -04:00
|
|
|
func (i *importCmd) importFromJekyll(cmd *cobra.Command, args []string) error {
|
2015-12-02 05:36:18 -05:00
|
|
|
if len(args) < 2 {
|
2018-09-06 11:53:18 -04:00
|
|
|
return newUserError(`import from jekyll requires two paths, e.g. ` + "`hugo import jekyll jekyll_root_path target_path`.")
|
2015-12-02 05:36:18 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
jekyllRoot, err := filepath.Abs(filepath.Clean(args[0]))
|
|
|
|
if err != nil {
|
2018-09-06 11:53:18 -04:00
|
|
|
return newUserError("path error:", args[0])
|
2015-12-02 05:36:18 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
targetDir, err := filepath.Abs(filepath.Clean(args[1]))
|
|
|
|
if err != nil {
|
2018-09-06 11:53:18 -04:00
|
|
|
return newUserError("path error:", args[1])
|
2015-12-02 05:36:18 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
jww.INFO.Println("Import Jekyll from:", jekyllRoot, "to:", targetDir)
|
2015-12-03 01:51:48 -05:00
|
|
|
|
2016-09-11 04:48:31 -04:00
|
|
|
if strings.HasPrefix(filepath.Dir(targetDir), jekyllRoot) {
|
2018-09-06 11:53:18 -04:00
|
|
|
return newUserError("abort: target path should not be inside the Jekyll root")
|
2015-12-03 01:51:48 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
forceImport, _ := cmd.Flags().GetBool("force")
|
2016-03-01 19:06:02 -05:00
|
|
|
|
|
|
|
fs := afero.NewOsFs()
|
2018-04-09 14:37:17 -04:00
|
|
|
jekyllPostDirs, hasAnyPost := i.getJekyllDirInfo(fs, jekyllRoot)
|
2016-03-01 19:06:02 -05:00
|
|
|
if !hasAnyPost {
|
2018-09-06 11:53:18 -04:00
|
|
|
return errors.New("abort: jekyll root contains neither posts nor drafts")
|
2016-03-01 19:06:02 -05:00
|
|
|
}
|
|
|
|
|
2019-11-24 12:57:39 -05:00
|
|
|
err = i.createSiteFromJekyll(jekyllRoot, targetDir, jekyllPostDirs, forceImport)
|
2016-03-01 19:06:02 -05:00
|
|
|
|
2017-01-03 11:28:51 -05:00
|
|
|
if err != nil {
|
2016-03-01 19:06:02 -05:00
|
|
|
return newUserError(err)
|
2015-12-03 01:51:48 -05:00
|
|
|
}
|
|
|
|
|
2016-11-22 12:47:20 -05:00
|
|
|
jww.FEEDBACK.Println("Importing...")
|
2015-12-02 05:36:18 -05:00
|
|
|
|
|
|
|
fileCount := 0
|
Add Hugo Modules
This commit implements Hugo Modules.
This is a broad subject, but some keywords include:
* A new `module` configuration section where you can import almost anything. You can configure both your own file mounts nd the file mounts of the modules you import. This is the new recommended way of configuring what you earlier put in `configDir`, `staticDir` etc. And it also allows you to mount folders in non-Hugo-projects, e.g. the `SCSS` folder in the Bootstrap GitHub project.
* A module consists of a set of mounts to the standard 7 component types in Hugo: `static`, `content`, `layouts`, `data`, `assets`, `i18n`, and `archetypes`. Yes, Theme Components can now include content, which should be very useful, especially in bigger multilingual projects.
* Modules not in your local file cache will be downloaded automatically and even "hot replaced" while the server is running.
* Hugo Modules supports and encourages semver versioned modules, and uses the minimal version selection algorithm to resolve versions.
* A new set of CLI commands are provided to manage all of this: `hugo mod init`, `hugo mod get`, `hugo mod graph`, `hugo mod tidy`, and `hugo mod vendor`.
All of the above is backed by Go Modules.
Fixes #5973
Fixes #5996
Fixes #6010
Fixes #5911
Fixes #5940
Fixes #6074
Fixes #6082
Fixes #6092
2019-05-03 03:16:58 -04:00
|
|
|
callback := func(path string, fi hugofs.FileMetaInfo, err error) error {
|
2015-12-02 05:36:18 -05:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
if fi.IsDir() {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
relPath, err := filepath.Rel(jekyllRoot, path)
|
|
|
|
if err != nil {
|
2018-09-06 11:53:18 -04:00
|
|
|
return newUserError("get rel path error:", path)
|
2015-12-02 05:36:18 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
relPath = filepath.ToSlash(relPath)
|
2016-03-24 18:34:12 -04:00
|
|
|
draft := false
|
2015-12-02 05:36:18 -05:00
|
|
|
|
|
|
|
switch {
|
2016-03-01 19:06:02 -05:00
|
|
|
case strings.Contains(relPath, "_posts/"):
|
|
|
|
relPath = filepath.Join("content/post", strings.Replace(relPath, "_posts/", "", -1))
|
|
|
|
case strings.Contains(relPath, "_drafts/"):
|
|
|
|
relPath = filepath.Join("content/draft", strings.Replace(relPath, "_drafts/", "", -1))
|
2015-12-02 05:36:18 -05:00
|
|
|
draft = true
|
|
|
|
default:
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
fileCount++
|
2019-11-24 12:57:39 -05:00
|
|
|
return convertJekyllPost(path, relPath, targetDir, draft)
|
2015-12-02 05:36:18 -05:00
|
|
|
}
|
|
|
|
|
2016-03-01 19:06:02 -05:00
|
|
|
for jekyllPostDir, hasAnyPostInDir := range jekyllPostDirs {
|
|
|
|
if hasAnyPostInDir {
|
|
|
|
if err = helpers.SymbolicWalk(hugofs.Os, filepath.Join(jekyllRoot, jekyllPostDir), callback); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
2015-12-02 05:36:18 -05:00
|
|
|
}
|
2016-03-01 19:06:02 -05:00
|
|
|
|
|
|
|
jww.FEEDBACK.Println("Congratulations!", fileCount, "post(s) imported!")
|
|
|
|
jww.FEEDBACK.Println("Now, start Hugo by yourself:\n" +
|
|
|
|
"$ git clone https://github.com/spf13/herring-cove.git " + args[1] + "/themes/herring-cove")
|
|
|
|
jww.FEEDBACK.Println("$ cd " + args[1] + "\n$ hugo server --theme=herring-cove")
|
|
|
|
|
2015-12-02 05:42:53 -05:00
|
|
|
return nil
|
2015-12-02 05:36:18 -05:00
|
|
|
}
|
|
|
|
|
2018-04-09 14:37:17 -04:00
|
|
|
func (i *importCmd) getJekyllDirInfo(fs afero.Fs, jekyllRoot string) (map[string]bool, bool) {
|
2016-03-01 19:06:02 -05:00
|
|
|
postDirs := make(map[string]bool)
|
|
|
|
hasAnyPost := false
|
|
|
|
if entries, err := ioutil.ReadDir(jekyllRoot); err == nil {
|
|
|
|
for _, entry := range entries {
|
|
|
|
if entry.IsDir() {
|
|
|
|
subDir := filepath.Join(jekyllRoot, entry.Name())
|
2018-04-09 14:37:17 -04:00
|
|
|
if isPostDir, hasAnyPostInDir := i.retrieveJekyllPostDir(fs, subDir); isPostDir {
|
2016-03-01 19:06:02 -05:00
|
|
|
postDirs[entry.Name()] = hasAnyPostInDir
|
|
|
|
if hasAnyPostInDir {
|
|
|
|
hasAnyPost = true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return postDirs, hasAnyPost
|
|
|
|
}
|
|
|
|
|
2018-04-09 14:37:17 -04:00
|
|
|
func (i *importCmd) retrieveJekyllPostDir(fs afero.Fs, dir string) (bool, bool) {
|
2016-03-01 19:06:02 -05:00
|
|
|
if strings.HasSuffix(dir, "_posts") || strings.HasSuffix(dir, "_drafts") {
|
|
|
|
isEmpty, _ := helpers.IsEmpty(dir, fs)
|
|
|
|
return true, !isEmpty
|
|
|
|
}
|
|
|
|
|
|
|
|
if entries, err := ioutil.ReadDir(dir); err == nil {
|
|
|
|
for _, entry := range entries {
|
|
|
|
if entry.IsDir() {
|
|
|
|
subDir := filepath.Join(dir, entry.Name())
|
2018-04-09 14:37:17 -04:00
|
|
|
if isPostDir, hasAnyPost := i.retrieveJekyllPostDir(fs, subDir); isPostDir {
|
2016-03-01 19:06:02 -05:00
|
|
|
return isPostDir, hasAnyPost
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return false, true
|
|
|
|
}
|
|
|
|
|
2019-11-24 12:57:39 -05:00
|
|
|
func (i *importCmd) createSiteFromJekyll(jekyllRoot, targetDir string, jekyllPostDirs map[string]bool, force bool) error {
|
2017-01-10 04:55:03 -05:00
|
|
|
s, err := hugolib.NewSiteDefaultLang()
|
|
|
|
if err != nil {
|
2019-11-24 12:57:39 -05:00
|
|
|
return err
|
2017-01-10 04:55:03 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
fs := s.Fs.Source
|
2015-12-03 01:51:48 -05:00
|
|
|
if exists, _ := helpers.Exists(targetDir, fs); exists {
|
|
|
|
if isDir, _ := helpers.IsDir(targetDir, fs); !isDir {
|
2019-11-24 12:57:39 -05:00
|
|
|
return errors.New("target path \"" + targetDir + "\" exists but is not a directory")
|
2015-12-03 01:51:48 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
isEmpty, _ := helpers.IsEmpty(targetDir, fs)
|
|
|
|
|
|
|
|
if !isEmpty && !force {
|
2019-11-24 12:57:39 -05:00
|
|
|
return errors.New("target path \"" + targetDir + "\" exists and is not empty")
|
2015-12-03 01:51:48 -05:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-04-09 14:37:17 -04:00
|
|
|
jekyllConfig := i.loadJekyllConfig(fs, jekyllRoot)
|
2015-12-03 01:51:48 -05:00
|
|
|
|
2015-12-02 05:36:18 -05:00
|
|
|
mkdir(targetDir, "layouts")
|
|
|
|
mkdir(targetDir, "content")
|
|
|
|
mkdir(targetDir, "archetypes")
|
|
|
|
mkdir(targetDir, "static")
|
|
|
|
mkdir(targetDir, "data")
|
|
|
|
mkdir(targetDir, "themes")
|
|
|
|
|
2018-04-09 14:37:17 -04:00
|
|
|
i.createConfigFromJekyll(fs, targetDir, "yaml", jekyllConfig)
|
2015-12-02 05:36:18 -05:00
|
|
|
|
2018-04-09 14:37:17 -04:00
|
|
|
i.copyJekyllFilesAndFolders(jekyllRoot, filepath.Join(targetDir, "static"), jekyllPostDirs)
|
2015-12-03 01:51:48 -05:00
|
|
|
|
2019-11-24 12:57:39 -05:00
|
|
|
return nil
|
2015-12-02 05:36:18 -05:00
|
|
|
}
|
|
|
|
|
2022-03-17 17:03:27 -04:00
|
|
|
func (i *importCmd) loadJekyllConfig(fs afero.Fs, jekyllRoot string) map[string]any {
|
2015-12-02 05:36:18 -05:00
|
|
|
path := filepath.Join(jekyllRoot, "_config.yml")
|
|
|
|
|
|
|
|
exists, err := helpers.Exists(path, fs)
|
|
|
|
|
|
|
|
if err != nil || !exists {
|
2015-12-03 01:51:48 -05:00
|
|
|
jww.WARN.Println("_config.yaml not found: Is the specified Jekyll root correct?")
|
2015-12-02 05:36:18 -05:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
f, err := fs.Open(path)
|
|
|
|
if err != nil {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
defer f.Close()
|
|
|
|
|
|
|
|
b, err := ioutil.ReadAll(f)
|
|
|
|
if err != nil {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2018-12-23 04:40:32 -05:00
|
|
|
c, err := metadecoders.Default.UnmarshalToMap(b, metadecoders.YAML)
|
2015-12-02 05:36:18 -05:00
|
|
|
if err != nil {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2018-01-25 22:54:15 -05:00
|
|
|
return c
|
2015-12-02 05:36:18 -05:00
|
|
|
}
|
|
|
|
|
2022-03-17 17:03:27 -04:00
|
|
|
func (i *importCmd) createConfigFromJekyll(fs afero.Fs, inpath string, kind metadecoders.Format, jekyllConfig map[string]any) (err error) {
|
2015-12-02 05:36:18 -05:00
|
|
|
title := "My New Hugo Site"
|
2016-10-24 14:56:00 -04:00
|
|
|
baseURL := "http://example.org/"
|
2015-12-02 05:36:18 -05:00
|
|
|
|
|
|
|
for key, value := range jekyllConfig {
|
|
|
|
lowerKey := strings.ToLower(key)
|
|
|
|
|
|
|
|
switch lowerKey {
|
|
|
|
case "title":
|
|
|
|
if str, ok := value.(string); ok {
|
|
|
|
title = str
|
|
|
|
}
|
|
|
|
|
|
|
|
case "url":
|
|
|
|
if str, ok := value.(string); ok {
|
2016-10-24 14:56:00 -04:00
|
|
|
baseURL = str
|
2015-12-02 05:36:18 -05:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-03-17 17:03:27 -04:00
|
|
|
in := map[string]any{
|
2016-10-24 14:56:00 -04:00
|
|
|
"baseURL": baseURL,
|
2015-12-02 05:36:18 -05:00
|
|
|
"title": title,
|
|
|
|
"languageCode": "en-us",
|
|
|
|
"disablePathToLower": true,
|
|
|
|
}
|
|
|
|
|
2016-12-26 16:23:20 -05:00
|
|
|
var buf bytes.Buffer
|
2018-10-20 11:38:49 -04:00
|
|
|
err = parser.InterfaceToConfig(in, kind, &buf)
|
2015-12-02 05:36:18 -05:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2018-10-20 11:38:49 -04:00
|
|
|
return helpers.WriteToDisk(filepath.Join(inpath, "config."+string(kind)), &buf, fs)
|
2015-12-02 05:36:18 -05:00
|
|
|
}
|
|
|
|
|
2018-04-09 14:37:17 -04:00
|
|
|
func (i *importCmd) copyJekyllFilesAndFolders(jekyllRoot, dest string, jekyllPostDirs map[string]bool) (err error) {
|
Add Hugo Modules
This commit implements Hugo Modules.
This is a broad subject, but some keywords include:
* A new `module` configuration section where you can import almost anything. You can configure both your own file mounts nd the file mounts of the modules you import. This is the new recommended way of configuring what you earlier put in `configDir`, `staticDir` etc. And it also allows you to mount folders in non-Hugo-projects, e.g. the `SCSS` folder in the Bootstrap GitHub project.
* A module consists of a set of mounts to the standard 7 component types in Hugo: `static`, `content`, `layouts`, `data`, `assets`, `i18n`, and `archetypes`. Yes, Theme Components can now include content, which should be very useful, especially in bigger multilingual projects.
* Modules not in your local file cache will be downloaded automatically and even "hot replaced" while the server is running.
* Hugo Modules supports and encourages semver versioned modules, and uses the minimal version selection algorithm to resolve versions.
* A new set of CLI commands are provided to manage all of this: `hugo mod init`, `hugo mod get`, `hugo mod graph`, `hugo mod tidy`, and `hugo mod vendor`.
All of the above is backed by Go Modules.
Fixes #5973
Fixes #5996
Fixes #6010
Fixes #5911
Fixes #5940
Fixes #6074
Fixes #6082
Fixes #6092
2019-05-03 03:16:58 -04:00
|
|
|
fs := hugofs.Os
|
|
|
|
|
|
|
|
fi, err := fs.Stat(jekyllRoot)
|
2015-12-02 05:36:18 -05:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
if !fi.IsDir() {
|
|
|
|
return errors.New(jekyllRoot + " is not a directory")
|
|
|
|
}
|
|
|
|
err = os.MkdirAll(dest, fi.Mode())
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
entries, err := ioutil.ReadDir(jekyllRoot)
|
2019-01-02 06:33:26 -05:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2015-12-02 05:36:18 -05:00
|
|
|
for _, entry := range entries {
|
|
|
|
sfp := filepath.Join(jekyllRoot, entry.Name())
|
|
|
|
dfp := filepath.Join(dest, entry.Name())
|
|
|
|
if entry.IsDir() {
|
|
|
|
if entry.Name()[0] != '_' && entry.Name()[0] != '.' {
|
2016-03-01 19:06:02 -05:00
|
|
|
if _, ok := jekyllPostDirs[entry.Name()]; !ok {
|
Add Hugo Modules
This commit implements Hugo Modules.
This is a broad subject, but some keywords include:
* A new `module` configuration section where you can import almost anything. You can configure both your own file mounts nd the file mounts of the modules you import. This is the new recommended way of configuring what you earlier put in `configDir`, `staticDir` etc. And it also allows you to mount folders in non-Hugo-projects, e.g. the `SCSS` folder in the Bootstrap GitHub project.
* A module consists of a set of mounts to the standard 7 component types in Hugo: `static`, `content`, `layouts`, `data`, `assets`, `i18n`, and `archetypes`. Yes, Theme Components can now include content, which should be very useful, especially in bigger multilingual projects.
* Modules not in your local file cache will be downloaded automatically and even "hot replaced" while the server is running.
* Hugo Modules supports and encourages semver versioned modules, and uses the minimal version selection algorithm to resolve versions.
* A new set of CLI commands are provided to manage all of this: `hugo mod init`, `hugo mod get`, `hugo mod graph`, `hugo mod tidy`, and `hugo mod vendor`.
All of the above is backed by Go Modules.
Fixes #5973
Fixes #5996
Fixes #6010
Fixes #5911
Fixes #5940
Fixes #6074
Fixes #6082
Fixes #6092
2019-05-03 03:16:58 -04:00
|
|
|
err = hugio.CopyDir(fs, sfp, dfp, nil)
|
2016-03-01 19:06:02 -05:00
|
|
|
if err != nil {
|
|
|
|
jww.ERROR.Println(err)
|
|
|
|
}
|
2015-12-02 05:36:18 -05:00
|
|
|
}
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
lowerEntryName := strings.ToLower(entry.Name())
|
2020-12-02 07:23:25 -05:00
|
|
|
exceptSuffix := []string{
|
|
|
|
".md", ".markdown", ".html", ".htm",
|
|
|
|
".xml", ".textile", "rakefile", "gemfile", ".lock",
|
|
|
|
}
|
2015-12-02 05:36:18 -05:00
|
|
|
isExcept := false
|
|
|
|
for _, suffix := range exceptSuffix {
|
|
|
|
if strings.HasSuffix(lowerEntryName, suffix) {
|
|
|
|
isExcept = true
|
|
|
|
break
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if !isExcept && entry.Name()[0] != '.' && entry.Name()[0] != '_' {
|
Add Hugo Modules
This commit implements Hugo Modules.
This is a broad subject, but some keywords include:
* A new `module` configuration section where you can import almost anything. You can configure both your own file mounts nd the file mounts of the modules you import. This is the new recommended way of configuring what you earlier put in `configDir`, `staticDir` etc. And it also allows you to mount folders in non-Hugo-projects, e.g. the `SCSS` folder in the Bootstrap GitHub project.
* A module consists of a set of mounts to the standard 7 component types in Hugo: `static`, `content`, `layouts`, `data`, `assets`, `i18n`, and `archetypes`. Yes, Theme Components can now include content, which should be very useful, especially in bigger multilingual projects.
* Modules not in your local file cache will be downloaded automatically and even "hot replaced" while the server is running.
* Hugo Modules supports and encourages semver versioned modules, and uses the minimal version selection algorithm to resolve versions.
* A new set of CLI commands are provided to manage all of this: `hugo mod init`, `hugo mod get`, `hugo mod graph`, `hugo mod tidy`, and `hugo mod vendor`.
All of the above is backed by Go Modules.
Fixes #5973
Fixes #5996
Fixes #6010
Fixes #5911
Fixes #5940
Fixes #6074
Fixes #6082
Fixes #6092
2019-05-03 03:16:58 -04:00
|
|
|
err = hugio.CopyFile(fs, sfp, dfp)
|
2015-12-02 05:36:18 -05:00
|
|
|
if err != nil {
|
|
|
|
jww.ERROR.Println(err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func parseJekyllFilename(filename string) (time.Time, string, error) {
|
|
|
|
re := regexp.MustCompile(`(\d+-\d+-\d+)-(.+)\..*`)
|
|
|
|
r := re.FindAllStringSubmatch(filename, -1)
|
|
|
|
if len(r) == 0 {
|
|
|
|
return time.Now(), "", errors.New("filename not match")
|
|
|
|
}
|
|
|
|
|
2017-03-13 07:50:57 -04:00
|
|
|
postDate, err := time.Parse("2006-1-2", r[0][1])
|
2015-12-02 05:36:18 -05:00
|
|
|
if err != nil {
|
|
|
|
return time.Now(), "", err
|
|
|
|
}
|
|
|
|
|
|
|
|
postName := r[0][2]
|
|
|
|
|
|
|
|
return postDate, postName, nil
|
|
|
|
}
|
|
|
|
|
2019-11-24 12:57:39 -05:00
|
|
|
func convertJekyllPost(path, relPath, targetDir string, draft bool) error {
|
2015-12-02 05:36:18 -05:00
|
|
|
jww.TRACE.Println("Converting", path)
|
|
|
|
|
|
|
|
filename := filepath.Base(path)
|
|
|
|
postDate, postName, err := parseJekyllFilename(filename)
|
|
|
|
if err != nil {
|
2015-12-14 09:30:58 -05:00
|
|
|
jww.WARN.Printf("Failed to parse filename '%s': %s. Skipping.", filename, err)
|
|
|
|
return nil
|
2015-12-02 05:36:18 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
jww.TRACE.Println(filename, postDate, postName)
|
|
|
|
|
|
|
|
targetFile := filepath.Join(targetDir, relPath)
|
|
|
|
targetParentDir := filepath.Dir(targetFile)
|
|
|
|
os.MkdirAll(targetParentDir, 0777)
|
|
|
|
|
|
|
|
contentBytes, err := ioutil.ReadFile(path)
|
|
|
|
if err != nil {
|
|
|
|
jww.ERROR.Println("Read file error:", path)
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2019-09-10 05:26:34 -04:00
|
|
|
pf, err := pageparser.ParseFrontMatterAndContent(bytes.NewReader(contentBytes))
|
2015-12-02 05:36:18 -05:00
|
|
|
if err != nil {
|
|
|
|
jww.ERROR.Println("Parse file error:", path)
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2019-09-10 05:26:34 -04:00
|
|
|
newmetadata, err := convertJekyllMetaData(pf.FrontMatter, postName, postDate, draft)
|
2015-12-02 05:36:18 -05:00
|
|
|
if err != nil {
|
|
|
|
jww.ERROR.Println("Convert metadata error:", path)
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2019-09-10 05:26:34 -04:00
|
|
|
content, err := convertJekyllContent(newmetadata, string(pf.Content))
|
2019-11-24 12:57:39 -05:00
|
|
|
if err != nil {
|
|
|
|
jww.ERROR.Println("Converting Jekyll error:", path)
|
|
|
|
return err
|
|
|
|
}
|
2015-12-02 05:36:18 -05:00
|
|
|
|
2018-10-20 11:38:49 -04:00
|
|
|
fs := hugofs.Os
|
|
|
|
if err := helpers.WriteToDisk(targetFile, strings.NewReader(content), fs); err != nil {
|
2019-01-02 06:33:26 -05:00
|
|
|
return fmt.Errorf("failed to save file %q: %s", filename, err)
|
2015-12-02 05:36:18 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2022-03-17 17:03:27 -04:00
|
|
|
func convertJekyllMetaData(m any, postName string, postDate time.Time, draft bool) (any, error) {
|
2019-11-21 15:59:38 -05:00
|
|
|
metadata, err := maps.ToStringMapE(m)
|
2015-12-02 05:36:18 -05:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
if draft {
|
|
|
|
metadata["draft"] = true
|
|
|
|
}
|
|
|
|
|
|
|
|
for key, value := range metadata {
|
|
|
|
lowerKey := strings.ToLower(key)
|
|
|
|
|
|
|
|
switch lowerKey {
|
|
|
|
case "layout":
|
|
|
|
delete(metadata, key)
|
|
|
|
case "permalink":
|
|
|
|
if str, ok := value.(string); ok {
|
2018-01-11 11:42:49 -05:00
|
|
|
metadata["url"] = str
|
2015-12-02 05:36:18 -05:00
|
|
|
}
|
|
|
|
delete(metadata, key)
|
|
|
|
case "category":
|
|
|
|
if str, ok := value.(string); ok {
|
|
|
|
metadata["categories"] = []string{str}
|
|
|
|
}
|
|
|
|
delete(metadata, key)
|
|
|
|
case "excerpt_separator":
|
|
|
|
if key != lowerKey {
|
|
|
|
delete(metadata, key)
|
|
|
|
metadata[lowerKey] = value
|
|
|
|
}
|
|
|
|
case "date":
|
|
|
|
if str, ok := value.(string); ok {
|
|
|
|
re := regexp.MustCompile(`(\d+):(\d+):(\d+)`)
|
|
|
|
r := re.FindAllStringSubmatch(str, -1)
|
|
|
|
if len(r) > 0 {
|
|
|
|
hour, _ := strconv.Atoi(r[0][1])
|
|
|
|
minute, _ := strconv.Atoi(r[0][2])
|
|
|
|
second, _ := strconv.Atoi(r[0][3])
|
|
|
|
postDate = time.Date(postDate.Year(), postDate.Month(), postDate.Day(), hour, minute, second, 0, time.UTC)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
delete(metadata, key)
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
metadata["date"] = postDate.Format(time.RFC3339)
|
|
|
|
|
|
|
|
return metadata, nil
|
|
|
|
}
|
|
|
|
|
2022-03-17 17:03:27 -04:00
|
|
|
func convertJekyllContent(m any, content string) (string, error) {
|
2019-11-21 15:59:38 -05:00
|
|
|
metadata, _ := maps.ToStringMapE(m)
|
2015-12-02 05:36:18 -05:00
|
|
|
|
|
|
|
lines := strings.Split(content, "\n")
|
|
|
|
var resultLines []string
|
|
|
|
for _, line := range lines {
|
|
|
|
resultLines = append(resultLines, strings.Trim(line, "\r\n"))
|
|
|
|
}
|
|
|
|
|
|
|
|
content = strings.Join(resultLines, "\n")
|
|
|
|
|
|
|
|
excerptSep := "<!--more-->"
|
|
|
|
if value, ok := metadata["excerpt_separator"]; ok {
|
|
|
|
if str, strOk := value.(string); strOk {
|
|
|
|
content = strings.Replace(content, strings.TrimSpace(str), excerptSep, -1)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
replaceList := []struct {
|
|
|
|
re *regexp.Regexp
|
|
|
|
replace string
|
|
|
|
}{
|
2017-03-18 11:16:25 -04:00
|
|
|
{regexp.MustCompile("(?i)<!-- more -->"), "<!--more-->"},
|
2015-12-02 05:36:18 -05:00
|
|
|
{regexp.MustCompile(`\{%\s*raw\s*%\}\s*(.*?)\s*\{%\s*endraw\s*%\}`), "$1"},
|
|
|
|
{regexp.MustCompile(`{%\s*endhighlight\s*%}`), "{{< / highlight >}}"},
|
|
|
|
}
|
|
|
|
|
|
|
|
for _, replace := range replaceList {
|
|
|
|
content = replace.re.ReplaceAllString(content, replace.replace)
|
|
|
|
}
|
|
|
|
|
2017-04-22 16:35:52 -04:00
|
|
|
replaceListFunc := []struct {
|
|
|
|
re *regexp.Regexp
|
|
|
|
replace func(string) string
|
|
|
|
}{
|
|
|
|
// Octopress image tag: http://octopress.org/docs/plugins/image-tag/
|
|
|
|
{regexp.MustCompile(`{%\s+img\s*(.*?)\s*%}`), replaceImageTag},
|
2018-11-19 11:33:54 -05:00
|
|
|
{regexp.MustCompile(`{%\s*highlight\s*(.*?)\s*%}`), replaceHighlightTag},
|
2017-04-22 16:35:52 -04:00
|
|
|
}
|
|
|
|
|
|
|
|
for _, replace := range replaceListFunc {
|
|
|
|
content = replace.re.ReplaceAllStringFunc(content, replace.replace)
|
|
|
|
}
|
|
|
|
|
2019-11-24 12:57:39 -05:00
|
|
|
var buf bytes.Buffer
|
|
|
|
if len(metadata) != 0 {
|
|
|
|
err := parser.InterfaceToFrontMatter(m, metadecoders.YAML, &buf)
|
|
|
|
if err != nil {
|
|
|
|
return "", err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
buf.WriteString(content)
|
|
|
|
|
|
|
|
return buf.String(), nil
|
2015-12-02 05:36:18 -05:00
|
|
|
}
|
2017-04-22 16:35:52 -04:00
|
|
|
|
2018-11-19 11:33:54 -05:00
|
|
|
func replaceHighlightTag(match string) string {
|
|
|
|
r := regexp.MustCompile(`{%\s*highlight\s*(.*?)\s*%}`)
|
|
|
|
parts := r.FindStringSubmatch(match)
|
|
|
|
lastQuote := rune(0)
|
|
|
|
f := func(c rune) bool {
|
|
|
|
switch {
|
|
|
|
case c == lastQuote:
|
|
|
|
lastQuote = rune(0)
|
|
|
|
return false
|
|
|
|
case lastQuote != rune(0):
|
|
|
|
return false
|
|
|
|
case unicode.In(c, unicode.Quotation_Mark):
|
|
|
|
lastQuote = c
|
|
|
|
return false
|
|
|
|
default:
|
|
|
|
return unicode.IsSpace(c)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
// splitting string by space but considering quoted section
|
|
|
|
items := strings.FieldsFunc(parts[1], f)
|
|
|
|
|
|
|
|
result := bytes.NewBufferString("{{< highlight ")
|
|
|
|
result.WriteString(items[0]) // language
|
|
|
|
options := items[1:]
|
|
|
|
for i, opt := range options {
|
|
|
|
opt = strings.Replace(opt, "\"", "", -1)
|
|
|
|
if opt == "linenos" {
|
|
|
|
opt = "linenos=table"
|
|
|
|
}
|
|
|
|
if i == 0 {
|
|
|
|
opt = " \"" + opt
|
|
|
|
}
|
|
|
|
if i < len(options)-1 {
|
|
|
|
opt += ","
|
|
|
|
} else if i == len(options)-1 {
|
|
|
|
opt += "\""
|
|
|
|
}
|
|
|
|
result.WriteString(opt)
|
|
|
|
}
|
|
|
|
|
|
|
|
result.WriteString(" >}}")
|
|
|
|
return result.String()
|
|
|
|
}
|
|
|
|
|
2017-04-22 16:35:52 -04:00
|
|
|
func replaceImageTag(match string) string {
|
|
|
|
r := regexp.MustCompile(`{%\s+img\s*(\p{L}*)\s+([\S]*/[\S]+)\s+(\d*)\s*(\d*)\s*(.*?)\s*%}`)
|
|
|
|
result := bytes.NewBufferString("{{< figure ")
|
|
|
|
parts := r.FindStringSubmatch(match)
|
|
|
|
// Index 0 is the entire string, ignore
|
|
|
|
replaceOptionalPart(result, "class", parts[1])
|
|
|
|
replaceOptionalPart(result, "src", parts[2])
|
|
|
|
replaceOptionalPart(result, "width", parts[3])
|
|
|
|
replaceOptionalPart(result, "height", parts[4])
|
|
|
|
// title + alt
|
|
|
|
part := parts[5]
|
|
|
|
if len(part) > 0 {
|
|
|
|
splits := strings.Split(part, "'")
|
|
|
|
lenSplits := len(splits)
|
|
|
|
if lenSplits == 1 {
|
|
|
|
replaceOptionalPart(result, "title", splits[0])
|
|
|
|
} else if lenSplits == 3 {
|
|
|
|
replaceOptionalPart(result, "title", splits[1])
|
|
|
|
} else if lenSplits == 5 {
|
|
|
|
replaceOptionalPart(result, "title", splits[1])
|
|
|
|
replaceOptionalPart(result, "alt", splits[3])
|
|
|
|
}
|
|
|
|
}
|
|
|
|
result.WriteString(">}}")
|
|
|
|
return result.String()
|
|
|
|
}
|
2020-12-02 07:23:25 -05:00
|
|
|
|
2017-04-22 16:35:52 -04:00
|
|
|
func replaceOptionalPart(buffer *bytes.Buffer, partName string, part string) {
|
|
|
|
if len(part) > 0 {
|
|
|
|
buffer.WriteString(partName + "=\"" + part + "\" ")
|
|
|
|
}
|
|
|
|
}
|