From 9a2f6c62a602fc701cc179ac9598092e85b342a8 Mon Sep 17 00:00:00 2001 From: coderzh Date: Thu, 1 Oct 2015 09:04:30 +0800 Subject: [PATCH] Hugo import from jekyll usage: hugo import jekyll jekyll_root_path target_path Implemented: * Create new hugo site * Create config.yaml * Convert all markdown contents. * Copy all other files and folders to static Fixes #101 --- commands/hugo.go | 1 + commands/import.go | 483 ++++++++++++++++++++++++++++++++++++++++ commands/import_test.go | 104 +++++++++ 3 files changed, 588 insertions(+) create mode 100644 commands/import.go create mode 100644 commands/import_test.go diff --git a/commands/hugo.go b/commands/hugo.go index 67ca8f19d..3fbbe0952 100644 --- a/commands/hugo.go +++ b/commands/hugo.go @@ -83,6 +83,7 @@ func AddCommands() { HugoCmd.AddCommand(undraftCmd) HugoCmd.AddCommand(genautocompleteCmd) HugoCmd.AddCommand(gendocCmd) + HugoCmd.AddCommand(importCmd) } //Initializes flags diff --git a/commands/import.go b/commands/import.go new file mode 100644 index 000000000..f6ca75a7a --- /dev/null +++ b/commands/import.go @@ -0,0 +1,483 @@ +// Copyright © 2015 Steve Francia . +// +// Licensed under the Simple Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://opensource.org/licenses/Simple-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package commands + +import ( + "bytes" + "errors" + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + "regexp" + "strconv" + "strings" + "time" + + "github.com/spf13/cast" + "github.com/spf13/cobra" + "github.com/spf13/hugo/helpers" + "github.com/spf13/hugo/hugofs" + "github.com/spf13/hugo/hugolib" + "github.com/spf13/hugo/parser" + jww "github.com/spf13/jwalterweatherman" +) + +func init() { + importCmd.AddCommand(importJekyllCmd) +} + +var importCmd = &cobra.Command{ + Use: "import", + Short: "import from others", + Long: `import from others like jekyll. + +Import requires a subcommand, e.g. ` + "`hugo import jekyll jekyll_root_path target_path`.", + Run: nil, +} + +var importJekyllCmd = &cobra.Command{ + Use: "jekyll", + Short: "hugo import from jekyll", + Long: `hugo import from jekyll. + +Import jekyll requires two path, e.g. ` + "`hugo import jekyll jekyll_root_path target_path`.", + Run: importFromJekyll, +} + +func importFromJekyll(cmd *cobra.Command, args []string) { + jww.SetLogThreshold(jww.LevelTrace) + jww.SetStdoutThreshold(jww.LevelWarn) + + if len(args) < 2 { + jww.ERROR.Println(`Import jekyll requires two path, e.g. ` + "`hugo import jekyll jekyll_root_path target_path`.") + return + } + + jekyllRoot, err := filepath.Abs(filepath.Clean(args[0])) + if err != nil { + jww.ERROR.Println("Path error:", args[0]) + return + } + + targetDir, err := filepath.Abs(filepath.Clean(args[1])) + if err != nil { + jww.ERROR.Println("Path error:", args[1]) + return + } + + createSiteFromJekyll(jekyllRoot, targetDir) + + jww.INFO.Println("Import jekyll from:", jekyllRoot, "to:", targetDir) + fmt.Println("Importing...") + + fileCount := 0 + callback := func(path string, fi os.FileInfo, err error) error { + if err != nil { + return err + } + + if fi.IsDir() { + return nil + } + + relPath, err := filepath.Rel(jekyllRoot, path) + if err != nil { + jww.ERROR.Println("Get rel path error:", path) + return err + } + + relPath = filepath.ToSlash(relPath) + var draft bool = false + + switch { + case strings.HasPrefix(relPath, "_posts/"): + relPath = "content/post" + relPath[len("_posts"):] + case strings.HasPrefix(relPath, "_drafts/"): + relPath = "content/draft" + relPath[len("_drafts"):] + draft = true + default: + return nil + } + + fileCount++ + return convertJekyllPost(path, relPath, targetDir, draft) + } + + err = filepath.Walk(jekyllRoot, callback) + + if err != nil { + fmt.Println(err) + } else { + fmt.Println("Congratulations!", fileCount, "posts imported!") + fmt.Println("Now, start hugo by yourself: \n" + + "$ git clone https://github.com/spf13/herring-cove.git " + args[1] + "/themes/herring-cove") + fmt.Println("$ cd " + args[1] + "\n$ hugo server -w --theme=herring-cove") + } +} + +func createSiteFromJekyll(jekyllRoot, targetDir string) { + mkdir(targetDir, "layouts") + mkdir(targetDir, "content") + mkdir(targetDir, "archetypes") + mkdir(targetDir, "static") + mkdir(targetDir, "data") + mkdir(targetDir, "themes") + + jekyllConfig := loadJekyllConfig(jekyllRoot) + createConfigFromJekyll(targetDir, "yaml", jekyllConfig) + + copyJekyllFilesAndFolders(jekyllRoot, filepath.Join(targetDir, "static")) +} + +func loadJekyllConfig(jekyllRoot string) map[string]interface{} { + fs := hugofs.SourceFs + path := filepath.Join(jekyllRoot, "_config.yml") + + exists, err := helpers.Exists(path, fs) + + if err != nil || !exists { + return nil + } + + f, err := fs.Open(path) + if err != nil { + return nil + } + + defer f.Close() + + b, err := ioutil.ReadAll(f) + + if err != nil { + return nil + } + + c, err := parser.HandleYAMLMetaData(b) + + if err != nil { + return nil + } + + return c.(map[string]interface{}) +} + +func createConfigFromJekyll(inpath string, kind string, jekyllConfig map[string]interface{}) (err error) { + title := "My New Hugo Site" + baseurl := "http://replace-this-with-your-hugo-site.com/" + + for key, value := range jekyllConfig { + lowerKey := strings.ToLower(key) + + switch lowerKey { + case "title": + if str, ok := value.(string); ok { + title = str + } + + case "url": + if str, ok := value.(string); ok { + baseurl = str + } + } + } + + in := map[string]interface{}{ + "baseurl": baseurl, + "title": title, + "languageCode": "en-us", + "disablePathToLower": true, + } + kind = parser.FormatSanitize(kind) + + by, err := parser.InterfaceToConfig(in, parser.FormatToLeadRune(kind)) + if err != nil { + return err + } + + err = helpers.WriteToDisk(filepath.Join(inpath, "config."+kind), bytes.NewReader(by), hugofs.SourceFs) + if err != nil { + return + } + + return nil +} + +func copyFile(source string, dest string) (err error) { + sf, err := os.Open(source) + if err != nil { + return err + } + defer sf.Close() + df, err := os.Create(dest) + if err != nil { + return err + } + defer df.Close() + _, err = io.Copy(df, sf) + if err == nil { + si, err := os.Stat(source) + if err != nil { + err = os.Chmod(dest, si.Mode()) + } + + } + return +} + +func copyDir(source string, dest string) (err error) { + fi, err := os.Stat(source) + if err != nil { + return err + } + if !fi.IsDir() { + return errors.New(source + " is not a directory") + } + err = os.MkdirAll(dest, fi.Mode()) + if err != nil { + return err + } + entries, err := ioutil.ReadDir(source) + for _, entry := range entries { + sfp := filepath.Join(source, entry.Name()) + dfp := filepath.Join(dest, entry.Name()) + if entry.IsDir() { + err = copyDir(sfp, dfp) + if err != nil { + jww.ERROR.Println(err) + } + } else { + err = copyFile(sfp, dfp) + if err != nil { + jww.ERROR.Println(err) + } + } + + } + return nil +} + +func copyJekyllFilesAndFolders(jekyllRoot string, dest string) (err error) { + fi, err := os.Stat(jekyllRoot) + if err != nil { + return err + } + if !fi.IsDir() { + return errors.New(jekyllRoot + " is not a directory") + } + err = os.MkdirAll(dest, fi.Mode()) + if err != nil { + return err + } + entries, err := ioutil.ReadDir(jekyllRoot) + for _, entry := range entries { + sfp := filepath.Join(jekyllRoot, entry.Name()) + dfp := filepath.Join(dest, entry.Name()) + if entry.IsDir() { + if entry.Name()[0] != '_' && entry.Name()[0] != '.' { + err = copyDir(sfp, dfp) + if err != nil { + jww.ERROR.Println(err) + } + } + } else { + lowerEntryName := strings.ToLower(entry.Name()) + exceptSuffix := []string{".md", ".markdown", ".html", ".htm", + ".xml", ".textile", "rakefile", "gemfile", ".lock"} + isExcept := false + for _, suffix := range exceptSuffix { + if strings.HasSuffix(lowerEntryName, suffix) { + isExcept = true + break + } + } + + if !isExcept && entry.Name()[0] != '.' && entry.Name()[0] != '_' { + err = copyFile(sfp, dfp) + if err != nil { + jww.ERROR.Println(err) + } + } + } + + } + return nil +} + +func parseJekyllFilename(filename string) (time.Time, string, error) { + re := regexp.MustCompile(`(\d+-\d+-\d+)-(.+)\..*`) + r := re.FindAllStringSubmatch(filename, -1) + if len(r) == 0 { + return time.Now(), "", errors.New("filename not match") + } + + postDate, err := time.Parse("2006-01-02", r[0][1]) + if err != nil { + return time.Now(), "", err + } + + postName := r[0][2] + + return postDate, postName, nil +} + +func convertJekyllPost(path, relPath, targetDir string, draft bool) error { + jww.TRACE.Println("Converting", path) + + filename := filepath.Base(path) + postDate, postName, err := parseJekyllFilename(filename) + if err != nil { + jww.ERROR.Println("Parse filename error:", filename) + return err + } + + jww.TRACE.Println(filename, postDate, postName) + + targetFile := filepath.Join(targetDir, relPath) + targetParentDir := filepath.Dir(targetFile) + os.MkdirAll(targetParentDir, 0777) + + contentBytes, err := ioutil.ReadFile(path) + if err != nil { + jww.ERROR.Println("Read file error:", path) + return err + } + + psr, err := parser.ReadFrom(bytes.NewReader(contentBytes)) + if err != nil { + jww.ERROR.Println("Parse file error:", path) + return err + } + + metadata, err := psr.Metadata() + if err != nil { + jww.ERROR.Println("Processing file error:", path) + return err + } + + newmetadata, err := convertJekyllMetaData(metadata, postName, postDate, draft) + if err != nil { + jww.ERROR.Println("Convert metadata error:", path) + return err + } + + jww.TRACE.Println(newmetadata) + content := convertJekyllContent(newmetadata, string(psr.Content())) + + page, err := hugolib.NewPage(filename) + if err != nil { + jww.ERROR.Println("New page error", filename) + return err + } + + page.SetDir(targetParentDir) + page.SetSourceContent([]byte(content)) + page.SetSourceMetaData(newmetadata, parser.FormatToLeadRune("yaml")) + page.SaveSourceAs(targetFile) + + jww.TRACE.Println("Target file:", targetFile) + + return nil +} + +func convertJekyllMetaData(m interface{}, postName string, postDate time.Time, draft bool) (interface{}, error) { + url := postDate.Format("/2006/01/02/") + postName + "/" + + metadata, err := cast.ToStringMapE(m) + if err != nil { + return nil, err + } + + if draft { + metadata["draft"] = true + } + + for key, value := range metadata { + lowerKey := strings.ToLower(key) + + switch lowerKey { + case "layout": + delete(metadata, key) + case "permalink": + if str, ok := value.(string); ok { + url = str + } + delete(metadata, key) + case "category": + if str, ok := value.(string); ok { + metadata["categories"] = []string{str} + } + delete(metadata, key) + case "excerpt_separator": + if key != lowerKey { + delete(metadata, key) + metadata[lowerKey] = value + } + case "date": + if str, ok := value.(string); ok { + re := regexp.MustCompile(`(\d+):(\d+):(\d+)`) + r := re.FindAllStringSubmatch(str, -1) + if len(r) > 0 { + hour, _ := strconv.Atoi(r[0][1]) + minute, _ := strconv.Atoi(r[0][2]) + second, _ := strconv.Atoi(r[0][3]) + postDate = time.Date(postDate.Year(), postDate.Month(), postDate.Day(), hour, minute, second, 0, time.UTC) + } + } + delete(metadata, key) + } + + } + + metadata["url"] = url + metadata["date"] = postDate.Format(time.RFC3339) + + return metadata, nil +} + +func convertJekyllContent(m interface{}, content string) string { + metadata, _ := cast.ToStringMapE(m) + + lines := strings.Split(content, "\n") + var resultLines []string + for _, line := range lines { + resultLines = append(resultLines, strings.Trim(line, "\r\n")) + } + + content = strings.Join(resultLines, "\n") + + excerptSep := "" + if value, ok := metadata["excerpt_separator"]; ok { + if str, strOk := value.(string); strOk { + content = strings.Replace(content, strings.TrimSpace(str), excerptSep, -1) + } + } + + replaceList := []struct { + re *regexp.Regexp + replace string + }{ + {regexp.MustCompile(""), ""}, + {regexp.MustCompile(`\{%\s*raw\s*%\}\s*(.*?)\s*\{%\s*endraw\s*%\}`), "$1"}, + {regexp.MustCompile(`{%\s*highlight\s*(.*?)\s*%}`), "{{< highlight $1 >}}"}, + {regexp.MustCompile(`{%\s*endhighlight\s*%}`), "{{< / highlight >}}"}, + } + + for _, replace := range replaceList { + content = replace.re.ReplaceAllString(content, replace.replace) + } + + return content +} diff --git a/commands/import_test.go b/commands/import_test.go new file mode 100644 index 000000000..801d3cff3 --- /dev/null +++ b/commands/import_test.go @@ -0,0 +1,104 @@ +// Copyright © 2015 Steve Francia . +// +// Licensed under the Simple Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://opensource.org/licenses/Simple-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package commands + +import ( + "encoding/json" + "github.com/stretchr/testify/assert" + "testing" + "time" +) + +func TestParseJekyllFilename(t *testing.T) { + filenameArray := []string{ + "2015-01-02-test.md", + "2012-03-15-中文.markup", + } + + expectResult := []struct { + postDate time.Time + postName string + }{ + {time.Date(2015, time.January, 2, 0, 0, 0, 0, time.UTC), "test"}, + {time.Date(2012, time.March, 15, 0, 0, 0, 0, time.UTC), "中文"}, + } + + for i, filename := range filenameArray { + postDate, postName, err := parseJekyllFilename(filename) + assert.Equal(t, err, nil) + assert.Equal(t, expectResult[i].postDate.Format("2006-01-02"), postDate.Format("2006-01-02")) + assert.Equal(t, expectResult[i].postName, postName) + } +} + +func TestConvertJekyllMetadata(t *testing.T) { + testDataList := []struct { + metadata interface{} + postName string + postDate time.Time + draft bool + expect string + }{ + {map[interface{}]interface{}{}, "testPost", time.Date(2015, 10, 1, 0, 0, 0, 0, time.UTC), false, + `{"date":"2015-10-01T00:00:00Z","url":"/2015/10/01/testPost/"}`}, + {map[interface{}]interface{}{}, "testPost", time.Date(2015, 10, 1, 0, 0, 0, 0, time.UTC), true, + `{"date":"2015-10-01T00:00:00Z","draft":true,"url":"/2015/10/01/testPost/"}`}, + {map[interface{}]interface{}{"Permalink": "/permalink.html", "layout": "post"}, + "testPost", time.Date(2015, 10, 1, 0, 0, 0, 0, time.UTC), false, + `{"date":"2015-10-01T00:00:00Z","url":"/permalink.html"}`}, + {map[interface{}]interface{}{"permalink": "/permalink.html"}, + "testPost", time.Date(2015, 10, 1, 0, 0, 0, 0, time.UTC), false, + `{"date":"2015-10-01T00:00:00Z","url":"/permalink.html"}`}, + {map[interface{}]interface{}{"category": nil, "permalink": 123}, + "testPost", time.Date(2015, 10, 1, 0, 0, 0, 0, time.UTC), false, + `{"date":"2015-10-01T00:00:00Z","url":"/2015/10/01/testPost/"}`}, + {map[interface{}]interface{}{"Excerpt_Separator": "sep"}, + "testPost", time.Date(2015, 10, 1, 0, 0, 0, 0, time.UTC), false, + `{"date":"2015-10-01T00:00:00Z","excerpt_separator":"sep","url":"/2015/10/01/testPost/"}`}, + {map[interface{}]interface{}{"category": "book", "layout": "post", "Others": "Goods", "Date": "2015-10-01 12:13:11"}, + "testPost", time.Date(2015, 10, 1, 0, 0, 0, 0, time.UTC), false, + `{"Others":"Goods","categories":["book"],"date":"2015-10-01T12:13:11Z","url":"/2015/10/01/testPost/"}`}, + } + + for _, data := range testDataList { + result, err := convertJekyllMetaData(data.metadata, data.postName, data.postDate, data.draft) + assert.Equal(t, nil, err) + jsonResult, err := json.Marshal(result) + assert.Equal(t, nil, err) + assert.Equal(t, data.expect, string(jsonResult)) + } +} + +func TestConvertJekyllContent(t *testing.T) { + testDataList := []struct { + metadata interface{} + content string + expect string + }{ + {map[interface{}]interface{}{}, + `Test content\n\npart2 content`, `Test content\n\npart2 content`}, + {map[interface{}]interface{}{"excerpt_separator": ""}, + `Test content\n\npart2 content`, `Test content\n\npart2 content`}, + {map[interface{}]interface{}{}, "{% raw %}text{% endraw %}", "text"}, + {map[interface{}]interface{}{}, "{%raw%} text2 {%endraw %}", "text2"}, + {map[interface{}]interface{}{}, + "{% highlight go %}\nvar s int\n{% endhighlight %}", + "{{< highlight go >}}\nvar s int\n{{< / highlight >}}"}, + } + + for _, data := range testDataList { + result := convertJekyllContent(data.metadata, data.content) + assert.Equal(t, data.expect, result) + } +}