Add build time math rendering

While very useful on its own (and combined with the passthrough render hooks), this also serves as a proof of concept of using WASI (WebAssembly System Interface) modules in Hugo.

This will be marked _experimental_ in the documentation. Not because it will be removed or changed in a dramatic way, but we need to think a little more how to best set up/configure similar services, define where these WASM files gets stored, maybe we can allow user provided WASM files plugins via Hugo Modules mounts etc.

See these issues for more context:

* https://github.com/gohugoio/hugo/issues/12736
* https://github.com/gohugoio/hugo/issues/12737

See #11927
This commit is contained in:
Bjørn Erik Pedersen 2024-08-07 10:40:54 +02:00
parent 0c3a1c7288
commit 33c0938cd5
26 changed files with 1598 additions and 13 deletions

View file

@ -46,6 +46,7 @@ const (
CacheKeyAssets = "assets" CacheKeyAssets = "assets"
CacheKeyModules = "modules" CacheKeyModules = "modules"
CacheKeyGetResource = "getresource" CacheKeyGetResource = "getresource"
CacheKeyMisc = "misc"
) )
type Configs map[string]FileCacheConfig type Configs map[string]FileCacheConfig
@ -70,10 +71,14 @@ var defaultCacheConfigs = Configs{
MaxAge: -1, MaxAge: -1,
Dir: resourcesGenDir, Dir: resourcesGenDir,
}, },
CacheKeyGetResource: FileCacheConfig{ CacheKeyGetResource: {
MaxAge: -1, // Never expire MaxAge: -1, // Never expire
Dir: cacheDirProject, Dir: cacheDirProject,
}, },
CacheKeyMisc: {
MaxAge: -1,
Dir: cacheDirProject,
},
} }
type FileCacheConfig struct { type FileCacheConfig struct {
@ -120,6 +125,11 @@ func (f Caches) AssetsCache() *Cache {
return f[CacheKeyAssets] return f[CacheKeyAssets]
} }
// MiscCache gets the file cache for miscellaneous stuff.
func (f Caches) MiscCache() *Cache {
return f[CacheKeyMisc]
}
// GetResourceCache gets the file cache for remote resources. // GetResourceCache gets the file cache for remote resources.
func (f Caches) GetResourceCache() *Cache { func (f Caches) GetResourceCache() *Cache {
return f[CacheKeyGetResource] return f[CacheKeyGetResource]

View file

@ -59,7 +59,7 @@ dir = "/path/to/c4"
c.Assert(err, qt.IsNil) c.Assert(err, qt.IsNil)
fs := afero.NewMemMapFs() fs := afero.NewMemMapFs()
decoded := testconfig.GetTestConfigs(fs, cfg).Base.Caches decoded := testconfig.GetTestConfigs(fs, cfg).Base.Caches
c.Assert(len(decoded), qt.Equals, 6) c.Assert(len(decoded), qt.Equals, 7)
c2 := decoded["getcsv"] c2 := decoded["getcsv"]
c.Assert(c2.MaxAge.String(), qt.Equals, "11h0m0s") c.Assert(c2.MaxAge.String(), qt.Equals, "11h0m0s")
@ -106,7 +106,7 @@ dir = "/path/to/c4"
c.Assert(err, qt.IsNil) c.Assert(err, qt.IsNil)
fs := afero.NewMemMapFs() fs := afero.NewMemMapFs()
decoded := testconfig.GetTestConfigs(fs, cfg).Base.Caches decoded := testconfig.GetTestConfigs(fs, cfg).Base.Caches
c.Assert(len(decoded), qt.Equals, 6) c.Assert(len(decoded), qt.Equals, 7)
for _, v := range decoded { for _, v := range decoded {
c.Assert(v.MaxAge, qt.Equals, time.Duration(0)) c.Assert(v.MaxAge, qt.Equals, time.Duration(0))
@ -129,7 +129,7 @@ func TestDecodeConfigDefault(t *testing.T) {
fs := afero.NewMemMapFs() fs := afero.NewMemMapFs()
decoded := testconfig.GetTestConfigs(fs, cfg).Base.Caches decoded := testconfig.GetTestConfigs(fs, cfg).Base.Caches
c.Assert(len(decoded), qt.Equals, 6) c.Assert(len(decoded), qt.Equals, 7)
imgConfig := decoded[filecache.CacheKeyImages] imgConfig := decoded[filecache.CacheKeyImages]
jsonConfig := decoded[filecache.CacheKeyGetJSON] jsonConfig := decoded[filecache.CacheKeyGetJSON]

View file

@ -81,3 +81,33 @@ func ToReadCloser(r io.Reader) io.ReadCloser {
io.NopCloser(nil), io.NopCloser(nil),
} }
} }
type ReadWriteCloser interface {
io.Reader
io.Writer
io.Closer
}
// PipeReadWriteCloser is a convenience type to create a pipe with a ReadCloser and a WriteCloser.
type PipeReadWriteCloser struct {
*io.PipeReader
*io.PipeWriter
}
// NewPipeReadWriteCloser creates a new PipeReadWriteCloser.
func NewPipeReadWriteCloser() PipeReadWriteCloser {
pr, pw := io.Pipe()
return PipeReadWriteCloser{pr, pw}
}
func (c PipeReadWriteCloser) Close() (err error) {
if err = c.PipeReader.Close(); err != nil {
return
}
err = c.PipeWriter.Close()
return
}
func (c PipeReadWriteCloser) WriteString(s string) (int, error) {
return c.PipeWriter.Write([]byte(s))
}

8
deps/deps.go vendored
View file

@ -23,6 +23,7 @@ import (
"github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/helpers"
"github.com/gohugoio/hugo/hugofs" "github.com/gohugoio/hugo/hugofs"
"github.com/gohugoio/hugo/identity" "github.com/gohugoio/hugo/identity"
"github.com/gohugoio/hugo/internal/warpc"
"github.com/gohugoio/hugo/media" "github.com/gohugoio/hugo/media"
"github.com/gohugoio/hugo/resources/page" "github.com/gohugoio/hugo/resources/page"
"github.com/gohugoio/hugo/resources/postpub" "github.com/gohugoio/hugo/resources/postpub"
@ -93,6 +94,10 @@ type Deps struct {
// This is common/global for all sites. // This is common/global for all sites.
BuildState *BuildState BuildState *BuildState
// Holds RPC dispatchers for Katex etc.
// TODO(bep) rethink this re. a plugin setup, but this will have to do for now.
WasmDispatchers *warpc.Dispatchers
*globalErrHandler *globalErrHandler
} }
@ -343,6 +348,9 @@ func (d *Deps) Close() error {
if d.MemCache != nil { if d.MemCache != nil {
d.MemCache.Stop() d.MemCache.Stop()
} }
if d.WasmDispatchers != nil {
d.WasmDispatchers.Close()
}
return d.BuildClosers.Close() return d.BuildClosers.Close()
} }

1
go.mod
View file

@ -69,6 +69,7 @@ require (
github.com/spf13/pflag v1.0.5 github.com/spf13/pflag v1.0.5
github.com/tdewolff/minify/v2 v2.20.37 github.com/tdewolff/minify/v2 v2.20.37
github.com/tdewolff/parse/v2 v2.7.15 github.com/tdewolff/parse/v2 v2.7.15
github.com/tetratelabs/wazero v1.7.4-0.20240805170331-2b12e189eeec
github.com/yuin/goldmark v1.7.4 github.com/yuin/goldmark v1.7.4
github.com/yuin/goldmark-emoji v1.0.3 github.com/yuin/goldmark-emoji v1.0.3
go.uber.org/automaxprocs v1.5.3 go.uber.org/automaxprocs v1.5.3

4
go.sum
View file

@ -233,8 +233,6 @@ github.com/gohugoio/httpcache v0.7.0 h1:ukPnn04Rgvx48JIinZvZetBfHaWE7I01JR2Q2RrQ
github.com/gohugoio/httpcache v0.7.0/go.mod h1:fMlPrdY/vVJhAriLZnrF5QpN3BNAcoBClgAyQd+lGFI= github.com/gohugoio/httpcache v0.7.0/go.mod h1:fMlPrdY/vVJhAriLZnrF5QpN3BNAcoBClgAyQd+lGFI=
github.com/gohugoio/hugo-goldmark-extensions/extras v0.2.0 h1:MNdY6hYCTQEekY0oAfsxWZU1CDt6iH+tMLgyMJQh/sg= github.com/gohugoio/hugo-goldmark-extensions/extras v0.2.0 h1:MNdY6hYCTQEekY0oAfsxWZU1CDt6iH+tMLgyMJQh/sg=
github.com/gohugoio/hugo-goldmark-extensions/extras v0.2.0/go.mod h1:oBdBVuiZ0fv9xd8xflUgt53QxW5jOCb1S+xntcN4SKo= github.com/gohugoio/hugo-goldmark-extensions/extras v0.2.0/go.mod h1:oBdBVuiZ0fv9xd8xflUgt53QxW5jOCb1S+xntcN4SKo=
github.com/gohugoio/hugo-goldmark-extensions/passthrough v0.2.0 h1:PCtO5l++psZf48yen2LxQ3JiOXxaRC6v0594NeHvGZg=
github.com/gohugoio/hugo-goldmark-extensions/passthrough v0.2.0/go.mod h1:g9CCh+Ci2IMbPUrVJuXbBTrA+rIIx5+hDQ4EXYaQDoM=
github.com/gohugoio/hugo-goldmark-extensions/passthrough v0.3.0 h1:7PY5PIJ2mck7v6R52yCFvvYHvsPMEbulgRviw3I9lP4= github.com/gohugoio/hugo-goldmark-extensions/passthrough v0.3.0 h1:7PY5PIJ2mck7v6R52yCFvvYHvsPMEbulgRviw3I9lP4=
github.com/gohugoio/hugo-goldmark-extensions/passthrough v0.3.0/go.mod h1:r8g5S7bHfdj0+9ShBog864ufCsVODKQZNjYYY8OnJpM= github.com/gohugoio/hugo-goldmark-extensions/passthrough v0.3.0/go.mod h1:r8g5S7bHfdj0+9ShBog864ufCsVODKQZNjYYY8OnJpM=
github.com/gohugoio/locales v0.14.0 h1:Q0gpsZwfv7ATHMbcTNepFd59H7GoykzWJIxi113XGDc= github.com/gohugoio/locales v0.14.0 h1:Q0gpsZwfv7ATHMbcTNepFd59H7GoykzWJIxi113XGDc=
@ -461,6 +459,8 @@ github.com/tdewolff/parse/v2 v2.7.15/go.mod h1:3FbJWZp3XT9OWVN3Hmfp0p/a08v4h8J9W
github.com/tdewolff/test v1.0.11-0.20231101010635-f1265d231d52/go.mod h1:6DAvZliBAAnD7rhVgwaM7DE5/d9NMOAJ09SqYqeK4QE= github.com/tdewolff/test v1.0.11-0.20231101010635-f1265d231d52/go.mod h1:6DAvZliBAAnD7rhVgwaM7DE5/d9NMOAJ09SqYqeK4QE=
github.com/tdewolff/test v1.0.11-0.20240106005702-7de5f7df4739 h1:IkjBCtQOOjIn03u/dMQK9g+Iw9ewps4mCl1nB8Sscbo= github.com/tdewolff/test v1.0.11-0.20240106005702-7de5f7df4739 h1:IkjBCtQOOjIn03u/dMQK9g+Iw9ewps4mCl1nB8Sscbo=
github.com/tdewolff/test v1.0.11-0.20240106005702-7de5f7df4739/go.mod h1:XPuWBzvdUzhCuxWO1ojpXsyzsA5bFoS3tO/Q3kFuTG8= github.com/tdewolff/test v1.0.11-0.20240106005702-7de5f7df4739/go.mod h1:XPuWBzvdUzhCuxWO1ojpXsyzsA5bFoS3tO/Q3kFuTG8=
github.com/tetratelabs/wazero v1.7.4-0.20240805170331-2b12e189eeec h1:KeQseLFSWb9qjW4PSWxciTBk1hbG7KsVx3rs1hIQnbQ=
github.com/tetratelabs/wazero v1.7.4-0.20240805170331-2b12e189eeec/go.mod h1:ytl6Zuh20R/eROuyDaGPkp82O9C/DJfXAwJfQ3X6/7Y=
github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0= github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0=
github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY=
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=

View file

@ -19,6 +19,7 @@ import (
"fmt" "fmt"
"html/template" "html/template"
"os" "os"
"path/filepath"
"sort" "sort"
"time" "time"
@ -34,6 +35,7 @@ import (
"github.com/gohugoio/hugo/hugolib/doctree" "github.com/gohugoio/hugo/hugolib/doctree"
"github.com/gohugoio/hugo/hugolib/pagesfromdata" "github.com/gohugoio/hugo/hugolib/pagesfromdata"
"github.com/gohugoio/hugo/identity" "github.com/gohugoio/hugo/identity"
"github.com/gohugoio/hugo/internal/warpc"
"github.com/gohugoio/hugo/langs" "github.com/gohugoio/hugo/langs"
"github.com/gohugoio/hugo/langs/i18n" "github.com/gohugoio/hugo/langs/i18n"
"github.com/gohugoio/hugo/lazy" "github.com/gohugoio/hugo/lazy"
@ -157,6 +159,15 @@ func NewHugoSites(cfg deps.DepsCfg) (*HugoSites, error) {
MemCache: memCache, MemCache: memCache,
TemplateProvider: tplimpl.DefaultTemplateProvider, TemplateProvider: tplimpl.DefaultTemplateProvider,
TranslationProvider: i18n.NewTranslationProvider(), TranslationProvider: i18n.NewTranslationProvider(),
WasmDispatchers: warpc.AllDispatchers(
warpc.Options{
CompilationCacheDir: filepath.Join(conf.Dirs().CacheDir, "_warpc"),
// Katex is relatively slow.
PoolSize: 8,
Infof: logger.InfoCommand("wasm").Logf,
},
),
} }
if err := firstSiteDeps.Init(); err != nil { if err := firstSiteDeps.Init(); err != nil {

5
internal/warpc/build.sh Executable file
View file

@ -0,0 +1,5 @@
# TODO1 clean up when done.
go generate ./gen
javy compile js/greet.bundle.js -d -o wasm/greet.wasm
javy compile js/renderkatex.bundle.js -d -o wasm/renderkatex.wasm
touch warpc_test.go

View file

@ -0,0 +1,55 @@
//go:generate go run main.go
package main
import (
"fmt"
"log"
"os"
"path/filepath"
"strings"
"github.com/evanw/esbuild/pkg/api"
)
var scripts = []string{
"greet.js",
"renderkatex.js",
}
func main() {
for _, script := range scripts {
filename := filepath.Join("../js", script)
err := buildJSBundle(filename)
if err != nil {
log.Fatal(err)
}
}
}
func buildJSBundle(filename string) error {
minify := true
result := api.Build(
api.BuildOptions{
EntryPoints: []string{filename},
Bundle: true,
MinifyWhitespace: minify,
MinifyIdentifiers: minify,
MinifySyntax: minify,
Target: api.ES2020,
Outfile: strings.Replace(filename, ".js", ".bundle.js", 1),
SourceRoot: "../js",
})
if len(result.Errors) > 0 {
return fmt.Errorf("build failed: %v", result.Errors)
}
if len(result.OutputFiles) != 1 {
return fmt.Errorf("expected 1 output file, got %d", len(result.OutputFiles))
}
of := result.OutputFiles[0]
if err := os.WriteFile(filepath.FromSlash(of.Path), of.Contents, 0o644); err != nil {
return fmt.Errorf("write file failed: %v", err)
}
return nil
}

2
internal/warpc/js/.gitignore vendored Normal file
View file

@ -0,0 +1,2 @@
node_modules/
package-lock.json

View file

@ -0,0 +1,56 @@
// Read JSONL from stdin.
export function readInput(handle) {
const buffSize = 1024;
let currentLine = [];
const buffer = new Uint8Array(buffSize);
// Read all the available bytes
while (true) {
// Stdin file descriptor
const fd = 0;
let bytesRead = 0;
try {
bytesRead = Javy.IO.readSync(fd, buffer);
} catch (e) {
// IO.readSync fails with os error 29 when stdin closes.
if (e.message.includes('os error 29')) {
break;
}
throw new Error('Error reading from stdin');
}
if (bytesRead < 0) {
throw new Error('Error reading from stdin');
break;
}
if (bytesRead === 0) {
break;
}
currentLine = [...currentLine, ...buffer.subarray(0, bytesRead)];
// Split array into chunks by newline.
let i = 0;
for (let j = 0; i < currentLine.length; i++) {
if (currentLine[i] === 10) {
const chunk = currentLine.splice(j, i + 1);
const arr = new Uint8Array(chunk);
const json = JSON.parse(new TextDecoder().decode(arr));
handle(json);
j = i + 1;
}
}
// Remove processed data.
currentLine = currentLine.slice(i);
}
}
// Write JSONL to stdout
export function writeOutput(output) {
const encodedOutput = new TextEncoder().encode(JSON.stringify(output) + '\n');
const buffer = new Uint8Array(encodedOutput);
// Stdout file descriptor
const fd = 1;
Javy.IO.writeSync(fd, buffer);
}

View file

@ -0,0 +1,2 @@
(()=>{function i(r){let e=[],a=new Uint8Array(1024);for(;;){let n=0;try{n=Javy.IO.readSync(0,a)}catch(o){if(o.message.includes("os error 29"))break;throw new Error("Error reading from stdin")}if(n<0)throw new Error("Error reading from stdin");if(n===0)break;e=[...e,...a.subarray(0,n)];let t=0;for(let o=0;t<e.length;t++)if(e[t]===10){let f=e.splice(o,t+1),s=new Uint8Array(f),u=JSON.parse(new TextDecoder().decode(s));r(u),o=t+1}e=e.slice(t)}}function d(r){let c=new TextEncoder().encode(JSON.stringify(r)+`
`),e=new Uint8Array(c);Javy.IO.writeSync(1,e)}var l=function(r){d({header:r.header,data:{greeting:"Hello "+r.data.name+"!"}})};console.log("Greet module loaded");i(l);})();

View file

@ -0,0 +1,9 @@
import { readInput, writeOutput } from './common';
const greet = function (input) {
writeOutput({ header: input.header, data: { greeting: 'Hello ' + input.data.name + '!' } });
};
console.log('Greet module loaded');
readInput(greet);

View file

@ -0,0 +1,14 @@
{
"name": "js",
"version": "1.0.0",
"main": "greet.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
},
"author": "",
"license": "ISC",
"description": "",
"devDependencies": {
"katex": "^0.16.11"
}
}

File diff suppressed because one or more lines are too long

View file

@ -0,0 +1,11 @@
import { readInput, writeOutput } from './common';
import katex from 'katex';
const render = function (input) {
const data = input.data;
const expression = data.expression;
const options = data.options;
writeOutput({ header: input.header, data: { output: katex.renderToString(expression, options) } });
};
readInput(render);

24
internal/warpc/katex.go Normal file
View file

@ -0,0 +1,24 @@
package warpc
import (
_ "embed"
)
//go:embed wasm/renderkatex.wasm
var katexWasm []byte
// See https://katex.org/docs/options.html
type KatexInput struct {
Expression string `json:"expression"`
Options KatexOptions `json:"options"`
}
type KatexOptions struct {
Output string `json:"output"` // html, mathml (default), htmlAndMathml
DisplayMode bool `json:"displayMode"`
ThrowOnError bool `json:"throwOnError"`
}
type KatexOutput struct {
Output string `json:"output"`
}

552
internal/warpc/warpc.go Normal file
View file

@ -0,0 +1,552 @@
package warpc
import (
"bytes"
"context"
_ "embed"
"encoding/json"
"errors"
"fmt"
"io"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/gohugoio/hugo/common/hugio"
"golang.org/x/sync/errgroup"
"github.com/tetratelabs/wazero"
"github.com/tetratelabs/wazero/api"
"github.com/tetratelabs/wazero/experimental"
"github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1"
)
const currentVersion = "v1"
//go:embed wasm/quickjs.wasm
var quickjsWasm []byte
// Header is in both the request and response.
type Header struct {
Version string `json:"version"`
ID uint32 `json:"id"`
}
type Message[T any] struct {
Header Header `json:"header"`
Data T `json:"data"`
}
func (m Message[T]) GetID() uint32 {
return m.Header.ID
}
type Dispatcher[Q, R any] interface {
Execute(ctx context.Context, q Message[Q]) (Message[R], error)
Close() error
}
func (p *dispatcherPool[Q, R]) getDispatcher() *dispatcher[Q, R] {
i := int(p.counter.Add(1)) % len(p.dispatchers)
return p.dispatchers[i]
}
func (p *dispatcherPool[Q, R]) Close() error {
return p.close()
}
type dispatcher[Q, R any] struct {
zero Message[R]
mu sync.RWMutex
encMu sync.Mutex
pending map[uint32]*call[Q, R]
inOut *inOut
shutdown bool
closing bool
}
type inOut struct {
sync.Mutex
stdin hugio.ReadWriteCloser
stdout hugio.ReadWriteCloser
dec *json.Decoder
enc *json.Encoder
}
var ErrShutdown = fmt.Errorf("dispatcher is shutting down")
var timerPool = sync.Pool{}
func getTimer(d time.Duration) *time.Timer {
if v := timerPool.Get(); v != nil {
timer := v.(*time.Timer)
timer.Reset(d)
return timer
}
return time.NewTimer(d)
}
func putTimer(t *time.Timer) {
if !t.Stop() {
select {
case <-t.C:
default:
}
}
timerPool.Put(t)
}
// Execute sends a request to the dispatcher and waits for the response.
func (p *dispatcherPool[Q, R]) Execute(ctx context.Context, q Message[Q]) (Message[R], error) {
d := p.getDispatcher()
if q.GetID() == 0 {
return d.zero, errors.New("ID must not be 0 (note that this must be unique within the current request set time window)")
}
call, err := d.newCall(q)
if err != nil {
return d.zero, err
}
if err := d.send(call); err != nil {
return d.zero, err
}
timer := getTimer(30 * time.Second)
defer putTimer(timer)
select {
case call = <-call.donec:
case <-p.donec:
return d.zero, p.Err()
case <-ctx.Done():
return d.zero, ctx.Err()
case <-timer.C:
return d.zero, errors.New("timeout")
}
if call.err != nil {
return d.zero, call.err
}
return call.response, p.Err()
}
func (d *dispatcher[Q, R]) newCall(q Message[Q]) (*call[Q, R], error) {
call := &call[Q, R]{
donec: make(chan *call[Q, R], 1),
request: q,
}
if d.shutdown || d.closing {
call.err = ErrShutdown
call.done()
return call, nil
}
d.mu.Lock()
d.pending[q.GetID()] = call
d.mu.Unlock()
return call, nil
}
func (d *dispatcher[Q, R]) send(call *call[Q, R]) error {
d.mu.RLock()
if d.closing || d.shutdown {
d.mu.RUnlock()
return ErrShutdown
}
d.mu.RUnlock()
d.encMu.Lock()
defer d.encMu.Unlock()
err := d.inOut.enc.Encode(call.request)
if err != nil {
return err
}
return nil
}
func (d *dispatcher[Q, R]) input() {
var inputErr error
for d.inOut.dec.More() {
var r Message[R]
if err := d.inOut.dec.Decode(&r); err != nil {
inputErr = err
break
}
d.mu.Lock()
call, found := d.pending[r.GetID()]
if !found {
d.mu.Unlock()
panic(fmt.Errorf("call with ID %d not found", r.GetID()))
}
delete(d.pending, r.GetID())
d.mu.Unlock()
call.response = r
call.done()
}
// Terminate pending calls.
d.shutdown = true
if inputErr != nil {
isEOF := inputErr == io.EOF || strings.Contains(inputErr.Error(), "already closed")
if isEOF {
if d.closing {
inputErr = ErrShutdown
} else {
inputErr = io.ErrUnexpectedEOF
}
}
}
d.mu.Lock()
defer d.mu.Unlock()
for _, call := range d.pending {
call.err = inputErr
call.done()
}
}
type call[Q, R any] struct {
request Message[Q]
response Message[R]
err error
donec chan *call[Q, R]
}
func (call *call[Q, R]) done() {
select {
case call.donec <- call:
default:
}
}
// Binary represents a WebAssembly binary.
type Binary struct {
// The name of the binary.
// For quickjs, this must match the instance import name, "javy_quickjs_provider_v2".
// For the main module, we only use this for caching.
Name string
// THe wasm binary.
Data []byte
}
type Options struct {
Ctx context.Context
Infof func(format string, v ...any)
// E.g. quickjs wasm. May be omitted if not needed.
Runtime Binary
// The main module to instantiate.
Main Binary
CompilationCacheDir string
PoolSize int
// Memory limit in MiB.
Memory int
}
type CompileModuleContext struct {
Opts Options
Runtime wazero.Runtime
}
type CompiledModule struct {
// Runtime (e.g. QuickJS) may be nil if not needed (e.g. embedded in Module).
Runtime wazero.CompiledModule
// If Runtime is not nil, this should be the name of the instance.
RuntimeName string
// The main module to instantiate.
// This will be insantiated multiple times in a pool,
// so it does not need a name.
Module wazero.CompiledModule
}
// Start creates a new dispatcher pool.
func Start[Q, R any](opts Options) (Dispatcher[Q, R], error) {
if opts.Main.Data == nil {
return nil, errors.New("Main.Data must be set")
}
if opts.Main.Name == "" {
return nil, errors.New("Main.Name must be set")
}
if opts.Runtime.Data != nil && opts.Runtime.Name == "" {
return nil, errors.New("Runtime.Name must be set")
}
if opts.PoolSize == 0 {
opts.PoolSize = 1
}
return newDispatcher[Q, R](opts)
}
type dispatcherPool[Q, R any] struct {
counter atomic.Uint32
dispatchers []*dispatcher[Q, R]
close func() error
errc chan error
donec chan struct{}
}
func (p *dispatcherPool[Q, R]) SendIfErr(err error) {
if err != nil {
p.errc <- err
}
}
func (p *dispatcherPool[Q, R]) Err() error {
select {
case err := <-p.errc:
return err
default:
return nil
}
}
func newDispatcher[Q, R any](opts Options) (*dispatcherPool[Q, R], error) {
if opts.Ctx == nil {
opts.Ctx = context.Background()
}
if opts.Infof == nil {
opts.Infof = func(format string, v ...any) {
// noop
}
}
if opts.Memory <= 0 {
// 32 MiB
opts.Memory = 32
}
ctx := opts.Ctx
// Page size is 64KB.
numPages := opts.Memory * 1024 / 64
runtimeConfig := wazero.NewRuntimeConfig().WithMemoryLimitPages(uint32(numPages))
if opts.CompilationCacheDir != "" {
compilationCache, err := wazero.NewCompilationCacheWithDir(opts.CompilationCacheDir)
if err != nil {
return nil, err
}
runtimeConfig = runtimeConfig.WithCompilationCache(compilationCache)
}
// Create a new WebAssembly Runtime.
r := wazero.NewRuntimeWithConfig(opts.Ctx, runtimeConfig)
// Instantiate WASI, which implements system I/O such as console output.
if _, err := wasi_snapshot_preview1.Instantiate(ctx, r); err != nil {
return nil, err
}
inOuts := make([]*inOut, opts.PoolSize)
for i := 0; i < opts.PoolSize; i++ {
var stdin, stdout hugio.ReadWriteCloser
stdin = hugio.NewPipeReadWriteCloser()
stdout = hugio.NewPipeReadWriteCloser()
inOuts[i] = &inOut{
stdin: stdin,
stdout: stdout,
dec: json.NewDecoder(stdout),
enc: json.NewEncoder(stdin),
}
}
var (
runtimeModule wazero.CompiledModule
mainModule wazero.CompiledModule
err error
)
if opts.Runtime.Data != nil {
runtimeModule, err = r.CompileModule(ctx, opts.Runtime.Data)
if err != nil {
return nil, err
}
}
mainModule, err = r.CompileModule(ctx, opts.Main.Data)
if err != nil {
return nil, err
}
toErr := func(what string, errBuff bytes.Buffer, err error) error {
return fmt.Errorf("%s: %s: %w", what, errBuff.String(), err)
}
run := func() error {
g, ctx := errgroup.WithContext(ctx)
for _, c := range inOuts {
c := c
g.Go(func() error {
var errBuff bytes.Buffer
ctx := context.WithoutCancel(ctx)
configBase := wazero.NewModuleConfig().WithStderr(&errBuff).WithStdout(c.stdout).WithStdin(c.stdin).WithStartFunctions()
if opts.Runtime.Data != nil {
// This needs to be anonymous, it will be resolved in the import resolver below.
runtimeInstance, err := r.InstantiateModule(ctx, runtimeModule, configBase.WithName(""))
if err != nil {
return toErr("quickjs", errBuff, err)
}
ctx = experimental.WithImportResolver(ctx,
func(name string) api.Module {
if name == opts.Runtime.Name {
return runtimeInstance
}
return nil
},
)
}
mainInstance, err := r.InstantiateModule(ctx, mainModule, configBase.WithName(""))
if err != nil {
return toErr(opts.Main.Name, errBuff, err)
}
if _, err := mainInstance.ExportedFunction("_start").Call(ctx); err != nil {
return toErr(opts.Main.Name, errBuff, err)
}
// The console.log in the Javy/quickjs WebAssembly module will write to stderr.
// In non-error situations, write that to the provided infof logger.
if errBuff.Len() > 0 {
opts.Infof("%s", errBuff.String())
}
return nil
})
}
return g.Wait()
}
dp := &dispatcherPool[Q, R]{
dispatchers: make([]*dispatcher[Q, R], len(inOuts)),
errc: make(chan error, 10),
donec: make(chan struct{}),
}
go func() {
// This will block until stdin is closed or it encounters an error.
err := run()
dp.SendIfErr(err)
close(dp.donec)
}()
for i := 0; i < len(inOuts); i++ {
d := &dispatcher[Q, R]{
pending: make(map[uint32]*call[Q, R]),
inOut: inOuts[i],
}
go d.input()
dp.dispatchers[i] = d
}
dp.close = func() error {
for _, d := range dp.dispatchers {
d.closing = true
if err := d.inOut.stdin.Close(); err != nil {
return err
}
if err := d.inOut.stdout.Close(); err != nil {
return err
}
}
// We need to wait for the WebAssembly instances to finish executing before we can close the runtime.
<-dp.donec
if err := r.Close(ctx); err != nil {
return err
}
// Return potential late compilation errors.
return dp.Err()
}
return dp, dp.Err()
}
type lazyDispatcher[Q, R any] struct {
opts Options
dispatcher Dispatcher[Q, R]
startOnce sync.Once
started bool
startErr error
}
func (d *lazyDispatcher[Q, R]) start() (Dispatcher[Q, R], error) {
d.startOnce.Do(func() {
start := time.Now()
d.dispatcher, d.startErr = Start[Q, R](d.opts)
d.started = true
d.opts.Infof("started dispatcher in %s", time.Since(start))
})
return d.dispatcher, d.startErr
}
// Dispatchers holds all the dispatchers for the warpc package.
type Dispatchers struct {
katex *lazyDispatcher[KatexInput, KatexOutput]
}
func (d *Dispatchers) Katex() (Dispatcher[KatexInput, KatexOutput], error) {
return d.katex.start()
}
func (d *Dispatchers) Close() error {
var errs []error
if d.katex.started {
if err := d.katex.dispatcher.Close(); err != nil {
errs = append(errs, err)
}
}
if len(errs) == 0 {
return nil
}
return fmt.Errorf("%v", errs)
}
// AllDispatchers creates all the dispatchers for the warpc package.
// Note that the individual dispatchers are started lazily.
// Remember to call Close on the returned Dispatchers when done.
func AllDispatchers(katexOpts Options) *Dispatchers {
if katexOpts.Runtime.Data == nil {
katexOpts.Runtime = Binary{Name: "javy_quickjs_provider_v2", Data: quickjsWasm}
}
if katexOpts.Main.Data == nil {
katexOpts.Main = Binary{Name: "renderkatex", Data: katexWasm}
}
if katexOpts.Infof == nil {
katexOpts.Infof = func(format string, v ...any) {
// noop
}
}
return &Dispatchers{
katex: &lazyDispatcher[KatexInput, KatexOutput]{opts: katexOpts},
}
}

View file

@ -0,0 +1,439 @@
package warpc
import (
"context"
_ "embed"
"fmt"
"sync"
"sync/atomic"
"testing"
qt "github.com/frankban/quicktest"
)
//go:embed wasm/greet.wasm
var greetWasm []byte
type person struct {
Name string `json:"name"`
}
func TestKatex(t *testing.T) {
c := qt.New(t)
opts := Options{
PoolSize: 8,
Runtime: quickjsBinary,
Main: katexBinary,
}
d, err := Start[KatexInput, KatexOutput](opts)
c.Assert(err, qt.IsNil)
defer d.Close()
ctx := context.Background()
input := KatexInput{
Expression: "c = \\pm\\sqrt{a^2 + b^2}",
Options: KatexOptions{
Output: "html",
DisplayMode: true,
},
}
message := Message[KatexInput]{
Header: Header{
Version: currentVersion,
ID: uint32(32),
},
Data: input,
}
result, err := d.Execute(ctx, message)
c.Assert(err, qt.IsNil)
c.Assert(result.GetID(), qt.Equals, message.GetID())
}
func TestGreet(t *testing.T) {
c := qt.New(t)
opts := Options{
PoolSize: 1,
Runtime: quickjsBinary,
Main: greetBinary,
Infof: t.Logf,
}
for i := 0; i < 2; i++ {
func() {
d, err := Start[person, greeting](opts)
if err != nil {
t.Fatal(err)
}
defer func() {
c.Assert(d.Close(), qt.IsNil)
}()
ctx := context.Background()
inputMessage := Message[person]{
Header: Header{
Version: currentVersion,
},
Data: person{
Name: "Person",
},
}
for j := 0; j < 20; j++ {
inputMessage.Header.ID = uint32(j + 1)
g, err := d.Execute(ctx, inputMessage)
if err != nil {
t.Fatal(err)
}
if g.Data.Greeting != "Hello Person!" {
t.Fatalf("got: %v", g)
}
if g.GetID() != inputMessage.GetID() {
t.Fatalf("%d vs %d", g.GetID(), inputMessage.GetID())
}
}
}()
}
}
func TestGreetParallel(t *testing.T) {
c := qt.New(t)
opts := Options{
Runtime: quickjsBinary,
Main: greetBinary,
PoolSize: 4,
}
d, err := Start[person, greeting](opts)
c.Assert(err, qt.IsNil)
defer func() {
c.Assert(d.Close(), qt.IsNil)
}()
var wg sync.WaitGroup
for i := 1; i <= 10; i++ {
wg.Add(1)
go func(i int) {
defer wg.Done()
ctx := context.Background()
for j := 0; j < 5; j++ {
base := i * 100
id := uint32(base + j)
inputPerson := person{
Name: fmt.Sprintf("Person %d", id),
}
inputMessage := Message[person]{
Header: Header{
Version: currentVersion,
ID: id,
},
Data: inputPerson,
}
g, err := d.Execute(ctx, inputMessage)
if err != nil {
t.Error(err)
return
}
c.Assert(g.Data.Greeting, qt.Equals, fmt.Sprintf("Hello Person %d!", id))
c.Assert(g.GetID(), qt.Equals, inputMessage.GetID())
}
}(i)
}
wg.Wait()
}
func TestKatexParallel(t *testing.T) {
c := qt.New(t)
opts := Options{
Runtime: quickjsBinary,
Main: katexBinary,
PoolSize: 6,
}
d, err := Start[KatexInput, KatexOutput](opts)
c.Assert(err, qt.IsNil)
defer func() {
c.Assert(d.Close(), qt.IsNil)
}()
var wg sync.WaitGroup
for i := 1; i <= 10; i++ {
wg.Add(1)
go func(i int) {
defer wg.Done()
ctx := context.Background()
for j := 0; j < 1; j++ {
base := i * 100
id := uint32(base + j)
input := katexInputTemplate
inputMessage := Message[KatexInput]{
Header: Header{
Version: currentVersion,
ID: id,
},
Data: input,
}
result, err := d.Execute(ctx, inputMessage)
if err != nil {
t.Error(err)
return
}
if result.GetID() != inputMessage.GetID() {
t.Errorf("%d vs %d", result.GetID(), inputMessage.GetID())
return
}
}
}(i)
}
wg.Wait()
}
func BenchmarkExecuteKatex(b *testing.B) {
opts := Options{
Runtime: quickjsBinary,
Main: katexBinary,
}
d, err := Start[KatexInput, KatexOutput](opts)
if err != nil {
b.Fatal(err)
}
defer d.Close()
ctx := context.Background()
input := katexInputTemplate
b.ResetTimer()
for i := 0; i < b.N; i++ {
message := Message[KatexInput]{
Header: Header{
Version: currentVersion,
ID: uint32(i + 1),
},
Data: input,
}
result, err := d.Execute(ctx, message)
if err != nil {
b.Fatal(err)
}
if result.GetID() != message.GetID() {
b.Fatalf("%d vs %d", result.GetID(), message.GetID())
}
}
}
func BenchmarkKatexStartStop(b *testing.B) {
optsTemplate := Options{
Runtime: quickjsBinary,
Main: katexBinary,
CompilationCacheDir: b.TempDir(),
}
runBench := func(b *testing.B, opts Options) {
for i := 0; i < b.N; i++ {
d, err := Start[KatexInput, KatexOutput](opts)
if err != nil {
b.Fatal(err)
}
if err := d.Close(); err != nil {
b.Fatal(err)
}
}
}
for _, poolSize := range []int{1, 8, 16} {
name := fmt.Sprintf("PoolSize%d", poolSize)
b.Run(name, func(b *testing.B) {
opts := optsTemplate
opts.PoolSize = poolSize
runBench(b, opts)
})
}
}
var katexInputTemplate = KatexInput{
Expression: "c = \\pm\\sqrt{a^2 + b^2}",
Options: KatexOptions{Output: "html", DisplayMode: true},
}
func BenchmarkExecuteKatexPara(b *testing.B) {
optsTemplate := Options{
Runtime: quickjsBinary,
Main: katexBinary,
}
runBench := func(b *testing.B, opts Options) {
d, err := Start[KatexInput, KatexOutput](opts)
if err != nil {
b.Fatal(err)
}
defer d.Close()
ctx := context.Background()
b.ResetTimer()
var id atomic.Uint32
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
message := Message[KatexInput]{
Header: Header{
Version: currentVersion,
ID: id.Add(1),
},
Data: katexInputTemplate,
}
result, err := d.Execute(ctx, message)
if err != nil {
b.Fatal(err)
}
if result.GetID() != message.GetID() {
b.Fatalf("%d vs %d", result.GetID(), message.GetID())
}
}
})
}
for _, poolSize := range []int{1, 8, 16} {
name := fmt.Sprintf("PoolSize%d", poolSize)
b.Run(name, func(b *testing.B) {
opts := optsTemplate
opts.PoolSize = poolSize
runBench(b, opts)
})
}
}
func BenchmarkExecuteGreet(b *testing.B) {
opts := Options{
Runtime: quickjsBinary,
Main: greetBinary,
}
d, err := Start[person, greeting](opts)
if err != nil {
b.Fatal(err)
}
defer d.Close()
ctx := context.Background()
input := person{
Name: "Person",
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
message := Message[person]{
Header: Header{
Version: currentVersion,
ID: uint32(i + 1),
},
Data: input,
}
result, err := d.Execute(ctx, message)
if err != nil {
b.Fatal(err)
}
if result.GetID() != message.GetID() {
b.Fatalf("%d vs %d", result.GetID(), message.GetID())
}
}
}
func BenchmarkExecuteGreetPara(b *testing.B) {
opts := Options{
Runtime: quickjsBinary,
Main: greetBinary,
PoolSize: 8,
}
d, err := Start[person, greeting](opts)
if err != nil {
b.Fatal(err)
}
defer d.Close()
ctx := context.Background()
inputTemplate := person{
Name: "Person",
}
b.ResetTimer()
var id atomic.Uint32
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
message := Message[person]{
Header: Header{
Version: currentVersion,
ID: id.Add(1),
},
Data: inputTemplate,
}
result, err := d.Execute(ctx, message)
if err != nil {
b.Fatal(err)
}
if result.GetID() != message.GetID() {
b.Fatalf("%d vs %d", result.GetID(), message.GetID())
}
}
})
}
type greeting struct {
Greeting string `json:"greeting"`
}
var (
greetBinary = Binary{
Name: "greet",
Data: greetWasm,
}
katexBinary = Binary{
Name: "renderkatex",
Data: katexWasm,
}
quickjsBinary = Binary{
Name: "javy_quickjs_provider_v2",
Data: quickjsWasm,
}
)

Binary file not shown.

Binary file not shown.

Binary file not shown.

View file

@ -0,0 +1,5 @@
#!/bin/bash
trap exit SIGINT
while true; do find . -type f -name "*.js" | entr -pd ./build.sh; done

View file

@ -18,16 +18,23 @@ import (
"bytes" "bytes"
"context" "context"
"encoding/xml" "encoding/xml"
"errors"
"html" "html"
"html/template" "html/template"
"io"
"strings" "strings"
"sync/atomic"
"github.com/gohugoio/hugo/cache/dynacache" "github.com/gohugoio/hugo/cache/dynacache"
"github.com/gohugoio/hugo/common/hashing"
"github.com/gohugoio/hugo/common/hugio"
"github.com/gohugoio/hugo/internal/warpc"
"github.com/gohugoio/hugo/markup/converter/hooks" "github.com/gohugoio/hugo/markup/converter/hooks"
"github.com/gohugoio/hugo/markup/highlight" "github.com/gohugoio/hugo/markup/highlight"
"github.com/gohugoio/hugo/markup/highlight/chromalexers" "github.com/gohugoio/hugo/markup/highlight/chromalexers"
"github.com/gohugoio/hugo/resources" "github.com/gohugoio/hugo/resources"
"github.com/gohugoio/hugo/tpl" "github.com/gohugoio/hugo/tpl"
"github.com/mitchellh/mapstructure"
"github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/deps"
"github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/helpers"
@ -42,18 +49,26 @@ func New(deps *deps.Deps) *Namespace {
return &Namespace{ return &Namespace{
deps: deps, deps: deps,
cache: dynacache.GetOrCreatePartition[string, *resources.StaleValue[any]]( cacheUnmarshal: dynacache.GetOrCreatePartition[string, *resources.StaleValue[any]](
deps.MemCache, deps.MemCache,
"/tmpl/transform", "/tmpl/transform/unmarshal",
dynacache.OptionsPartition{Weight: 30, ClearWhen: dynacache.ClearOnChange}, dynacache.OptionsPartition{Weight: 30, ClearWhen: dynacache.ClearOnChange},
), ),
cacheMath: dynacache.GetOrCreatePartition[string, string](
deps.MemCache,
"/tmpl/transform/math",
dynacache.OptionsPartition{Weight: 30, ClearWhen: dynacache.ClearNever},
),
} }
} }
// Namespace provides template functions for the "transform" namespace. // Namespace provides template functions for the "transform" namespace.
type Namespace struct { type Namespace struct {
cache *dynacache.Partition[string, *resources.StaleValue[any]] cacheUnmarshal *dynacache.Partition[string, *resources.StaleValue[any]]
deps *deps.Deps cacheMath *dynacache.Partition[string, string]
id atomic.Uint32
deps *deps.Deps
} }
// Emojify returns a copy of s with all emoji codes replaced with actual emojis. // Emojify returns a copy of s with all emoji codes replaced with actual emojis.
@ -182,7 +197,64 @@ func (ns *Namespace) Plainify(s any) (string, error) {
return tpl.StripHTML(ss), nil return tpl.StripHTML(ss), nil
} }
// ToMath converts a LaTeX string to math in the given format, default MathML.
// This uses KaTeX to render the math, see https://katex.org/.
func (ns *Namespace) ToMath(ctx context.Context, args ...any) (string, error) {
if len(args) < 1 {
return "", errors.New("must provide at least one argument")
}
expression, err := cast.ToStringE(args[0])
if err != nil {
return "", err
}
katexInput := warpc.KatexInput{
Expression: expression,
Options: warpc.KatexOptions{
Output: "mathml",
ThrowOnError: false,
},
}
if len(args) > 1 {
if err := mapstructure.WeakDecode(args[1], &katexInput); err != nil {
return "", err
}
}
s := hashing.HashString(args...)
key := "tomath/" + s[:2] + "/" + s[2:]
fileCache := ns.deps.ResourceSpec.FileCaches.MiscCache()
return ns.cacheMath.GetOrCreate(key, func(string) (string, error) {
_, r, err := fileCache.GetOrCreate(key, func() (io.ReadCloser, error) {
message := warpc.Message[warpc.KatexInput]{
Header: warpc.Header{
Version: "v1",
ID: ns.id.Add(1),
},
Data: katexInput,
}
k, err := ns.deps.WasmDispatchers.Katex()
if err != nil {
return nil, err
}
result, err := k.Execute(ctx, message)
if err != nil {
return nil, err
}
return hugio.NewReadSeekerNoOpCloserFromString(result.Data.Output), nil
})
if err != nil {
return "", err
}
return hugio.ReadString(r)
})
}
// For internal use. // For internal use.
func (ns *Namespace) Reset() { func (ns *Namespace) Reset() {
ns.cache.Clear() ns.cacheUnmarshal.Clear()
} }

View file

@ -133,3 +133,20 @@ Scar,"a "dead cat",11
[[name description age] [Spot a nice dog 3] [Rover a big dog 5] [Felix a "malicious" cat 7] [Bella an "evil" cat 9] [Scar a "dead cat 11]] [[name description age] [Spot a nice dog 3] [Rover a big dog 5] [Felix a "malicious" cat 7] [Bella an "evil" cat 9] [Scar a "dead cat 11]]
`) `)
} }
func TestToMath(t *testing.T) {
t.Parallel()
files := `
-- hugo.toml --
disableKinds = ['page','rss','section','sitemap','taxonomy','term']
-- layouts/index.html --
{{ $result := transform.ToMath "c = \\pm\\sqrt{a^2 + b^2}" }}
{{ printf "%v" $result | safeHTML }}
`
b := hugolib.Test(t, files)
b.AssertFileContent("public/index.html", `
<span class="katex"><math
`)
}

View file

@ -71,7 +71,7 @@ func (ns *Namespace) Unmarshal(args ...any) (any, error) {
key += decoder.OptionsKey() key += decoder.OptionsKey()
} }
v, err := ns.cache.GetOrCreate(key, func(string) (*resources.StaleValue[any], error) { v, err := ns.cacheUnmarshal.GetOrCreate(key, func(string) (*resources.StaleValue[any], error) {
f := metadecoders.FormatFromStrings(r.MediaType().Suffixes()...) f := metadecoders.FormatFromStrings(r.MediaType().Suffixes()...)
if f == "" { if f == "" {
return nil, fmt.Errorf("MIME %q not supported", r.MediaType()) return nil, fmt.Errorf("MIME %q not supported", r.MediaType())
@ -119,7 +119,7 @@ func (ns *Namespace) Unmarshal(args ...any) (any, error) {
key := hashing.MD5FromStringHexEncoded(dataStr) key := hashing.MD5FromStringHexEncoded(dataStr)
v, err := ns.cache.GetOrCreate(key, func(string) (*resources.StaleValue[any], error) { v, err := ns.cacheUnmarshal.GetOrCreate(key, func(string) (*resources.StaleValue[any], error) {
f := decoder.FormatFromContentString(dataStr) f := decoder.FormatFromContentString(dataStr)
if f == "" { if f == "" {
return nil, errors.New("unknown format") return nil, errors.New("unknown format")