Merge pull request #7880 from overleaf/jk-cm6-parser-migrate-to-web

[web] CM6: Migrate lezer parser to web

GitOrigin-RevId: d6f63ab2f3440292373bcc814c3769285be47c2b
This commit is contained in:
June Kelly 2022-06-23 09:19:07 +01:00 committed by Copybot
parent 496f283892
commit 3d38087297
8 changed files with 94 additions and 1 deletions

25
package-lock.json generated
View file

@ -4067,6 +4067,19 @@
"@lezer/lr": "^0.16.0"
}
},
"node_modules/@lezer/generator": {
"version": "0.16.0",
"resolved": "https://registry.npmjs.org/@lezer/generator/-/generator-0.16.0.tgz",
"integrity": "sha512-dhkChgTGXdgj++0VV/Av1ARNe35Fj5uYuVt7W6g/7cDtHToS1dNn73rUtRb1cRO6kQXaISE/ne1cr3VofyBcng==",
"dev": true,
"dependencies": {
"@lezer/common": "^0.16.0",
"@lezer/lr": "^0.16.0"
},
"bin": {
"lezer-generator": "dist/lezer-generator.cjs"
}
},
"node_modules/@lezer/highlight": {
"version": "0.16.0",
"resolved": "https://registry.npmjs.org/@lezer/highlight/-/highlight-0.16.0.tgz",
@ -35168,6 +35181,7 @@
"@babel/register": "^7.14.5",
"@cypress/react": "^5.12.5",
"@juggle/resize-observer": "^3.3.1",
"@lezer/generator": "^0.16.0",
"@testing-library/cypress": "^8.0.3",
"@testing-library/dom": "^8.13.0",
"@testing-library/react": "^12.1.5",
@ -39606,6 +39620,16 @@
"@lezer/lr": "^0.16.0"
}
},
"@lezer/generator": {
"version": "0.16.0",
"resolved": "https://registry.npmjs.org/@lezer/generator/-/generator-0.16.0.tgz",
"integrity": "sha512-dhkChgTGXdgj++0VV/Av1ARNe35Fj5uYuVt7W6g/7cDtHToS1dNn73rUtRb1cRO6kQXaISE/ne1cr3VofyBcng==",
"dev": true,
"requires": {
"@lezer/common": "^0.16.0",
"@lezer/lr": "^0.16.0"
}
},
"@lezer/highlight": {
"version": "0.16.0",
"resolved": "https://registry.npmjs.org/@lezer/highlight/-/highlight-0.16.0.tgz",
@ -42588,6 +42612,7 @@
"@cypress/react": "^5.12.5",
"@juggle/resize-observer": "^3.3.1",
"@lezer/common": "^0.16.0",
"@lezer/generator": "^0.16.0",
"@lezer/highlight": "^0.16.0",
"@lezer/lr": "^0.16.3",
"@overleaf/logger": "^3.1.0",

View file

@ -4,3 +4,5 @@ modules/**/scripts
frontend/js/vendor
modules/**/frontend/js/vendor
/public/
modules/source-editor/frontend/js/lezer-latex/latex.mjs
modules/source-editor/frontend/js/lezer-latex/latex.terms.mjs

View file

@ -92,4 +92,8 @@ cypress/downloads/
# Ace themes for conversion
modules/source-editor/frontend/js/themes/ace/
# Compiled parser files
modules/source-editor/frontend/js/lezer-latex/latex.mjs
modules/source-editor/frontend/js/lezer-latex/latex.terms.mjs
!**/fixtures/**/*.log

View file

@ -6,3 +6,5 @@ modules/**/frontend/js/vendor
public/js
public/minjs
frontend/stylesheets/components/nvd3.less
modules/source-editor/frontend/js/lezer-latex/latex.mjs
modules/source-editor/frontend/js/lezer-latex/latex.terms.mjs

View file

@ -29,6 +29,9 @@ FROM deps as dev
COPY services/web /overleaf/services/web
# Build the latex parser
RUN cd /overleaf/services/web && npm run 'lezer-latex:generate'
RUN mkdir -p /overleaf/services/web/data/dumpFolder \
&& mkdir -p /overleaf/services/web/data/logs \
&& mkdir -p /overleaf/services/web/data/pdf \

View file

@ -13,7 +13,7 @@
"test:unit:all": "npm run test:unit:run_dir -- test/unit/src modules/*/test/unit/src",
"test:unit:all:silent": "npm run test:unit:all -- --reporter dot",
"test:unit:app": "npm run test:unit:run_dir -- test/unit/src",
"test:frontend": "NODE_ENV=test TZ=GMT mocha --recursive --timeout 5000 --exit --extension js,ts,tsx --grep=$MOCHA_GREP --require test/frontend/bootstrap.js --ignore '**/*.spec.{js,ts,tsx}' test/frontend modules/*/test/frontend",
"test:frontend": "NODE_ENV=test TZ=GMT mocha --recursive --timeout 5000 --exit --extension js,mjs,ts,tsx --grep=$MOCHA_GREP --require test/frontend/bootstrap.js --ignore '**/*.spec.{js,ts,tsx}' test/frontend modules/*/test/frontend",
"test:frontend:coverage": "c8 --all --include 'frontend/js' --include 'modules/*/frontend/js' --exclude 'frontend/js/vendor' --reporter=lcov --reporter=text-summary npm run test:frontend",
"test:karma": "karma start",
"test:karma:single": "karma start --no-auto-watch --single-run",
@ -34,6 +34,8 @@
"convert-themes": "node modules/source-editor/frontend/js/themes/convert.js",
"cypress:open-ct": "SHARELATEX_CONFIG=$PWD/config/settings.webpack.js cypress open --component",
"cypress:run-ct": "SHARELATEX_CONFIG=$PWD/config/settings.webpack.js cypress run --component",
"lezer-latex:generate": "node modules/source-editor/scripts/lezer-latex/generate.js",
"lezer-latex:run": "node modules/source-editor/scripts/lezer-latex/run.mjs",
"routes": "bin/routes"
},
"browserslist": [
@ -205,6 +207,7 @@
"@babel/register": "^7.14.5",
"@cypress/react": "^5.12.5",
"@juggle/resize-observer": "^3.3.1",
"@lezer/generator": "^0.16.0",
"@testing-library/cypress": "^8.0.3",
"@testing-library/dom": "^8.13.0",
"@testing-library/react": "^12.1.5",

View file

@ -0,0 +1,49 @@
const {
compile,
options,
} = require('../modules/source-editor/scripts/lezer-latex/generate')
const fs = require('fs')
const PLUGIN_NAME = 'lezer-grammar-compiler'
class LezerGrammarCompilerPlugin {
apply(compiler) {
compiler.hooks.make.tap(PLUGIN_NAME, compilation => {
// Add the grammar file to the file paths watched by webpack
compilation.fileDependencies.add(options.grammarPath)
})
compiler.hooks.beforeCompile.tapAsync(
PLUGIN_NAME,
(_compilation, callback) => {
// Check timestamps on grammar and parser files, and re-compile if needed.
// (Note: the compiled parser file is watched by webpack, and so will trigger
// a second compilation immediately after. This seems harmless.)
if (
!fs.existsSync(options.parserOutputPath) ||
!fs.existsSync(options.termsOutputPath)
) {
console.log('Parser does not exist, compiling')
compile()
return callback()
}
fs.stat(options.grammarPath, (err, grammarStat) => {
if (err) {
return callback(err)
}
fs.stat(options.parserOutputPath, (err, parserStat) => {
if (err) {
return callback(err)
}
callback()
if (grammarStat.mtime > parserStat.mtime) {
console.log('Grammar file newer than parser file, re-compiling')
compile()
}
})
})
}
)
}
}
module.exports = { LezerGrammarCompilerPlugin }

View file

@ -4,6 +4,9 @@ const webpack = require('webpack')
const CopyPlugin = require('copy-webpack-plugin')
const WebpackAssetsManifest = require('webpack-assets-manifest')
const MiniCssExtractPlugin = require('mini-css-extract-plugin')
const {
LezerGrammarCompilerPlugin,
} = require('./webpack-plugins/lezer-grammar-compiler')
const PackageVersions = require('./app/src/infrastructure/PackageVersions')
@ -226,6 +229,8 @@ module.exports = {
},
plugins: [
new LezerGrammarCompilerPlugin(),
// Generate a manifest.json file which is used by the backend to map the
// base filenames to the generated output filenames
new WebpackAssetsManifest({