Use Hunspell for client-side spellchecking (#20286)

GitOrigin-RevId: c4d0d9e06fe0cc9d7cb7a058fd0768eb024e44f5
This commit is contained in:
Alf Eaton 2024-10-02 09:25:02 +01:00 committed by Copybot
parent 2d737810d4
commit 24c8629cd4
40 changed files with 5391 additions and 514 deletions

11
package-lock.json generated
View file

@ -8371,6 +8371,11 @@
"resolved": "services/contacts",
"link": true
},
"node_modules/@overleaf/dictionaries": {
"version": "0.0.2",
"resolved": "https://github.com/overleaf/dictionaries/archive/refs/tags/v0.0.2.tar.gz",
"integrity": "sha512-p8QJhmQcZ33PEPe0Lqi7NfFkA9IbZNOvlPk3URKFTeb3qAHW+s5+U9qEnHp1s3xr7ZOgu6D5YPK0fRNOE9+zmg=="
},
"node_modules/@overleaf/docstore": {
"resolved": "services/docstore",
"link": true
@ -43751,6 +43756,7 @@
"@node-oauth/oauth2-server": "^5.1.0",
"@node-saml/passport-saml": "^4.0.4",
"@overleaf/access-token-encryptor": "*",
"@overleaf/dictionaries": "https://github.com/overleaf/dictionaries/archive/refs/tags/v0.0.2.tar.gz",
"@overleaf/fetch-utils": "*",
"@overleaf/logger": "*",
"@overleaf/metrics": "*",
@ -51353,6 +51359,10 @@
}
}
},
"@overleaf/dictionaries": {
"version": "https://github.com/overleaf/dictionaries/archive/refs/tags/v0.0.2.tar.gz",
"integrity": "sha512-p8QJhmQcZ33PEPe0Lqi7NfFkA9IbZNOvlPk3URKFTeb3qAHW+s5+U9qEnHp1s3xr7ZOgu6D5YPK0fRNOE9+zmg=="
},
"@overleaf/docstore": {
"version": "file:services/docstore",
"requires": {
@ -52361,6 +52371,7 @@
"@opentelemetry/semantic-conventions": "^1.15.2",
"@overleaf/access-token-encryptor": "*",
"@overleaf/codemirror-tree-view": "^0.1.3",
"@overleaf/dictionaries": "https://github.com/overleaf/dictionaries/archive/refs/tags/v0.0.2.tar.gz",
"@overleaf/fetch-utils": "*",
"@overleaf/logger": "*",
"@overleaf/metrics": "*",

View file

@ -9,3 +9,4 @@ frontend/js/features/source-editor/lezer-latex/latex.mjs
frontend/js/features/source-editor/lezer-latex/latex.terms.mjs
frontend/js/features/source-editor/lezer-bibtex/bibtex.mjs
frontend/js/features/source-editor/lezer-bibtex/bibtex.terms.mjs
frontend/js/features/source-editor/hunspell/wasm/hunspell.mjs

View file

@ -11,3 +11,4 @@ frontend/js/features/source-editor/lezer-latex/latex.mjs
frontend/js/features/source-editor/lezer-latex/latex.terms.mjs
frontend/js/features/source-editor/lezer-bibtex/bibtex.mjs
frontend/js/features/source-editor/lezer-bibtex/bibtex.terms.mjs
frontend/js/features/source-editor/hunspell/wasm/hunspell.mjs

View file

@ -79,6 +79,7 @@ async function joinProject(req, res, next) {
// disable spellchecking for currently unsupported spell check languages
// preserve the value in the db so they can use it again once we add back
// support.
// TODO: allow these if in client-side spell check split test
if (
unsupportedSpellcheckLanguages.indexOf(project.spellCheckLanguage) !== -1
) {

View file

@ -340,6 +340,7 @@ const _ProjectController = {
'ieee-stylesheet',
'write-and-cite',
'default-visual-for-beginners',
'spell-check-client',
].filter(Boolean)
const getUserValues = async userId =>

View file

@ -7,7 +7,7 @@ const LearnedWordsManager = require('./LearnedWordsManager')
const TEN_SECONDS = 1000 * 10
const languageCodeIsSupported = code =>
Settings.languages.some(lang => lang.code === code)
Settings.languages.some(lang => lang.code === code && lang.server !== false)
module.exports = {
learn(req, res, next) {

View file

@ -174,6 +174,7 @@ module.exports = function (webRouter, privateApiRouter, publicApiRouter) {
}
res.locals.mathJaxPath = `/js/libs/mathjax-${PackageVersions.version.mathjax}/es5/tex-svg-full.js`
res.locals.dictionariesRoot = `/js/dictionaries/${PackageVersions.version.dictionaries}/`
res.locals.lib = PackageVersions.lib

View file

@ -1,5 +1,6 @@
const version = {
mathjax: '3.2.2',
dictionaries: '0.0.2',
}
module.exports = {

View file

@ -47,6 +47,7 @@ html(
//- See: https://webpack.js.org/guides/public-path/#on-the-fly
meta(name="ol-baseAssetPath" content=buildBaseAssetPath())
meta(name="ol-mathJaxPath" content=mathJaxPath)
meta(name="ol-dictionariesRoot" content=dictionariesRoot)
meta(name="ol-usersEmail" content=getUserEmail())
meta(name="ol-ab" data-type="json" content={})

View file

@ -8,10 +8,9 @@ if [[ "$BRANCH_NAME" == "master" || "$BRANCH_NAME" == "main" ]]; then
cd sentry_upload/public
SENTRY_RELEASE=${COMMIT_SHA}
OPTS="--no-rewrite --url-prefix ~"
sentry-cli releases new "$SENTRY_RELEASE"
sentry-cli releases set-commits --auto "$SENTRY_RELEASE"
sentry-cli releases files "$SENTRY_RELEASE" upload-sourcemaps ${OPTS} .
sentry-cli releases files "$SENTRY_RELEASE" upload-sourcemaps --ignore '*/dictionaries/*' --no-rewrite --url-prefix ~ .
sentry-cli releases finalize "$SENTRY_RELEASE"
rm -rf sentry_upload

View file

@ -428,58 +428,115 @@ module.exports = {
},
// Spelling languages
// dic = available in client
// server: false = not available on server
// ------------------
//
// You must have the corresponding aspell package installed to
// be able to use a language.
languages: [
{ code: 'en', name: 'English' },
{ code: 'en_US', name: 'English (American)' },
{ code: 'en_GB', name: 'English (British)' },
{ code: 'en_CA', name: 'English (Canadian)' },
{ code: 'af', name: 'Afrikaans' },
{ code: 'ar', name: 'Arabic' },
{ code: 'gl', name: 'Galician' },
{ code: 'eu', name: 'Basque' },
{ code: 'br', name: 'Breton' },
{ code: 'bg', name: 'Bulgarian' },
{ code: 'ca', name: 'Catalan' },
{ code: 'hr', name: 'Croatian' },
{ code: 'cs', name: 'Czech' },
{ code: 'da', name: 'Danish' },
{ code: 'nl', name: 'Dutch' },
{ code: 'eo', name: 'Esperanto' },
{ code: 'et', name: 'Estonian' },
{ code: 'fo', name: 'Faroese' },
{ code: 'fr', name: 'French' },
{ code: 'de', name: 'German' },
{ code: 'el', name: 'Greek' },
{ code: 'id', name: 'Indonesian' },
{ code: 'ga', name: 'Irish' },
{ code: 'it', name: 'Italian' },
{ code: 'kk', name: 'Kazakh' },
{ code: 'en_US', dic: 'en_US', name: 'English (American)' },
{ code: 'en_GB', dic: 'en_GB', name: 'English (British)' },
{ code: 'en_CA', dic: 'en_CA', name: 'English (Canadian)' },
{
code: 'en_AU',
dic: 'en_AU',
name: 'English (Australian)',
server: false,
},
{
code: 'en_ZA',
dic: 'en_ZA',
name: 'English (South African)',
server: false,
},
{ code: 'af', dic: 'af_ZA', name: 'Afrikaans' },
{ code: 'an', dic: 'an_ES', name: 'Aragonese', server: false },
{ code: 'ar', dic: 'ar', name: 'Arabic' },
{ code: 'be_BY', dic: 'be_BY', name: 'Belarusian', server: false },
{ code: 'gl', dic: 'gl_ES', name: 'Galician' },
{ code: 'eu', dic: 'eu', name: 'Basque' },
{ code: 'bn_BD', dic: 'bn_BD', name: 'Bengali', server: false },
{ code: 'bs_BA', dic: 'bs_BA', name: 'Bosnian', server: false },
{ code: 'br', dic: 'br_FR', name: 'Breton' },
{ code: 'bg', dic: 'bg_BG', name: 'Bulgarian' },
{ code: 'ca', dic: 'ca', name: 'Catalan' },
{ code: 'hr', dic: 'hr_HR', name: 'Croatian' },
{ code: 'cs', dic: 'cs_CZ', name: 'Czech' },
{
code: 'da',
// dic: 'da_DK', TODO: re-enable client spell check
name: 'Danish',
},
{ code: 'nl', dic: 'nl', name: 'Dutch' },
{ code: 'dz', dic: 'dz', name: 'Dzongkha', server: false },
{ code: 'eo', dic: 'eo', name: 'Esperanto' },
{ code: 'et', dic: 'et_EE', name: 'Estonian' },
{ code: 'fo', dic: 'fo', name: 'Faroese' },
{ code: 'fr', dic: 'fr', name: 'French' },
{ code: 'gl_ES', dic: 'gl_ES', name: 'Galician', server: false },
{ code: 'de', dic: 'de_DE', name: 'German' },
{ code: 'de_AT', dic: 'de_AT', name: 'German (Austria)', server: false },
{
code: 'de_CH',
dic: 'de_CH',
name: 'German (Switzerland)',
server: false,
},
{ code: 'el', dic: 'el_GR', name: 'Greek' },
{ code: 'gug_PY', dic: 'gug_PY', name: 'Guarani', server: false },
{ code: 'gu_IN', dic: 'gu_IN', name: 'Gujarati', server: false },
{ code: 'he_IL', dic: 'he_IL', name: 'Hebrew', server: false },
{ code: 'hi_IN', dic: 'hi_IN', name: 'Hindi', server: false },
{ code: 'hu_HU', dic: 'hu_HU', name: 'Hungarian', server: false },
{ code: 'is_IS', dic: 'is_IS', name: 'Icelandic', server: false },
{ code: 'id', dic: 'id_ID', name: 'Indonesian' },
{ code: 'ga', dic: 'ga_IE', name: 'Irish' },
{ code: 'it', dic: 'it_IT', name: 'Italian' },
{ code: 'kk', dic: 'kk_KZ', name: 'Kazakh' },
{ code: 'ko', dic: 'ko', name: 'Korean', server: false },
{ code: 'ku', name: 'Kurdish' },
{ code: 'lv', name: 'Latvian' },
{ code: 'lt', name: 'Lithuanian' },
{ code: 'kmr', dic: 'kmr_Latn', name: 'Kurmanji', server: false },
{ code: 'lv', dic: 'lv_LV', name: 'Latvian' },
{ code: 'lt', dic: 'lt_LT', name: 'Lithuanian' },
{ code: 'lo_LA', dic: 'lo_LA', name: 'Laotian', server: false },
{ code: 'ml_IN', dic: 'ml_IN', name: 'Malayalam', server: false },
{ code: 'mn_MN', dic: 'mn_MN', name: 'Mongolian', server: false },
{ code: 'nr', name: 'Ndebele' },
{ code: 'ne_NP', dic: 'ne_NP', name: 'Nepali', server: false },
{ code: 'ns', name: 'Northern Sotho' },
{ code: 'no', name: 'Norwegian' },
{ code: 'fa', name: 'Persian' },
{ code: 'pl', name: 'Polish' },
{ code: 'pt_BR', name: 'Portuguese (Brazilian)' },
{ code: 'pt_PT', name: 'Portuguese (European)' },
{ code: 'no', dic: 'nn_NO', name: 'Norwegian' },
{ code: 'oc_FR', dic: 'oc_FR', name: 'Occitan', server: false },
{ code: 'fa', dic: 'fa_IR', name: 'Persian' },
{ code: 'pl', dic: 'pl_PL', name: 'Polish' },
{ code: 'pt_BR', dic: 'pt_BR', name: 'Portuguese (Brazilian)' },
{
code: 'pt_PT',
dic: 'pt_PT',
name: 'Portuguese (European)',
server: true,
},
{ code: 'pa', name: 'Punjabi' },
{ code: 'ro', name: 'Romanian' },
{ code: 'ru', name: 'Russian' },
{ code: 'sk', name: 'Slovak' },
{ code: 'sl', name: 'Slovenian' },
{ code: 'ro', dic: 'ro_RO', name: 'Romanian' },
{ code: 'ru', dic: 'ru_RU', name: 'Russian' },
{ code: 'gd_GB', dic: 'gd_GB', name: 'Scottish Gaelic', server: false },
{ code: 'sr_RS', dic: 'sr_RS', name: 'Serbian', server: false },
{ code: 'si_LK', dic: 'si_LK', name: 'Sinhala', server: false },
{ code: 'sk', dic: 'sk_SK', name: 'Slovak' },
{ code: 'sl', dic: 'sl_SI', name: 'Slovenian' },
{ code: 'st', name: 'Southern Sotho' },
{ code: 'es', name: 'Spanish' },
{ code: 'sv', name: 'Swedish' },
{ code: 'tl', name: 'Tagalog' },
{ code: 'es', dic: 'es_ES', name: 'Spanish' },
{ code: 'sw_TZ', dic: 'sw_TZ', name: 'Swahili', server: false },
{ code: 'sv', dic: 'sv_SE', name: 'Swedish' },
{ code: 'tl', dic: 'tl', name: 'Tagalog' },
{ code: 'te_IN', dic: 'te_IN', name: 'Telugu', server: false },
{ code: 'th_TH', dic: 'th_TH', name: 'Thai', server: false },
{ code: 'bo', dic: 'bo', name: 'Tibetan', server: false },
{ code: 'ts', name: 'Tsonga' },
{ code: 'tn', name: 'Tswana' },
{ code: 'tr_TR', dic: 'tr_TR', name: 'Turkish', server: false },
{ code: 'uk_UA', dic: 'uk_UA', name: 'Ukrainian', server: false },
{ code: 'hsb', name: 'Upper Sorbian' },
{ code: 'uz_UZ', dic: 'uz_UZ', name: 'Uzbek', server: false },
{ code: 'vi_VN', dic: 'vi_VN', name: 'Vietnamese', server: false },
{ code: 'cy', name: 'Welsh' },
{ code: 'xh', name: 'Xhosa' },
],

View file

@ -34,6 +34,12 @@ const buildConfig = () => {
'../../frontend/js/features/source-editor/languages/latex/linter/latex-linter.worker'
)
// add entrypoint under '/' for hunspell worker
addWorker(
'hunspell-worker',
'../../frontend/js/features/source-editor/hunspell/hunspell.worker'
)
// add entrypoints under '/' for pdfjs workers
addWorker('pdfjs-dist213', 'pdfjs-dist213/legacy/build/pdf.worker.js')
addWorker('pdfjs-dist401', 'pdfjs-dist401/legacy/build/pdf.worker.mjs')

View file

@ -65,6 +65,7 @@
"add_or_remove_project_from_tag": "",
"add_people": "",
"add_role_and_department": "",
"add_to_dictionary": "",
"add_to_tag": "",
"add_your_comment_here": "",
"add_your_first_group_member_now": "",

View file

@ -1,6 +1,6 @@
import getMeta from '../../utils/meta'
const IGNORED_MISSPELLINGS = [
export const globalLearnedWords = new Set([
'Overleaf',
'overleaf',
'ShareLaTeX',
@ -21,15 +21,15 @@ const IGNORED_MISSPELLINGS = [
'lockdown',
'Coronavirus',
'coronavirus',
]
])
export class IgnoredWords {
public learnedWords!: Set<string>
private ignoredMisspellings: Set<string>
private readonly ignoredMisspellings: Set<string>
constructor() {
this.reset()
this.ignoredMisspellings = new Set(IGNORED_MISSPELLINGS)
this.ignoredMisspellings = globalLearnedWords
window.addEventListener('learnedWords:doreset', () => this.reset()) // for tests
}

View file

@ -4,25 +4,39 @@ import getMeta from '../../../../utils/meta'
import { useProjectSettingsContext } from '../../context/project-settings-context'
import SettingsMenuSelect from './settings-menu-select'
import type { Optgroup } from './settings-menu-select'
import { useFeatureFlag } from '@/shared/context/split-test-context'
// TODO: set to true when ready to show new languages that are only available in the client
const showClientOnlyLanguages = false
export default function SettingsSpellCheckLanguage() {
const { t } = useTranslation()
const languages = getMeta('ol-languages')
const spellCheckClientEnabled = useFeatureFlag('spell-check-client')
const { spellCheckLanguage, setSpellCheckLanguage } =
useProjectSettingsContext()
const optgroup: Optgroup = useMemo(
() => ({
const optgroup: Optgroup = useMemo(() => {
const options = (languages ?? []).filter(lang => {
const clientOnly = lang.server === false
if (clientOnly && !showClientOnlyLanguages) {
return false
}
return spellCheckClientEnabled || !clientOnly
})
return {
label: 'Language',
options:
languages?.map(language => ({
value: language.code,
label: language.name,
})) ?? [],
}),
[languages]
)
options: options.map(language => ({
value: language.code,
label: language.name,
})),
}
}, [languages, spellCheckClientEnabled])
return (
<SettingsMenuSelect

View file

@ -1,327 +0,0 @@
import {
StateField,
StateEffect,
EditorSelection,
Prec,
} from '@codemirror/state'
import { EditorView, showTooltip, Tooltip, keymap } from '@codemirror/view'
import { addIgnoredWord } from './ignored-words'
import { learnWordRequest } from './backend'
import { Word, Mark, getMarkAtPosition } from './spellchecker'
import { debugConsole } from '@/utils/debugging'
const ITEMS_TO_SHOW = 8
/*
* The time until which a click event will be ignored, so it doesn't immediately close the spelling menu.
* Safari emits an additional "click" event when event.preventDefault() is called in the "contextmenu" event listener.
*/
let openingUntil = 0
/*
* Hide the spelling menu on click
*/
const handleClickEvent = (event: MouseEvent, view: EditorView) => {
if (Date.now() < openingUntil) {
return
}
if (view.state.field(spellingMenuField, false)) {
view.dispatch({
effects: hideSpellingMenu.of(null),
})
}
}
/*
* Detect when the user right-clicks on a misspelled word,
* and show a menu of suggestions
*/
const handleContextMenuEvent = (event: MouseEvent, view: EditorView) => {
const position = view.posAtCoords(
{
x: event.pageX,
y: event.pageY,
},
false
)
const targetMark = getMarkAtPosition(view, position)
if (!targetMark) {
return
}
const { from, to, value } = targetMark
const targetWord = value.spec.word
if (!targetWord) {
debugConsole.debug(
'>> spelling no word associated with decorated range, stopping'
)
return
}
event.preventDefault()
openingUntil = Date.now() + 100
view.dispatch({
selection: EditorSelection.range(from, to),
effects: showSpellingMenu.of({
mark: targetMark,
word: targetWord,
}),
})
}
const handleShortcutEvent = (view: EditorView) => {
const targetMark = getMarkAtPosition(view, view.state.selection.main.from)
if (!targetMark || !targetMark.value) {
return false
}
view.dispatch({
selection: EditorSelection.range(targetMark.from, targetMark.to),
effects: showSpellingMenu.of({
mark: targetMark,
word: targetMark.value.spec.word,
}),
})
return true
}
/*
* Spelling menu "tooltip" field.
* Manages the menu of suggestions shown on right-click
*/
export const spellingMenuField = StateField.define<Tooltip | null>({
create() {
return null
},
update(value, transaction) {
if (value) {
value = {
...value,
pos: transaction.changes.mapPos(value.pos),
end: value.end ? transaction.changes.mapPos(value.end) : undefined,
}
}
for (const effect of transaction.effects) {
if (effect.is(hideSpellingMenu)) {
value = null
} else if (effect.is(showSpellingMenu)) {
const { mark, word } = effect.value
// Build a "Tooltip" showing the suggestions
value = {
pos: mark.from,
end: mark.to,
above: false,
strictSide: false,
create: view => {
return createSpellingSuggestionList(word, view)
},
}
}
}
return value
},
provide: field => {
return [
showTooltip.from(field),
EditorView.domEventHandlers({
contextmenu: handleContextMenuEvent,
click: handleClickEvent,
}),
Prec.highest(
keymap.of([
{ key: 'Ctrl-Space', run: handleShortcutEvent },
{ key: 'Alt-Space', run: handleShortcutEvent },
])
),
]
},
})
const showSpellingMenu = StateEffect.define<{ mark: Mark; word: Word }>()
const hideSpellingMenu = StateEffect.define()
/*
* Creates the suggestion menu dom, to be displayed in the
* spelling menu "tooltip"
* */
const createSpellingSuggestionList = (word: Word, view: EditorView) => {
// Wrapper div.
// Note, CM6 doesn't like showing complex elements
// 'inside' its Tooltip element, so we style this
// wrapper div to be basically invisible, and allow
// the dropdown list to hang off of it, giving the illusion that
// the list _is_ the tooltip.
// See the theme in spelling/index for styling that makes this work.
const dom = document.createElement('div')
dom.classList.add('ol-cm-spelling-context-menu-tooltip')
// List
const list = document.createElement('ul')
list.setAttribute('tabindex', '0')
list.setAttribute('role', 'menu')
list.addEventListener('keydown', event => {
if (event.code === 'Tab') {
// preventing selecting next element
event.preventDefault()
}
})
list.addEventListener('keyup', event => {
switch (event.code) {
case 'ArrowDown': {
// get currently selected option
const selectedButton =
list.querySelector<HTMLButtonElement>('li button:focus')
if (!selectedButton) {
return list
.querySelector<HTMLButtonElement>('li[role="option"] button')
?.focus()
}
// get next option
let nextElement = selectedButton.parentElement?.nextElementSibling
if (nextElement?.getAttribute('role') !== 'option') {
nextElement = nextElement?.nextElementSibling
}
nextElement?.querySelector('button')?.focus()
break
}
case 'ArrowUp': {
// get currently selected option
const selectedButton =
list.querySelector<HTMLButtonElement>('li button:focus')
if (!selectedButton) {
return list
.querySelector<HTMLButtonElement>(
'li[role="option"]:last-child button'
)
?.focus()
}
// get previous option
let previousElement =
selectedButton.parentElement?.previousElementSibling
if (previousElement?.getAttribute('role') !== 'option') {
previousElement = previousElement?.previousElementSibling
}
previousElement?.querySelector('button')?.focus()
break
}
case 'Escape':
case 'Tab': {
view.dispatch({
effects: hideSpellingMenu.of(null),
})
view.focus()
break
}
}
})
list.classList.add('dropdown-menu', 'dropdown-menu-unpositioned')
// List items, with links inside
if (Array.isArray(word.suggestions)) {
for (const suggestion of word.suggestions.slice(0, ITEMS_TO_SHOW)) {
const li = makeLinkItem(suggestion, event => {
const text = (event.target as HTMLElement).innerText
handleCorrectWord(word, text, view)
event.preventDefault()
})
list.appendChild(li)
}
}
setTimeout(() => {
list.querySelector<HTMLButtonElement>('li:first-child button')?.focus()
}, 0)
// Divider
const divider = document.createElement('li')
divider.classList.add('divider')
list.append(divider)
// Add to Dictionary
const addToDictionary = makeLinkItem(
'Add to Dictionary',
async function (event: Event) {
await handleLearnWord(word, view)
event.preventDefault()
}
)
list.append(addToDictionary)
dom.appendChild(list)
return { dom }
}
const makeLinkItem = (suggestion: string, handler: EventListener) => {
const li = document.createElement('li')
const button = document.createElement('button')
li.setAttribute('role', 'option')
button.classList.add('btn-link', 'text-left', 'dropdown-menu-button')
button.onclick = handler
button.textContent = suggestion
li.appendChild(button)
return li
}
/*
* Learn a word, adding it to the local cache
* and sending it to the spelling backend
*/
const handleLearnWord = async function (word: Word, view: EditorView) {
try {
await learnWordRequest(word)
view.dispatch({
effects: [addIgnoredWord.of(word), hideSpellingMenu.of(null)],
})
} catch (err) {
debugConsole.error(err)
}
}
/*
* Correct a word, removing the marked range
* and replacing it with the chosen text
*/
const handleCorrectWord = (word: Word, text: string, view: EditorView) => {
const tooltip = view.state.field(spellingMenuField)
if (!tooltip) {
throw new Error('No active tooltip')
}
const existingText = view.state.doc.sliceString(tooltip.pos, tooltip.end)
// Defend against erroneous replacement, if the word at this
// position is not actually what we think it is
if (existingText !== word.text) {
debugConsole.debug(
'>> spelling word-to-correct does not match, stopping',
tooltip.pos,
tooltip.end,
existingText,
word
)
return
}
view.dispatch({
changes: [
{
from: tooltip.pos,
to: tooltip.end,
insert: text,
},
],
effects: [hideSpellingMenu.of(null)],
})
view.focus()
}

View file

@ -0,0 +1,224 @@
import {
StateField,
StateEffect,
EditorSelection,
Prec,
} from '@codemirror/state'
import { EditorView, showTooltip, Tooltip, keymap } from '@codemirror/view'
import { addIgnoredWord } from './ignored-words'
import { learnWordRequest } from './backend'
import { Word, Mark, getMarkAtPosition } from './spellchecker'
import { debugConsole } from '@/utils/debugging'
import {
getSpellChecker,
getSpellCheckLanguage,
} from '@/features/source-editor/extensions/spelling/index'
import { sendMB } from '@/infrastructure/event-tracking'
import ReactDOM from 'react-dom'
import { SpellingSuggestions } from '@/features/source-editor/extensions/spelling/spelling-suggestions'
import { SplitTestProvider } from '@/shared/context/split-test-context'
/*
* The time until which a click event will be ignored, so it doesn't immediately close the spelling menu.
* Safari emits an additional "click" event when event.preventDefault() is called in the "contextmenu" event listener.
*/
let openingUntil = 0
/*
* Hide the spelling menu on click
*/
const handleClickEvent = (event: MouseEvent, view: EditorView) => {
if (Date.now() < openingUntil) {
return
}
if (view.state.field(spellingMenuField, false)) {
view.dispatch({
effects: hideSpellingMenu.of(null),
})
}
}
/*
* Detect when the user right-clicks on a misspelled word,
* and show a menu of suggestions
*/
const handleContextMenuEvent = (event: MouseEvent, view: EditorView) => {
const position = view.posAtCoords(
{
x: event.pageX,
y: event.pageY,
},
false
)
const targetMark = getMarkAtPosition(view, position)
if (!targetMark) {
return
}
const { from, to, value } = targetMark
const targetWord = value.spec.word
if (!targetWord) {
debugConsole.debug(
'>> spelling no word associated with decorated range, stopping'
)
return
}
event.preventDefault()
openingUntil = Date.now() + 100
view.dispatch({
selection: EditorSelection.range(from, to),
effects: showSpellingMenu.of({
mark: targetMark,
word: targetWord,
}),
})
}
const handleShortcutEvent = (view: EditorView) => {
const targetMark = getMarkAtPosition(view, view.state.selection.main.from)
if (!targetMark || !targetMark.value) {
return false
}
view.dispatch({
selection: EditorSelection.range(targetMark.from, targetMark.to),
effects: showSpellingMenu.of({
mark: targetMark,
word: targetMark.value.spec.word,
}),
})
return true
}
/*
* Spelling menu "tooltip" field.
* Manages the menu of suggestions shown on right-click
*/
export const spellingMenuField = StateField.define<Tooltip | null>({
create() {
return null
},
update(value, transaction) {
if (value) {
value = {
...value,
pos: transaction.changes.mapPos(value.pos),
end: value.end ? transaction.changes.mapPos(value.end) : undefined,
}
}
for (const effect of transaction.effects) {
if (effect.is(hideSpellingMenu)) {
value = null
} else if (effect.is(showSpellingMenu)) {
const { mark, word } = effect.value
// Build a "Tooltip" showing the suggestions
value = {
pos: mark.from,
end: mark.to,
above: false,
strictSide: false,
create: createSpellingSuggestionList(word),
}
}
}
return value
},
provide: field => {
return [
showTooltip.from(field),
EditorView.domEventHandlers({
contextmenu: handleContextMenuEvent,
click: handleClickEvent,
}),
Prec.highest(
keymap.of([
{ key: 'Ctrl-Space', run: handleShortcutEvent },
{ key: 'Alt-Space', run: handleShortcutEvent },
])
),
]
},
})
const showSpellingMenu = StateEffect.define<{ mark: Mark; word: Word }>()
export const hideSpellingMenu = StateEffect.define()
/*
* Creates the suggestion menu dom, to be displayed in the
* spelling menu "tooltip"
* */
const createSpellingSuggestionList = (word: Word) => (view: EditorView) => {
const dom = document.createElement('div')
dom.classList.add('ol-cm-spelling-context-menu-tooltip')
ReactDOM.render(
<SplitTestProvider>
<SpellingSuggestions
word={word}
spellCheckLanguage={getSpellCheckLanguage(view.state)}
spellChecker={getSpellChecker(view.state)}
handleClose={() => {
view.dispatch({
effects: hideSpellingMenu.of(null),
})
view.focus()
}}
handleLearnWord={() => {
learnWordRequest(word)
.then(() => {
view.dispatch({
effects: [addIgnoredWord.of(word), hideSpellingMenu.of(null)],
})
sendMB('spelling-word-added', {
language: getSpellCheckLanguage(view.state),
})
})
.catch(error => {
debugConsole.error(error)
})
}}
handleCorrectWord={(text: string) => {
const tooltip = view.state.field(spellingMenuField)
if (!tooltip) {
throw new Error('No active tooltip')
}
const existingText = view.state.doc.sliceString(
tooltip.pos,
tooltip.end
)
if (existingText !== word.text) {
return
}
view.dispatch({
changes: [{ from: tooltip.pos, to: tooltip.end, insert: text }],
effects: [hideSpellingMenu.of(null)],
})
view.focus()
sendMB('spelling-suggestion-click', {
language: getSpellCheckLanguage(view.state),
})
}}
/>
</SplitTestProvider>,
dom
)
const destroy = () => {
ReactDOM.unmountComponentAtNode(dom)
}
return { dom, destroy }
}

View file

@ -20,6 +20,12 @@ export const addIgnoredWord = StateEffect.define<{
text: string
}>()
export const removeIgnoredWord = StateEffect.define<{
text: string
}>()
export const updateAfterAddingIgnoredWord = StateEffect.define<string>()
export const updateAfterRemovingIgnoredWord = StateEffect.define<string>()
export const resetSpellChecker = StateEffect.define()

View file

@ -1,26 +1,29 @@
import { EditorView, ViewPlugin } from '@codemirror/view'
import {
Compartment,
Facet,
EditorState,
StateEffect,
StateField,
TransactionSpec,
} from '@codemirror/state'
import { misspelledWordsField, resetMisspelledWords } from './misspelled-words'
import { misspelledWordsField } from './misspelled-words'
import {
addIgnoredWord,
ignoredWordsField,
removeIgnoredWord,
resetSpellChecker,
updateAfterAddingIgnoredWord,
} from './ignored-words'
import { addWordToCache, cacheField, removeWordFromCache } from './cache'
import { spellingMenuField } from './context-menu'
import { hideSpellingMenu, spellingMenuField } from './context-menu'
import { SpellChecker } from './spellchecker'
import { parserWatcher } from '../wait-for-parser'
import type { HunspellManager } from '@/features/source-editor/hunspell/HunspellManager'
import { debugConsole } from '@/utils/debugging'
const spellCheckLanguageConf = new Compartment()
const spellCheckLanguageFacet = Facet.define<string | undefined>()
type Options = { spellCheckLanguage?: string }
type Options = {
spellCheckLanguage?: string
hunspellManager?: HunspellManager
}
/**
* A custom extension that creates a spell checker for the current language (from the user settings).
@ -29,12 +32,16 @@ type Options = { spellCheckLanguage?: string }
* Mis-spelled words are decorated with a Mark decoration.
* The suggestions menu is displayed in a tooltip, activated with a right-click on the decoration.
*/
export const spelling = ({ spellCheckLanguage }: Options) => {
export const spelling = ({ spellCheckLanguage, hunspellManager }: Options) => {
return [
spellingTheme,
parserWatcher,
spellCheckLanguageConf.of(spellCheckLanguageFacet.of(spellCheckLanguage)),
spellCheckField,
spellCheckLanguageField.init(() => spellCheckLanguage),
spellCheckerField.init(() =>
spellCheckLanguage
? new SpellChecker(spellCheckLanguage, hunspellManager)
: null
),
misspelledWordsField,
ignoredWordsField,
cacheField,
@ -56,16 +63,27 @@ const spellingTheme = EditorView.baseTheme({
},
})
const spellCheckField = StateField.define<SpellChecker | null>({
create(state) {
const [spellCheckLanguage] = state.facet(spellCheckLanguageFacet)
return spellCheckLanguage ? new SpellChecker(spellCheckLanguage) : null
export const getSpellChecker = (state: EditorState) =>
state.field(spellCheckerField, false)
const spellCheckerField = StateField.define<SpellChecker | null>({
create() {
return null
},
update(value, tr) {
for (const effect of tr.effects) {
if (effect.is(setSpellCheckLanguageEffect)) {
value?.destroy()
return effect.value ? new SpellChecker(effect.value) : null
value = effect.value.spellCheckLanguage
? new SpellChecker(
effect.value.spellCheckLanguage,
effect.value.hunspellManager
)
: null
} else if (effect.is(addIgnoredWord)) {
value?.addWord(effect.value.text).catch(debugConsole.error)
} else if (effect.is(removeIgnoredWord)) {
value?.removeWord(effect.value.text).catch(debugConsole.error)
}
}
return value
@ -80,7 +98,7 @@ const spellCheckField = StateField.define<SpellChecker | null>({
}
}),
EditorView.domEventHandlers({
focus: (event, view) => {
focus: (_event, view) => {
if (view.state.facet(EditorView.editable)) {
view.state.field(field)?.scheduleSpellCheck(view)
}
@ -95,18 +113,39 @@ const spellCheckField = StateField.define<SpellChecker | null>({
},
})
const setSpellCheckLanguageEffect = StateEffect.define<string | undefined>()
export const getSpellCheckLanguage = (state: EditorState) =>
state.field(spellCheckLanguageField, false)
export const setSpelling = ({
const spellCheckLanguageField = StateField.define<string | undefined>({
create() {
return undefined
},
update(value, tr) {
for (const effect of tr.effects) {
if (effect.is(setSpellCheckLanguageEffect)) {
value = effect.value.spellCheckLanguage
}
}
return value
},
})
export const setSpellCheckLanguageEffect = StateEffect.define<{
spellCheckLanguage: string | undefined
hunspellManager?: HunspellManager
}>()
export const setSpellCheckLanguage = ({
spellCheckLanguage,
hunspellManager,
}: Options): TransactionSpec => {
return {
effects: [
resetMisspelledWords.of(null),
spellCheckLanguageConf.reconfigure(
spellCheckLanguageFacet.of(spellCheckLanguage)
),
setSpellCheckLanguageEffect.of(spellCheckLanguage),
setSpellCheckLanguageEffect.of({
spellCheckLanguage,
hunspellManager,
}),
hideSpellingMenu.of(null),
],
}
}
@ -137,6 +176,7 @@ export const removeLearnedWord = (
lang: spellCheckLanguage,
wordText: word,
}),
removeIgnoredWord.of({ text: word }),
],
}
}

View file

@ -2,11 +2,10 @@ import { StateField, StateEffect } from '@codemirror/state'
import { EditorView, Decoration, DecorationSet } from '@codemirror/view'
import { updateAfterAddingIgnoredWord } from './ignored-words'
import { Word } from './spellchecker'
import { setSpellCheckLanguageEffect } from '@/features/source-editor/extensions/spelling/index'
export const addMisspelledWords = StateEffect.define<Word[]>()
export const resetMisspelledWords = StateEffect.define()
const createMark = (word: Word) => {
return Decoration.mark({
class: 'ol-cm-spelling-error',
@ -39,14 +38,17 @@ export const misspelledWordsField = StateField.define<DecorationSet>({
if (effect.is(addMisspelledWords)) {
// Merge the new misspelled words into the existing set of marks
marks = marks.update({
add: effect.value.map(word => createMark(word)),
add: effect.value.map(word => createMark(word)), // TODO: make sure these positions are still accurate
sort: true,
})
} else if (effect.is(updateAfterAddingIgnoredWord)) {
// Remove a misspelled word, all instances that match text
const word = effect.value
marks = removeAllMarksMatchingWordText(marks, word)
} else if (effect.is(resetMisspelledWords)) {
// Remove existing marks matching the text of a supplied word
marks = marks.update({
filter(_from, _to, mark) {
return mark.spec.word.text !== effect.value
},
})
} else if (effect.is(setSpellCheckLanguageEffect)) {
marks = Decoration.none
}
}
@ -56,14 +58,3 @@ export const misspelledWordsField = StateField.define<DecorationSet>({
return EditorView.decorations.from(field)
},
})
/*
* Remove existing marks matching the text of a supplied word
*/
const removeAllMarksMatchingWordText = (marks: DecorationSet, word: string) => {
return marks.update({
filter: (from, to, mark) => {
return mark.spec.word.text !== word
},
})
}

View file

@ -13,6 +13,7 @@ import {
} from '../../utils/tree-query'
import { waitForParser } from '../wait-for-parser'
import { debugConsole } from '@/utils/debugging'
import type { HunspellManager } from '../../hunspell/HunspellManager'
/*
* Spellchecker, handles updates, schedules spelling checks
@ -26,13 +27,17 @@ export class SpellChecker {
private trackedChanges: ChangeSet
// eslint-disable-next-line no-useless-constructor
constructor(private readonly language: string) {
this.language = language
constructor(
private readonly language: string,
private hunspellManager?: HunspellManager
) {
debugConsole.log('SpellChecker', language, hunspellManager)
this.trackedChanges = ChangeSet.empty(0)
}
destroy() {
this._clearPendingSpellCheck()
// this.hunspellManager?.destroy()
}
_abortRequest() {
@ -86,7 +91,7 @@ export class SpellChecker {
wordsToCheck
)
const processResult = (
misspellings: { index: number; suggestions: string[] }[]
misspellings: { index: number; suggestions?: string[] }[]
) => {
this.trackedChanges = ChangeSet.empty(0)
@ -108,18 +113,79 @@ export class SpellChecker {
} else {
this._abortRequest()
this.abortController = new AbortController()
spellCheckRequest(this.language, unknownWords, this.abortController)
.then(result => {
this.abortController = null
processResult(result.misspellings)
})
.catch(error => {
this.abortController = null
debugConsole.error(error)
})
if (this.hunspellManager) {
const signal = this.abortController.signal
this.hunspellManager.send(
{
type: 'spell',
words: unknownWords.map(word => word.text),
},
(result: any) => {
if (!signal.aborted) {
if (result.error) {
debugConsole.error(result.error)
} else {
processResult(result.misspellings)
}
}
}
)
} else {
spellCheckRequest(this.language, unknownWords, this.abortController)
.then(result => {
this.abortController = null
return processResult(result.misspellings)
})
.catch(error => {
this.abortController = null
debugConsole.error(error)
})
}
}
}
suggest(word: string) {
return new Promise<{ suggestions: string[] }>((resolve, reject) => {
if (this.hunspellManager) {
this.hunspellManager.send({ type: 'suggest', word }, result => {
if ((result as { error: true }).error) {
reject(new Error())
} else {
resolve(result as { suggestions: string[] })
}
})
}
})
}
addWord(word: string) {
return new Promise<void>((resolve, reject) => {
if (this.hunspellManager) {
this.hunspellManager.send({ type: 'add_word', word }, result => {
if ((result as { error: true }).error) {
reject(new Error())
} else {
resolve()
}
})
}
})
}
removeWord(word: string) {
return new Promise<void>((resolve, reject) => {
if (this.hunspellManager) {
this.hunspellManager.send({ type: 'remove_word', word }, result => {
if ((result as { error: true }).error) {
reject(new Error())
} else {
resolve()
}
})
}
})
}
_spellCheckWhenParserReady(view: EditorView) {
if (this.waitingForParser) {
return
@ -163,7 +229,7 @@ export class SpellChecker {
const { from, to } = view.viewport
const changedLineNumbers = new Set<number>()
if (this.trackedChanges.length > 0) {
this.trackedChanges.iterChangedRanges((fromA, toA, fromB, toB) => {
this.trackedChanges.iterChangedRanges((_fromA, _toA, fromB, toB) => {
if (fromB <= to && toB >= from) {
const fromLine = view.state.doc.lineAt(fromB).number
const toLine = view.state.doc.lineAt(toB).number
@ -180,7 +246,9 @@ export class SpellChecker {
}
}
const ignoredWords = view.state.field(ignoredWordsField)
const ignoredWords = this.hunspellManager
? null
: view.state.field(ignoredWordsField)
for (const i of changedLineNumbers) {
const line = view.state.doc.line(i)
wordsToCheck.push(
@ -228,7 +296,7 @@ export class Word {
export const buildSpellCheckResult = (
knownMisspelledWords: Word[],
unknownWords: Word[],
misspellings: { index: number; suggestions: string[] }[]
misspellings: { index: number; suggestions?: string[] }[]
) => {
const cacheAdditions: [Word, string[] | boolean][] = []
@ -277,7 +345,7 @@ export const compileEffects = (results: {
export const getWordsFromLine = (
view: EditorView,
line: Line,
ignoredWords: IgnoredWords,
ignoredWords: IgnoredWords | null,
lang: string
): Word[] => {
const normalTextSpans: Array<NormalTextSpan> = getNormalTextSpansFromLine(
@ -287,14 +355,8 @@ export const getWordsFromLine = (
const words: Word[] = []
for (const span of normalTextSpans) {
for (const match of span.text.matchAll(WORD_REGEX)) {
let word = match[0]
if (word.startsWith("'")) {
word = word.slice(1)
}
if (word.endsWith("'")) {
word = word.slice(0, -1)
}
if (!ignoredWords.has(word)) {
const word = match[0].replace(/^'+/, '').replace(/'+$/, '')
if (!ignoredWords?.has(word)) {
const from = span.from + match.index
words.push(
new Word({

View file

@ -0,0 +1,189 @@
import {
FC,
MouseEventHandler,
useCallback,
useEffect,
useMemo,
useState,
} from 'react'
import { SpellChecker, Word } from './spellchecker'
import { useTranslation } from 'react-i18next'
import SplitTestBadge from '@/shared/components/split-test-badge'
import getMeta from '@/utils/meta'
import classnames from 'classnames'
import { SpellCheckLanguage } from '../../../../../../types/project-settings'
import Icon from '@/shared/components/icon'
import { useFeatureFlag } from '@/shared/context/split-test-context'
import { sendMB } from '@/infrastructure/event-tracking'
const ITEMS_TO_SHOW = 8
// TODO: messaging below the spelling suggestions
const SHOW_FOOTER = false
// (index % length) that works for negative index
const wrapArrayIndex = (index: number, length: number) =>
((index % length) + length) % length
export const SpellingSuggestions: FC<{
word: Word
spellCheckLanguage?: string
spellChecker?: SpellChecker | null
handleClose: () => void
handleLearnWord: () => void
handleCorrectWord: (text: string) => void
}> = ({
word,
spellCheckLanguage,
spellChecker,
handleClose,
handleLearnWord,
handleCorrectWord,
}) => {
const { t } = useTranslation()
const [suggestions, setSuggestions] = useState(() =>
Array.isArray(word.suggestions)
? word.suggestions.slice(0, ITEMS_TO_SHOW)
: []
)
const [waiting, setWaiting] = useState(!word.suggestions)
const [selectedIndex, setSelectedIndex] = useState(0)
const itemsLength = suggestions.length + 1
useEffect(() => {
if (!word.suggestions) {
spellChecker?.suggest(word.text).then(result => {
setSuggestions(result.suggestions.slice(0, ITEMS_TO_SHOW))
setWaiting(false)
sendMB('spelling-suggestion-shown', {
language: spellCheckLanguage,
count: result.suggestions.length,
// word: transaction.state.sliceDoc(mark.from, mark.to),
})
})
}
}, [word, spellChecker, spellCheckLanguage])
const language = useMemo(() => {
if (spellCheckLanguage) {
return getMeta('ol-languages').find(
item => item.code === spellCheckLanguage
)
}
}, [spellCheckLanguage])
return (
<ul
className={classnames('dropdown-menu', 'dropdown-menu-unpositioned', {
hidden: waiting,
})}
tabIndex={0}
role="menu"
onKeyDown={event => {
switch (event.code) {
case 'ArrowDown':
setSelectedIndex(value => wrapArrayIndex(value + 1, itemsLength))
break
case 'ArrowUp':
setSelectedIndex(value => wrapArrayIndex(value - 1, itemsLength))
break
case 'Escape':
case 'Tab':
event.preventDefault()
handleClose()
break
}
}}
>
{Array.isArray(suggestions) && (
<>
{suggestions.map((suggestion, index) => (
<ListItem
key={suggestion}
content={suggestion}
selected={index === selectedIndex}
handleClick={event => {
event.preventDefault()
handleCorrectWord(suggestion)
}}
/>
))}
{suggestions.length > 0 && <li className="divider" />}
</>
)}
<ListItem
content={t('add_to_dictionary')}
selected={selectedIndex === itemsLength - 1}
handleClick={event => {
event.preventDefault()
handleLearnWord()
}}
/>
{SHOW_FOOTER && language && (
<>
<li className="divider" />
<li>
<Footer language={language} />
</li>
</>
)}
</ul>
)
}
const Footer: FC<{ language: SpellCheckLanguage }> = ({ language }) => {
const spellCheckClientEnabled = useFeatureFlag('spell-check-client')
if (!spellCheckClientEnabled) {
return null
}
return language.dic ? (
<div>
<SplitTestBadge
splitTestName="spell-check-client"
displayOnVariants={['enabled']}
/>{' '}
<small>{language?.name}</small>
</div>
) : (
<div>
<Icon type="warning" fw /> <small>{language?.name}</small>
</div>
)
}
const ListItem: FC<{
content: string
selected: boolean
handleClick: MouseEventHandler<HTMLButtonElement>
}> = ({ content, selected, handleClick }) => {
const handleListItem = useCallback(
(element: HTMLElement | null) => {
if (element && selected) {
window.setTimeout(() => {
element.focus()
})
}
},
[selected]
)
return (
<li role="menuitem">
<button
className="btn-link text-left dropdown-menu-button"
onClick={handleClick}
ref={handleListItem}
>
{content}
</button>
</li>
)
}

View file

@ -37,7 +37,7 @@ import {
addLearnedWord,
removeLearnedWord,
resetLearnedWords,
setSpelling,
setSpellCheckLanguage,
} from '../extensions/spelling'
import {
createChangeManager,
@ -65,6 +65,7 @@ import { setMathPreview } from '@/features/source-editor/extensions/math-preview
import { useRangesContext } from '@/features/review-panel-new/context/ranges-context'
import { updateRanges } from '@/features/source-editor/extensions/ranges'
import { useThreadsContext } from '@/features/review-panel-new/context/threads-context'
import { useHunspell } from '@/features/source-editor/hooks/use-hunspell'
function useCodeMirrorScope(view: EditorView) {
const { fileTreeData } = useFileTreeData()
@ -111,6 +112,8 @@ function useCodeMirrorScope(view: EditorView) {
'project.spellCheckLanguage'
)
const hunspellManager = useHunspell(spellCheckLanguage)
const [visual] = useScopeValue<boolean>('editor.showVisual')
const { referenceKeys } = useReferencesContext()
@ -214,14 +217,16 @@ function useCodeMirrorScope(view: EditorView) {
const spellingRef = useRef({
spellCheckLanguage,
hunspellManager,
})
useEffect(() => {
spellingRef.current = {
spellCheckLanguage,
hunspellManager,
}
view.dispatch(setSpelling(spellingRef.current))
}, [view, spellCheckLanguage])
view.dispatch(setSpellCheckLanguage(spellingRef.current))
}, [view, spellCheckLanguage, hunspellManager])
// listen to doc:after-opened, and focus the editor if it's not a new doc
useEffect(() => {

View file

@ -0,0 +1,35 @@
import { isSplitTestEnabled } from '@/utils/splitTestUtils'
import { useEffect, useState } from 'react'
import getMeta from '@/utils/meta'
import { globalLearnedWords } from '@/features/dictionary/ignored-words'
import { HunspellManager } from '@/features/source-editor/hunspell/HunspellManager'
import { debugConsole } from '@/utils/debugging'
export const useHunspell = (spellCheckLanguage: string | null) => {
const [hunspellManager, setHunspellManager] = useState<HunspellManager>()
useEffect(() => {
if (isSplitTestEnabled('spell-check-client')) {
if (spellCheckLanguage) {
const languages = getMeta('ol-languages')
const lang = languages.find(item => item.code === spellCheckLanguage)
if (lang?.dic) {
const hunspellManager = new HunspellManager(lang.dic, [
...globalLearnedWords,
...getMeta('ol-learnedWords'),
])
setHunspellManager(hunspellManager)
debugConsole.log(spellCheckLanguage, hunspellManager)
return () => {
hunspellManager.destroy()
}
} else {
setHunspellManager(undefined)
}
}
}
}, [spellCheckLanguage])
return hunspellManager
}

View file

@ -0,0 +1,9 @@
FROM emscripten/emsdk:3.1.67
RUN apt-get update && apt-get install -y \
autoconf \
automake \
autopoint \
build-essential \
libtool \
cmake

View file

@ -0,0 +1,111 @@
import { v4 as uuid } from 'uuid'
import { createWorker } from '@/utils/worker'
import getMeta from '@/utils/meta'
import { debugConsole } from '@/utils/debugging'
type Message =
| {
id?: string
type: 'spell'
words: string[]
}
| {
id?: string
type: 'suggest'
word: string
}
| {
id?: string
type: 'add_word'
word: string
}
| {
id?: string
type: 'remove_word'
word: string
}
| {
id?: string
type: 'destroy'
}
export class HunspellManager {
dictionariesRoot: string
hunspellWorker!: Worker
abortController: AbortController | undefined
listening = false
loaded = false
pendingMessages: Message[] = []
callbacks: Map<string, (value: unknown) => void> = new Map()
constructor(
private readonly language: string,
private readonly learnedWords: string[]
) {
const baseAssetPath = new URL(
getMeta('ol-baseAssetPath'),
window.location.href
)
this.dictionariesRoot = new URL(
getMeta('ol-dictionariesRoot'),
baseAssetPath
).toString()
createWorker(() => {
this.hunspellWorker = new Worker(
new URL('./hunspell.worker.ts', import.meta.url),
{ type: 'module' }
)
this.hunspellWorker.addEventListener('message', this.receive.bind(this))
})
}
destroy() {
this.send({ type: 'destroy' }, () => {
this.hunspellWorker.terminate()
})
}
send(message: Message, callback: (value: unknown) => void) {
debugConsole.log(message)
if (callback) {
message.id = uuid()
this.callbacks.set(message.id, callback)
}
if (this.listening) {
this.hunspellWorker.postMessage(message)
} else {
this.pendingMessages.push(message)
}
}
receive(event: MessageEvent) {
debugConsole.log(event.data)
const { id, listening, loaded, ...rest } = event.data
if (id) {
const callback = this.callbacks.get(id)
if (callback) {
this.callbacks.delete(id)
callback(rest)
}
} else if (listening) {
this.listening = true
this.hunspellWorker.postMessage({
type: 'init',
lang: this.language,
learnedWords: this.learnedWords, // TODO: add words
dictionariesRoot: this.dictionariesRoot,
})
for (const message of this.pendingMessages) {
this.hunspellWorker.postMessage(message)
this.pendingMessages.length = 0
}
} else if (loaded) {
this.loaded = true
// TODO: use this to display pending state?
}
}
}

View file

@ -0,0 +1,13 @@
#!/usr/bin/env bash
set -e
# build an Emscripten SDK Docker image with Hunspell's build dependencies installed
docker build --pull --tag overleaf/emsdk .
# compile Hunspell to WASM and copy the output files from the Docker container
docker run --rm \
--workdir /opt \
--volume "$(pwd)/wasm":/wasm \
--volume "$(pwd)/compile.sh":/opt/compile.sh:ro \
overleaf/emsdk \
bash compile.sh

View file

@ -0,0 +1,24 @@
#!/usr/bin/env bash
set -e
COMMIT="e994dceb97fb695bca6bfe5ad5665525426bf01f"
curl -L "https://github.com/hunspell/hunspell/archive/${COMMIT}.tar.gz" | tar xvz
cd "hunspell-${COMMIT}"
autoreconf -fiv
emconfigure ./configure --disable-shared --enable-static
emmake make
em++ \
-s EXPORTED_FUNCTIONS="['_Hunspell_create', '_Hunspell_destroy', '_Hunspell_spell', '_Hunspell_suggest', '_Hunspell_free_list', '_Hunspell_add_dic', '_Hunspell_add', '_Hunspell_remove', '_free', '_malloc', 'FS']" \
-s EXPORTED_RUNTIME_METHODS="['ccall', 'cwrap', 'getValue', 'stringToNewUTF8', 'UTF8ToString', 'WORKERFS']" \
-s ENVIRONMENT=worker \
-s ALLOW_MEMORY_GROWTH \
-lworkerfs.js \
-O2 \
-g2 \
src/hunspell/.libs/libhunspell-1.7.a \
-o hunspell.mjs
cp hunspell.{mjs,wasm} /wasm/

View file

@ -0,0 +1,237 @@
import Hunspell from './wasm/hunspell'
type SpellChecker = {
spell(words: string[]): { index: number }[]
suggest(word: string): string[]
addWord(word: string): void
removeWord(word: string): void
destroy(): void
}
const createSpellChecker = async ({
lang,
learnedWords,
dictionariesRoot,
}: {
lang: string
learnedWords: string[]
dictionariesRoot: string
}) => {
const hunspell = await Hunspell()
const {
cwrap,
FS,
WORKERFS,
stringToNewUTF8,
_malloc,
_free,
getValue,
UTF8ToString,
} = hunspell
// https://github.com/hunspell/hunspell/blob/master/src/hunspell/hunspell.h
// https://github.com/kwonoj/hunspell-asm/blob/master/src/wrapHunspellInterface.ts
const create = cwrap('Hunspell_create', 'number', ['number', 'number'])
const destroy = cwrap('Hunspell_destroy', 'number', ['number', 'number'])
const spell = cwrap('Hunspell_spell', 'number', ['number', 'number'])
const suggest = cwrap('Hunspell_suggest', 'number', [
'number',
'number',
'number',
])
const addWord = cwrap('Hunspell_add', 'number', ['number', 'number'])
const removeWord = cwrap('Hunspell_remove', 'number', ['number', 'number'])
const freeList = cwrap('Hunspell_free_list', 'number', [
'number',
'number',
'number',
])
FS.mkdir('/dictionaries')
const [dic, aff] = await Promise.all([
fetch(new URL(`./${lang}.dic`, dictionariesRoot)).then(response =>
response.blob()
),
fetch(new URL(`./${lang}.aff`, dictionariesRoot)).then(response =>
response.blob()
),
])
FS.mount(
WORKERFS,
{
blobs: [
{ name: 'index.dic', data: dic },
{ name: 'index.aff', data: aff },
],
},
'/dictionaries'
)
const dicPtr = stringToNewUTF8('/dictionaries/index.dic')
const affPtr = stringToNewUTF8('/dictionaries/index.aff')
const spellPtr = create(affPtr, dicPtr)
for (const word of learnedWords) {
const wordPtr = stringToNewUTF8(word)
addWord(spellPtr, wordPtr)
_free(wordPtr)
}
const spellChecker: SpellChecker = {
spell(words) {
const misspellings: { index: number }[] = []
for (const [index, word] of words.entries()) {
const wordPtr = stringToNewUTF8(word)
const spellResult = spell(spellPtr, wordPtr)
_free(wordPtr)
if (spellResult === 0) {
misspellings.push({ index })
}
}
return misspellings
},
suggest(word) {
const suggestions: string[] = []
const suggestionListPtr = _malloc(4)
const wordPtr = stringToNewUTF8(word)
const suggestionCount = suggest(spellPtr, suggestionListPtr, wordPtr)
_free(wordPtr)
const suggestionListValuePtr = getValue(suggestionListPtr, '*')
for (let i = 0; i < suggestionCount; i++) {
const suggestion = UTF8ToString(
getValue(suggestionListValuePtr + i * 4, '*')
)
suggestions.push(suggestion)
}
freeList(spellPtr, suggestionListPtr, suggestionCount)
_free(suggestionListPtr)
return suggestions
},
addWord(word) {
const wordPtr = stringToNewUTF8(word)
const result = addWord(spellPtr, wordPtr)
_free(wordPtr)
if (result !== 0) {
throw new Error('The word could not be added to the dictionary')
}
},
removeWord(word) {
const wordPtr = stringToNewUTF8(word)
const result = removeWord(spellPtr, wordPtr)
_free(wordPtr)
if (result !== 0) {
throw new Error('The word could not be removed from the dictionary')
}
},
destroy() {
destroy(spellPtr)
_free(spellPtr)
_free(dicPtr)
_free(affPtr)
},
}
return spellChecker
}
let spellCheckerPromise: Promise<SpellChecker>
self.addEventListener('message', async event => {
switch (event.data.type) {
case 'init':
try {
spellCheckerPromise = createSpellChecker(event.data)
await spellCheckerPromise
self.postMessage({ loaded: true })
} catch (error) {
console.error(error)
self.postMessage({ error: true })
}
break
case 'spell':
{
const { id, words } = event.data
try {
const spellChecker = await spellCheckerPromise
const misspellings = spellChecker.spell(words)
self.postMessage({ id, misspellings })
} catch (error) {
console.error(error)
self.postMessage({ id, error: true })
}
}
break
case 'suggest':
{
const { id, word } = event.data
try {
const spellChecker = await spellCheckerPromise
const suggestions = spellChecker.suggest(word)
self.postMessage({ id, suggestions })
} catch (error) {
console.error(error)
self.postMessage({ id, error: true })
}
}
break
case 'add_word':
{
const { id, word } = event.data
try {
const spellChecker = await spellCheckerPromise
spellChecker.addWord(word)
self.postMessage({ id })
} catch (error) {
console.error(error)
self.postMessage({ id, error: true })
}
}
break
case 'remove_word':
{
const { id, word } = event.data
try {
const spellChecker = await spellCheckerPromise
spellChecker.removeWord(word)
self.postMessage({ id })
} catch (error) {
console.error(error)
self.postMessage({ id, error: true })
}
}
break
case 'destroy':
{
const { id } = event.data
try {
const spellChecker = await spellCheckerPromise
spellChecker.destroy()
self.postMessage({ id })
} catch (error) {
console.error(error)
self.postMessage({ id, error: true })
}
}
break
}
})
self.postMessage({ listening: true })

View file

@ -0,0 +1,66 @@
/* eslint no-dupe-class-members: 0 */
declare class Hunspell {
cwrap(
method: 'Hunspell_create',
output: string,
input: string[]
): (affPtr: number, dicPtr: number) => number
cwrap(
method: 'Hunspell_destroy',
output: string,
input: string[]
): (spellPtr: number) => number
cwrap(
method: 'Hunspell_spell',
output: string,
input: string[]
): (spellPtr: number, wordPtr: number) => number
cwrap(
method: 'Hunspell_suggest',
output: string,
input: string[]
): (spellPtr: number, suggestionListPtr: number, wordPtr: number) => number
cwrap(
method: 'Hunspell_add',
output: string,
input: string[]
): (spellPtr: number, wordPtr: number) => number
cwrap(
method: 'Hunspell_remove',
output: string,
input: string[]
): (spellPtr: number, wordPtr: number) => number
cwrap(
method: 'Hunspell_free_list',
output: string,
input: string[]
): (spellPtr: number, suggestionListPtr: number, n: number) => number
stringToNewUTF8(input: string): number
UTF8ToString(input: number): string
_malloc(length: number): number
_free(ptr: number): void
getValue(ptr: number, type: string): number
FS: {
mkdir(path: string): void
mount(
type: any,
data: { blobs: Record<{ name: string; data: BlobPart }>[] },
dir: string
): void
}
WORKERFS: any
}
declare const factory = async (options?: Record<string, any>) =>
new Hunspell(options)
export default factory

File diff suppressed because it is too large Load diff

View file

@ -54,6 +54,7 @@ export interface Meta {
'ol-allowedExperiments': string[]
'ol-allowedImageNames': AllowedImageName[]
'ol-anonymous': boolean
'ol-baseAssetPath': string
'ol-bootstrapVersion': 3 | 5
'ol-brandVariation': Record<string, any>
@ -76,6 +77,7 @@ export interface Meta {
'ol-currentUrl': string
'ol-debugPdfDetach': boolean
'ol-detachRole': 'detached' | 'detacher' | ''
'ol-dictionariesRoot': 'string'
'ol-dropbox': { error: boolean; registered: boolean }
'ol-editorThemes': string[]
'ol-email': string

View file

@ -77,6 +77,7 @@
"add_or_remove_project_from_tag": "Add or remove project from tag __tagName__",
"add_people": "Add people",
"add_role_and_department": "Add role and department",
"add_to_dictionary": "Add to Dictionary",
"add_to_tag": "Add to tag",
"add_your_comment_here": "Add your comment here",
"add_your_first_group_member_now": "Add your first group members now",

View file

@ -76,6 +76,7 @@
"@node-oauth/oauth2-server": "^5.1.0",
"@node-saml/passport-saml": "^4.0.4",
"@overleaf/access-token-encryptor": "*",
"@overleaf/dictionaries": "https://github.com/overleaf/dictionaries/archive/refs/tags/v0.0.2.tar.gz",
"@overleaf/fetch-utils": "*",
"@overleaf/logger": "*",
"@overleaf/metrics": "*",

View file

@ -0,0 +1,100 @@
import DictionaryModal from '@/features/dictionary/components/dictionary-modal'
import { EditorProviders } from '../../../helpers/editor-providers'
describe('<DictionaryModalContent />', function () {
beforeEach(function () {
cy.interceptCompile()
})
afterEach(function () {
cy.window().then(win => {
win.dispatchEvent(new CustomEvent('learnedWords:doreset'))
})
})
it('list words', function () {
cy.window().then(win => {
win.metaAttributesCache.set('ol-learnedWords', ['foo', 'bar'])
win.dispatchEvent(new CustomEvent('learnedWords:doreset'))
})
cy.mount(
<EditorProviders>
<DictionaryModal show handleHide={cy.stub()} />
</EditorProviders>
)
cy.findByText('foo')
cy.findByText('bar')
})
it('shows message when empty', function () {
cy.window().then(win => {
win.metaAttributesCache.set('ol-learnedWords', [])
win.dispatchEvent(new CustomEvent('learnedWords:doreset'))
})
cy.mount(
<EditorProviders>
<DictionaryModal show handleHide={cy.stub()} />
</EditorProviders>
)
cy.contains('Your custom dictionary is empty.')
})
it('removes words', function () {
cy.intercept('/spelling/unlearn', { statusCode: 200 })
cy.window().then(win => {
win.metaAttributesCache.set('ol-learnedWords', ['Foo', 'bar'])
win.dispatchEvent(new CustomEvent('learnedWords:doreset'))
})
cy.mount(
<EditorProviders>
<DictionaryModal show handleHide={cy.stub()} />
</EditorProviders>
)
cy.findByText('Foo')
cy.findByText('bar')
cy.findAllByRole('button', {
name: 'Remove from dictionary',
})
.eq(0)
.click()
cy.findByText('bar').should('not.exist')
cy.findByText('Foo')
})
it('handles errors', function () {
cy.intercept('/spelling/unlearn', { statusCode: 500 }).as('unlearn')
cy.window().then(win => {
win.metaAttributesCache.set('ol-learnedWords', ['foo'])
win.dispatchEvent(new CustomEvent('learnedWords:doreset'))
})
cy.mount(
<EditorProviders>
<DictionaryModal show handleHide={cy.stub()} />
</EditorProviders>
)
cy.findByText('foo')
cy.findAllByRole('button', {
name: 'Remove from dictionary',
})
.eq(0)
.click()
cy.wait('@unlearn')
cy.findByText('Sorry, something went wrong')
cy.findByText('foo')
})
})

View file

@ -1,60 +0,0 @@
import {
screen,
fireEvent,
waitForElementToBeRemoved,
} from '@testing-library/react'
import fetchMock from 'fetch-mock'
import DictionaryModal from '../../../../../frontend/js/features/dictionary/components/dictionary-modal'
import { renderWithEditorContext } from '../../../helpers/render-with-context'
function setLearnedWords(words) {
window.metaAttributesCache.set('ol-learnedWords', words)
window.dispatchEvent(new CustomEvent('learnedWords:doreset'))
}
describe('<DictionaryModalContent />', function () {
afterEach(function () {
fetchMock.reset()
setLearnedWords([])
})
it('list words', async function () {
setLearnedWords(['foo', 'bar'])
renderWithEditorContext(<DictionaryModal show handleHide={() => {}} />)
screen.getByText('foo')
screen.getByText('bar')
})
it('shows message when empty', async function () {
setLearnedWords([])
renderWithEditorContext(<DictionaryModal show handleHide={() => {}} />)
screen.getByText('Your custom dictionary is empty.')
})
it('removes words', async function () {
fetchMock.post('/spelling/unlearn', 200)
setLearnedWords(['Foo', 'bar'])
renderWithEditorContext(<DictionaryModal show handleHide={() => {}} />)
screen.getByText('Foo')
screen.getByText('bar')
const [firstButton] = screen.getAllByRole('button', {
name: 'Remove from dictionary',
})
fireEvent.click(firstButton)
await waitForElementToBeRemoved(() => screen.getByText('bar'))
screen.getByText('Foo')
})
it('handles errors', async function () {
fetchMock.post('/spelling/unlearn', 500)
setLearnedWords(['foo'])
renderWithEditorContext(<DictionaryModal show handleHide={() => {}} />)
screen.getByText('foo')
const [firstButton] = screen.getAllByRole('button', {
name: 'Remove from dictionary',
})
fireEvent.click(firstButton)
await fetchMock.flush()
screen.getByText('Sorry, something went wrong')
screen.getByText('foo')
})
})

View file

@ -0,0 +1,85 @@
import '../../../helpers/bootstrap-3'
import { mockScope } from '../helpers/mock-scope'
import { EditorProviders } from '../../../helpers/editor-providers'
import CodeMirrorEditor from '../../../../../frontend/js/features/source-editor/components/codemirror-editor'
import { TestContainer } from '../helpers/test-container'
import forEach from 'mocha-each'
import PackageVersions from '../../../../../app/src/infrastructure/PackageVersions'
const languages = [
{ code: 'en_GB', dic: 'en_GB', name: 'English (British)' },
{ code: 'fr', dic: 'fr', name: 'French' },
{ code: 'sv', dic: 'sv_SE', name: 'Swedish' },
]
const suggestions = {
en_GB: ['medecine', 'medicine'],
fr: ['medecin', 'médecin'],
sv: ['medecin', 'medicin'],
}
forEach(Object.keys(suggestions)).describe(
'Spell check in client (%s)',
(spellCheckLanguage: keyof typeof suggestions) => {
const content = `
\\documentclass{}
\\title{}
\\author{}
\\begin{document}
\\maketitle
\\begin{abstract}
\\end{abstract}
\\section{}
\\end{document}`
beforeEach(function () {
cy.window().then(win => {
win.metaAttributesCache.set('ol-preventCompileOnLoad', true)
win.metaAttributesCache.set('ol-splitTestVariants', {
'spell-check-client': 'enabled',
})
win.metaAttributesCache.set('ol-splitTestInfo', {})
win.metaAttributesCache.set('ol-learnedWords', ['baz'])
win.metaAttributesCache.set(
'ol-dictionariesRoot',
`js/dictionaries/${PackageVersions.version.dictionaries}/`
)
win.metaAttributesCache.set('ol-baseAssetPath', '/__cypress/src/')
win.metaAttributesCache.set('ol-languages', languages)
})
cy.interceptEvents()
const scope = mockScope(content)
scope.project.spellCheckLanguage = spellCheckLanguage
cy.mount(
<TestContainer>
<EditorProviders scope={scope}>
<CodeMirrorEditor />
</EditorProviders>
</TestContainer>
)
cy.get('.cm-line').eq(13).as('line')
cy.get('@line').click()
})
it('shows suggestions for misspelled word', function () {
const [from, to] = suggestions[spellCheckLanguage]
cy.get('@line').type(from)
cy.get('@line').get('.ol-cm-spelling-error').contains(from)
cy.get('@line').get('.ol-cm-spelling-error').rightclick()
cy.findByText(to).click()
cy.get('@line').contains(to)
cy.get('@line').find('.ol-cm-spelling-error').should('not.exist')
})
}
)

View file

@ -30,4 +30,6 @@ export type PdfViewer = 'pdfjs' | 'native'
export type SpellCheckLanguage = {
name: string
code: string
dic?: string
server?: false
}

View file

@ -66,6 +66,7 @@ function getModuleDirectory(moduleName) {
}
const mathjaxDir = getModuleDirectory('mathjax')
const dictionariesDir = getModuleDirectory('@overleaf/dictionaries')
const pdfjsVersions = ['pdfjs-dist213', 'pdfjs-dist401']
@ -78,6 +79,14 @@ if (MATHJAX_VERSION !== PackageVersions.version.mathjax) {
)
}
const DICTIONARIES_VERSION =
require('@overleaf/dictionaries/package.json').version
if (DICTIONARIES_VERSION !== PackageVersions.version.dictionaries) {
throw new Error(
'"@overleaf/dictionaries" version de-synced, update services/web/app/src/infrastructure/PackageVersions.js'
)
}
module.exports = {
// Defines the "entry point(s)" for the application - i.e. the file which
// bootstraps the application
@ -141,6 +150,13 @@ module.exports = {
],
type: 'javascript/auto',
},
{
test: /\.wasm$/,
type: 'asset/resource',
generator: {
filename: 'js/[name]-[contenthash][ext]',
},
},
{
// Pass Less files through less-loader/css-loader/mini-css-extract-
// plugin (note: run in reverse order)
@ -275,6 +291,10 @@ module.exports = {
},
},
experiments: {
asyncWebAssembly: true,
},
plugins: [
new LezerGrammarCompilerPlugin(),
@ -346,6 +366,12 @@ module.exports = {
toType: 'dir',
context: mathjaxDir,
},
{
from: '*',
to: `js/dictionaries/${PackageVersions.version.dictionaries}`,
toType: 'dir',
context: `${dictionariesDir}/dictionaries`,
},
...pdfjsVersions.flatMap(version => {
const dir = getModuleDirectory(version)