mirror of
https://github.com/overleaf/overleaf.git
synced 2024-11-21 20:47:08 -05:00
Merge pull request #12195 from overleaf/bg-use-glob-ignore
Change ignored file matching to use standard `minimatch` module GitOrigin-RevId: ef8f13c039015e7bb9894441fb6170703941653c
This commit is contained in:
Brian Gough
Copybot
parent
a94f241dcb
commit
e46567b696
5 changed files with 164 additions and 50 deletions
147
package-lock.json
generated
147
package-lock.json
generated
|
|
@ -34693,6 +34693,7 @@
|
|||
"express": "^4.18.2",
|
||||
"lodash": "^4.17.21",
|
||||
"lru-cache": "^4.1.5",
|
||||
"minimatch": "^7.4.2",
|
||||
"mongodb": "^4.11.0",
|
||||
"node-fetch": "^2.6.7",
|
||||
"p-limit": "^2.3.0",
|
||||
|
|
@ -34709,6 +34710,14 @@
|
|||
"sinon-stub-promise": "^4.0.0"
|
||||
}
|
||||
},
|
||||
"services/third-party-datastore/node_modules/brace-expansion": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
|
||||
"integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==",
|
||||
"dependencies": {
|
||||
"balanced-match": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"services/third-party-datastore/node_modules/bson": {
|
||||
"version": "4.7.0",
|
||||
"resolved": "https://registry.npmjs.org/bson/-/bson-4.7.0.tgz",
|
||||
|
|
@ -34769,6 +34778,20 @@
|
|||
"yallist": "^2.1.2"
|
||||
}
|
||||
},
|
||||
"services/third-party-datastore/node_modules/minimatch": {
|
||||
"version": "7.4.2",
|
||||
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-7.4.2.tgz",
|
||||
"integrity": "sha512-xy4q7wou3vUoC9k1xGTXc+awNdGaGVHtFUaey8tiX4H1QRc04DZ/rmDFwNm2EBsuYEhAZ6SgMmYf3InGY6OauA==",
|
||||
"dependencies": {
|
||||
"brace-expansion": "^2.0.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=10"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/isaacs"
|
||||
}
|
||||
},
|
||||
"services/third-party-datastore/node_modules/mongodb": {
|
||||
"version": "4.11.0",
|
||||
"resolved": "https://registry.npmjs.org/mongodb/-/mongodb-4.11.0.tgz",
|
||||
|
|
@ -35210,6 +35233,7 @@
|
|||
"mathjax": "^2.7.9",
|
||||
"mathjax-3": "npm:mathjax@^3.2.2",
|
||||
"method-override": "^2.3.3",
|
||||
"minimatch": "^7.4.2",
|
||||
"minimist": "^1.2.7",
|
||||
"mmmagic": "^0.5.3",
|
||||
"moment": "^2.29.4",
|
||||
|
|
@ -36405,6 +36429,18 @@
|
|||
"eslint": "^3 || ^4 || ^5 || ^6 || ^7 || ^8"
|
||||
}
|
||||
},
|
||||
"services/web/node_modules/eslint-plugin-jsx-a11y/node_modules/minimatch": {
|
||||
"version": "3.1.2",
|
||||
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
|
||||
"integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"brace-expansion": "^1.1.7"
|
||||
},
|
||||
"engines": {
|
||||
"node": "*"
|
||||
}
|
||||
},
|
||||
"services/web/node_modules/eslint-plugin-react": {
|
||||
"version": "7.28.0",
|
||||
"resolved": "https://registry.npmjs.org/eslint-plugin-react/-/eslint-plugin-react-7.28.0.tgz",
|
||||
|
|
@ -36457,6 +36493,18 @@
|
|||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"services/web/node_modules/eslint-plugin-react/node_modules/minimatch": {
|
||||
"version": "3.1.2",
|
||||
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
|
||||
"integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"brace-expansion": "^1.1.7"
|
||||
},
|
||||
"engines": {
|
||||
"node": "*"
|
||||
}
|
||||
},
|
||||
"services/web/node_modules/events": {
|
||||
"version": "3.3.0",
|
||||
"resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz",
|
||||
|
|
@ -36690,6 +36738,18 @@
|
|||
"webpack": "^5.0.0"
|
||||
}
|
||||
},
|
||||
"services/web/node_modules/karma-webpack/node_modules/minimatch": {
|
||||
"version": "3.1.2",
|
||||
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
|
||||
"integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"brace-expansion": "^1.1.7"
|
||||
},
|
||||
"engines": {
|
||||
"node": "*"
|
||||
}
|
||||
},
|
||||
"services/web/node_modules/karma-webpack/node_modules/webpack-merge": {
|
||||
"version": "4.2.2",
|
||||
"resolved": "https://registry.npmjs.org/webpack-merge/-/webpack-merge-4.2.2.tgz",
|
||||
|
|
@ -36832,6 +36892,28 @@
|
|||
"url": "https://opencollective.com/webpack"
|
||||
}
|
||||
},
|
||||
"services/web/node_modules/minimatch": {
|
||||
"version": "7.4.2",
|
||||
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-7.4.2.tgz",
|
||||
"integrity": "sha512-xy4q7wou3vUoC9k1xGTXc+awNdGaGVHtFUaey8tiX4H1QRc04DZ/rmDFwNm2EBsuYEhAZ6SgMmYf3InGY6OauA==",
|
||||
"dependencies": {
|
||||
"brace-expansion": "^2.0.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=10"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/isaacs"
|
||||
}
|
||||
},
|
||||
"services/web/node_modules/minimatch/node_modules/brace-expansion": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
|
||||
"integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==",
|
||||
"dependencies": {
|
||||
"balanced-match": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"services/web/node_modules/mongodb": {
|
||||
"version": "4.13.0",
|
||||
"resolved": "https://registry.npmjs.org/mongodb/-/mongodb-4.13.0.tgz",
|
||||
|
|
@ -44467,6 +44549,7 @@
|
|||
"express": "^4.18.2",
|
||||
"lodash": "^4.17.21",
|
||||
"lru-cache": "^4.1.5",
|
||||
"minimatch": "*",
|
||||
"mocha": "^10.2.0",
|
||||
"mongodb": "^4.11.0",
|
||||
"nock": "0.15.2",
|
||||
|
|
@ -44479,6 +44562,14 @@
|
|||
"sinon-stub-promise": "^4.0.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"brace-expansion": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
|
||||
"integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==",
|
||||
"requires": {
|
||||
"balanced-match": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"bson": {
|
||||
"version": "4.7.0",
|
||||
"resolved": "https://registry.npmjs.org/bson/-/bson-4.7.0.tgz",
|
||||
|
|
@ -44516,6 +44607,14 @@
|
|||
"yallist": "^2.1.2"
|
||||
}
|
||||
},
|
||||
"minimatch": {
|
||||
"version": "7.4.2",
|
||||
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-7.4.2.tgz",
|
||||
"integrity": "sha512-xy4q7wou3vUoC9k1xGTXc+awNdGaGVHtFUaey8tiX4H1QRc04DZ/rmDFwNm2EBsuYEhAZ6SgMmYf3InGY6OauA==",
|
||||
"requires": {
|
||||
"brace-expansion": "^2.0.1"
|
||||
}
|
||||
},
|
||||
"mongodb": {
|
||||
"version": "4.11.0",
|
||||
"resolved": "https://registry.npmjs.org/mongodb/-/mongodb-4.11.0.tgz",
|
||||
|
|
@ -44958,6 +45057,7 @@
|
|||
"mensch": "^0.3.4",
|
||||
"method-override": "^2.3.3",
|
||||
"mini-css-extract-plugin": "^2.6.0",
|
||||
"minimatch": "^7.4.2",
|
||||
"minimist": "^1.2.7",
|
||||
"mmmagic": "^0.5.3",
|
||||
"mocha": "^10.2.0",
|
||||
|
|
@ -45772,6 +45872,17 @@
|
|||
"jsx-ast-utils": "^3.2.1",
|
||||
"language-tags": "^1.0.5",
|
||||
"minimatch": "^3.0.4"
|
||||
},
|
||||
"dependencies": {
|
||||
"minimatch": {
|
||||
"version": "3.1.2",
|
||||
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
|
||||
"integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"brace-expansion": "^1.1.7"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"eslint-plugin-react": {
|
||||
|
|
@ -45804,6 +45915,15 @@
|
|||
"requires": {
|
||||
"esutils": "^2.0.2"
|
||||
}
|
||||
},
|
||||
"minimatch": {
|
||||
"version": "3.1.2",
|
||||
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
|
||||
"integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"brace-expansion": "^1.1.7"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
|
|
@ -45976,6 +46096,15 @@
|
|||
"webpack-merge": "^4.1.5"
|
||||
},
|
||||
"dependencies": {
|
||||
"minimatch": {
|
||||
"version": "3.1.2",
|
||||
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
|
||||
"integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"brace-expansion": "^1.1.7"
|
||||
}
|
||||
},
|
||||
"webpack-merge": {
|
||||
"version": "4.2.2",
|
||||
"resolved": "https://registry.npmjs.org/webpack-merge/-/webpack-merge-4.2.2.tgz",
|
||||
|
|
@ -46083,6 +46212,24 @@
|
|||
}
|
||||
}
|
||||
},
|
||||
"minimatch": {
|
||||
"version": "7.4.2",
|
||||
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-7.4.2.tgz",
|
||||
"integrity": "sha512-xy4q7wou3vUoC9k1xGTXc+awNdGaGVHtFUaey8tiX4H1QRc04DZ/rmDFwNm2EBsuYEhAZ6SgMmYf3InGY6OauA==",
|
||||
"requires": {
|
||||
"brace-expansion": "^2.0.1"
|
||||
},
|
||||
"dependencies": {
|
||||
"brace-expansion": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
|
||||
"integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==",
|
||||
"requires": {
|
||||
"balanced-match": "^1.0.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"mongodb": {
|
||||
"version": "4.13.0",
|
||||
"resolved": "https://registry.npmjs.org/mongodb/-/mongodb-4.13.0.tgz",
|
||||
|
|
|
|||
|
|
@ -3,41 +3,17 @@ const Path = require('path')
|
|||
const isUtf8 = require('utf-8-validate')
|
||||
const { promisifyAll } = require('../../util/promises')
|
||||
const Settings = require('@overleaf/settings')
|
||||
const Minimatch = require('minimatch').Minimatch
|
||||
|
||||
const fileIgnoreMatcher = new Minimatch(Settings.fileIgnorePattern, {
|
||||
nocase: true, // make the whole path matching case-insensitive
|
||||
// (previously we were only matching the extension case-insensitively but it seems safer to match the whole path)
|
||||
dot: true, // allows matching on paths containing a dot e.g. /.git/foo/bar.txt
|
||||
})
|
||||
|
||||
const FileTypeManager = {
|
||||
TEXT_EXTENSIONS: new Set(Settings.textExtensions.map(ext => `.${ext}`)),
|
||||
|
||||
IGNORE_EXTENSIONS: new Set([
|
||||
'.dvi',
|
||||
'.aux',
|
||||
'.log',
|
||||
'.toc',
|
||||
'.out',
|
||||
'.pdfsync',
|
||||
'.synctex', // synctex.gz is handled by the .gz extension
|
||||
'.synctex(busy)',
|
||||
'.fdb_latexmk',
|
||||
'.fls',
|
||||
// Index and glossary files
|
||||
'.nlo',
|
||||
'.ind',
|
||||
'.glo',
|
||||
'.gls',
|
||||
'.glg',
|
||||
// Bibtex
|
||||
'.bbl',
|
||||
'.blg',
|
||||
// Misc/bad
|
||||
'.doc',
|
||||
'.docx',
|
||||
'.gz',
|
||||
'.swp',
|
||||
]),
|
||||
|
||||
IGNORE_FILENAMES: new Set(['.gitignore']),
|
||||
|
||||
IGNORE_FOLDERS: new Set(['__MACOSX', '.git', '.texpadtmp', '.R']),
|
||||
|
||||
MAX_TEXT_FILE_SIZE: 1 * 1024 * 1024, // 1 MB
|
||||
|
||||
isDirectory(path, callback) {
|
||||
|
|
@ -120,26 +96,10 @@ const FileTypeManager = {
|
|||
return true
|
||||
},
|
||||
|
||||
// FIXME: we can convert this to a synchronous function if we want to
|
||||
shouldIgnore(path, callback) {
|
||||
const basename = Path.basename(path)
|
||||
const extension = Path.extname(basename).toLowerCase()
|
||||
const folders = path.split('/')
|
||||
let ignore = false
|
||||
if (basename.startsWith('.') && basename !== '.latexmkrc') {
|
||||
ignore = true
|
||||
}
|
||||
if (FileTypeManager.IGNORE_EXTENSIONS.has(extension)) {
|
||||
ignore = true
|
||||
} else if (FileTypeManager.IGNORE_FILENAMES.has(basename)) {
|
||||
ignore = true
|
||||
} else {
|
||||
for (const folder of folders) {
|
||||
if (FileTypeManager.IGNORE_FOLDERS.has(folder)) {
|
||||
ignore = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
// use minimatch file matching to check if the path should be ignored
|
||||
const ignore = fileIgnoreMatcher.match(path)
|
||||
callback(null, ignore)
|
||||
},
|
||||
}
|
||||
|
|
|
|||
|
|
@ -675,6 +675,10 @@ module.exports = {
|
|||
parseTextExtensions(process.env.ADDITIONAL_TEXT_EXTENSIONS)
|
||||
),
|
||||
|
||||
fileIgnorePattern:
|
||||
process.env.FILE_IGNORE_PATTERN ||
|
||||
'**/{{__MACOSX,.git,.texpadtmp,.R}{,/**},.!(latexmkrc),*.{dvi,aux,log,toc,out,pdfsync,synctex,synctex(busy),fdb_latexmk,fls,nlo,ind,glo,gls,glg,bbl,blg,doc,docx,gz,swp}}',
|
||||
|
||||
validRootDocExtensions: ['tex', 'Rtex', 'ltx'],
|
||||
|
||||
emailConfirmationDisabled:
|
||||
|
|
|
|||
|
|
@ -188,6 +188,7 @@
|
|||
"mathjax": "^2.7.9",
|
||||
"mathjax-3": "npm:mathjax@^3.2.2",
|
||||
"method-override": "^2.3.3",
|
||||
"minimatch": "^7.4.2",
|
||||
"minimist": "^1.2.7",
|
||||
"mmmagic": "^0.5.3",
|
||||
"moment": "^2.29.4",
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ const { expect } = require('chai')
|
|||
const mockFs = require('mock-fs')
|
||||
const SandboxedModule = require('sandboxed-module')
|
||||
const { ObjectId } = require('mongodb')
|
||||
const Settings = require('@overleaf/settings')
|
||||
|
||||
const MODULE_PATH =
|
||||
'../../../../app/src/Features/Uploads/FileSystemImportManager.js'
|
||||
|
|
@ -27,6 +28,7 @@ describe('FileSystemImportManager', function () {
|
|||
requires: {
|
||||
'@overleaf/settings': {
|
||||
textExtensions: ['tex', 'txt'],
|
||||
fileIgnorePattern: Settings.fileIgnorePattern, // use the real pattern from the default settings
|
||||
},
|
||||
'../Editor/EditorController': this.EditorController,
|
||||
},
|
||||
|
|
|
|||
Loading…
Reference in a new issue