Merge pull request #12195 from overleaf/bg-use-glob-ignore

Change ignored file matching to use standard `minimatch` module

GitOrigin-RevId: ef8f13c039015e7bb9894441fb6170703941653c
This commit is contained in:
Brian Gough 2023-05-15 09:31:23 +01:00 committed by Copybot
parent a94f241dcb
commit e46567b696
5 changed files with 164 additions and 50 deletions

147
package-lock.json generated
View file

@ -34693,6 +34693,7 @@
"express": "^4.18.2",
"lodash": "^4.17.21",
"lru-cache": "^4.1.5",
"minimatch": "^7.4.2",
"mongodb": "^4.11.0",
"node-fetch": "^2.6.7",
"p-limit": "^2.3.0",
@ -34709,6 +34710,14 @@
"sinon-stub-promise": "^4.0.0"
}
},
"services/third-party-datastore/node_modules/brace-expansion": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
"integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==",
"dependencies": {
"balanced-match": "^1.0.0"
}
},
"services/third-party-datastore/node_modules/bson": {
"version": "4.7.0",
"resolved": "https://registry.npmjs.org/bson/-/bson-4.7.0.tgz",
@ -34769,6 +34778,20 @@
"yallist": "^2.1.2"
}
},
"services/third-party-datastore/node_modules/minimatch": {
"version": "7.4.2",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-7.4.2.tgz",
"integrity": "sha512-xy4q7wou3vUoC9k1xGTXc+awNdGaGVHtFUaey8tiX4H1QRc04DZ/rmDFwNm2EBsuYEhAZ6SgMmYf3InGY6OauA==",
"dependencies": {
"brace-expansion": "^2.0.1"
},
"engines": {
"node": ">=10"
},
"funding": {
"url": "https://github.com/sponsors/isaacs"
}
},
"services/third-party-datastore/node_modules/mongodb": {
"version": "4.11.0",
"resolved": "https://registry.npmjs.org/mongodb/-/mongodb-4.11.0.tgz",
@ -35210,6 +35233,7 @@
"mathjax": "^2.7.9",
"mathjax-3": "npm:mathjax@^3.2.2",
"method-override": "^2.3.3",
"minimatch": "^7.4.2",
"minimist": "^1.2.7",
"mmmagic": "^0.5.3",
"moment": "^2.29.4",
@ -36405,6 +36429,18 @@
"eslint": "^3 || ^4 || ^5 || ^6 || ^7 || ^8"
}
},
"services/web/node_modules/eslint-plugin-jsx-a11y/node_modules/minimatch": {
"version": "3.1.2",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
"integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
"dev": true,
"dependencies": {
"brace-expansion": "^1.1.7"
},
"engines": {
"node": "*"
}
},
"services/web/node_modules/eslint-plugin-react": {
"version": "7.28.0",
"resolved": "https://registry.npmjs.org/eslint-plugin-react/-/eslint-plugin-react-7.28.0.tgz",
@ -36457,6 +36493,18 @@
"node": ">=0.10.0"
}
},
"services/web/node_modules/eslint-plugin-react/node_modules/minimatch": {
"version": "3.1.2",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
"integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
"dev": true,
"dependencies": {
"brace-expansion": "^1.1.7"
},
"engines": {
"node": "*"
}
},
"services/web/node_modules/events": {
"version": "3.3.0",
"resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz",
@ -36690,6 +36738,18 @@
"webpack": "^5.0.0"
}
},
"services/web/node_modules/karma-webpack/node_modules/minimatch": {
"version": "3.1.2",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
"integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
"dev": true,
"dependencies": {
"brace-expansion": "^1.1.7"
},
"engines": {
"node": "*"
}
},
"services/web/node_modules/karma-webpack/node_modules/webpack-merge": {
"version": "4.2.2",
"resolved": "https://registry.npmjs.org/webpack-merge/-/webpack-merge-4.2.2.tgz",
@ -36832,6 +36892,28 @@
"url": "https://opencollective.com/webpack"
}
},
"services/web/node_modules/minimatch": {
"version": "7.4.2",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-7.4.2.tgz",
"integrity": "sha512-xy4q7wou3vUoC9k1xGTXc+awNdGaGVHtFUaey8tiX4H1QRc04DZ/rmDFwNm2EBsuYEhAZ6SgMmYf3InGY6OauA==",
"dependencies": {
"brace-expansion": "^2.0.1"
},
"engines": {
"node": ">=10"
},
"funding": {
"url": "https://github.com/sponsors/isaacs"
}
},
"services/web/node_modules/minimatch/node_modules/brace-expansion": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
"integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==",
"dependencies": {
"balanced-match": "^1.0.0"
}
},
"services/web/node_modules/mongodb": {
"version": "4.13.0",
"resolved": "https://registry.npmjs.org/mongodb/-/mongodb-4.13.0.tgz",
@ -44467,6 +44549,7 @@
"express": "^4.18.2",
"lodash": "^4.17.21",
"lru-cache": "^4.1.5",
"minimatch": "*",
"mocha": "^10.2.0",
"mongodb": "^4.11.0",
"nock": "0.15.2",
@ -44479,6 +44562,14 @@
"sinon-stub-promise": "^4.0.0"
},
"dependencies": {
"brace-expansion": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
"integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==",
"requires": {
"balanced-match": "^1.0.0"
}
},
"bson": {
"version": "4.7.0",
"resolved": "https://registry.npmjs.org/bson/-/bson-4.7.0.tgz",
@ -44516,6 +44607,14 @@
"yallist": "^2.1.2"
}
},
"minimatch": {
"version": "7.4.2",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-7.4.2.tgz",
"integrity": "sha512-xy4q7wou3vUoC9k1xGTXc+awNdGaGVHtFUaey8tiX4H1QRc04DZ/rmDFwNm2EBsuYEhAZ6SgMmYf3InGY6OauA==",
"requires": {
"brace-expansion": "^2.0.1"
}
},
"mongodb": {
"version": "4.11.0",
"resolved": "https://registry.npmjs.org/mongodb/-/mongodb-4.11.0.tgz",
@ -44958,6 +45057,7 @@
"mensch": "^0.3.4",
"method-override": "^2.3.3",
"mini-css-extract-plugin": "^2.6.0",
"minimatch": "^7.4.2",
"minimist": "^1.2.7",
"mmmagic": "^0.5.3",
"mocha": "^10.2.0",
@ -45772,6 +45872,17 @@
"jsx-ast-utils": "^3.2.1",
"language-tags": "^1.0.5",
"minimatch": "^3.0.4"
},
"dependencies": {
"minimatch": {
"version": "3.1.2",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
"integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
"dev": true,
"requires": {
"brace-expansion": "^1.1.7"
}
}
}
},
"eslint-plugin-react": {
@ -45804,6 +45915,15 @@
"requires": {
"esutils": "^2.0.2"
}
},
"minimatch": {
"version": "3.1.2",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
"integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
"dev": true,
"requires": {
"brace-expansion": "^1.1.7"
}
}
}
},
@ -45976,6 +46096,15 @@
"webpack-merge": "^4.1.5"
},
"dependencies": {
"minimatch": {
"version": "3.1.2",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
"integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
"dev": true,
"requires": {
"brace-expansion": "^1.1.7"
}
},
"webpack-merge": {
"version": "4.2.2",
"resolved": "https://registry.npmjs.org/webpack-merge/-/webpack-merge-4.2.2.tgz",
@ -46083,6 +46212,24 @@
}
}
},
"minimatch": {
"version": "7.4.2",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-7.4.2.tgz",
"integrity": "sha512-xy4q7wou3vUoC9k1xGTXc+awNdGaGVHtFUaey8tiX4H1QRc04DZ/rmDFwNm2EBsuYEhAZ6SgMmYf3InGY6OauA==",
"requires": {
"brace-expansion": "^2.0.1"
},
"dependencies": {
"brace-expansion": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
"integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==",
"requires": {
"balanced-match": "^1.0.0"
}
}
}
},
"mongodb": {
"version": "4.13.0",
"resolved": "https://registry.npmjs.org/mongodb/-/mongodb-4.13.0.tgz",

View file

@ -3,41 +3,17 @@ const Path = require('path')
const isUtf8 = require('utf-8-validate')
const { promisifyAll } = require('../../util/promises')
const Settings = require('@overleaf/settings')
const Minimatch = require('minimatch').Minimatch
const fileIgnoreMatcher = new Minimatch(Settings.fileIgnorePattern, {
nocase: true, // make the whole path matching case-insensitive
// (previously we were only matching the extension case-insensitively but it seems safer to match the whole path)
dot: true, // allows matching on paths containing a dot e.g. /.git/foo/bar.txt
})
const FileTypeManager = {
TEXT_EXTENSIONS: new Set(Settings.textExtensions.map(ext => `.${ext}`)),
IGNORE_EXTENSIONS: new Set([
'.dvi',
'.aux',
'.log',
'.toc',
'.out',
'.pdfsync',
'.synctex', // synctex.gz is handled by the .gz extension
'.synctex(busy)',
'.fdb_latexmk',
'.fls',
// Index and glossary files
'.nlo',
'.ind',
'.glo',
'.gls',
'.glg',
// Bibtex
'.bbl',
'.blg',
// Misc/bad
'.doc',
'.docx',
'.gz',
'.swp',
]),
IGNORE_FILENAMES: new Set(['.gitignore']),
IGNORE_FOLDERS: new Set(['__MACOSX', '.git', '.texpadtmp', '.R']),
MAX_TEXT_FILE_SIZE: 1 * 1024 * 1024, // 1 MB
isDirectory(path, callback) {
@ -120,26 +96,10 @@ const FileTypeManager = {
return true
},
// FIXME: we can convert this to a synchronous function if we want to
shouldIgnore(path, callback) {
const basename = Path.basename(path)
const extension = Path.extname(basename).toLowerCase()
const folders = path.split('/')
let ignore = false
if (basename.startsWith('.') && basename !== '.latexmkrc') {
ignore = true
}
if (FileTypeManager.IGNORE_EXTENSIONS.has(extension)) {
ignore = true
} else if (FileTypeManager.IGNORE_FILENAMES.has(basename)) {
ignore = true
} else {
for (const folder of folders) {
if (FileTypeManager.IGNORE_FOLDERS.has(folder)) {
ignore = true
break
}
}
}
// use minimatch file matching to check if the path should be ignored
const ignore = fileIgnoreMatcher.match(path)
callback(null, ignore)
},
}

View file

@ -675,6 +675,10 @@ module.exports = {
parseTextExtensions(process.env.ADDITIONAL_TEXT_EXTENSIONS)
),
fileIgnorePattern:
process.env.FILE_IGNORE_PATTERN ||
'**/{{__MACOSX,.git,.texpadtmp,.R}{,/**},.!(latexmkrc),*.{dvi,aux,log,toc,out,pdfsync,synctex,synctex(busy),fdb_latexmk,fls,nlo,ind,glo,gls,glg,bbl,blg,doc,docx,gz,swp}}',
validRootDocExtensions: ['tex', 'Rtex', 'ltx'],
emailConfirmationDisabled:

View file

@ -188,6 +188,7 @@
"mathjax": "^2.7.9",
"mathjax-3": "npm:mathjax@^3.2.2",
"method-override": "^2.3.3",
"minimatch": "^7.4.2",
"minimist": "^1.2.7",
"mmmagic": "^0.5.3",
"moment": "^2.29.4",

View file

@ -3,6 +3,7 @@ const { expect } = require('chai')
const mockFs = require('mock-fs')
const SandboxedModule = require('sandboxed-module')
const { ObjectId } = require('mongodb')
const Settings = require('@overleaf/settings')
const MODULE_PATH =
'../../../../app/src/Features/Uploads/FileSystemImportManager.js'
@ -27,6 +28,7 @@ describe('FileSystemImportManager', function () {
requires: {
'@overleaf/settings': {
textExtensions: ['tex', 'txt'],
fileIgnorePattern: Settings.fileIgnorePattern, // use the real pattern from the default settings
},
'../Editor/EditorController': this.EditorController,
},