Merge pull request #12195 from overleaf/bg-use-glob-ignore

Change ignored file matching to use standard `minimatch` module

GitOrigin-RevId: ef8f13c039015e7bb9894441fb6170703941653c

package-lock.json

@@ -34693,6 +34693,7 @@
         "express": "^4.18.2",
         "lodash": "^4.17.21",
         "lru-cache": "^4.1.5",
+        "minimatch": "^7.4.2",
         "mongodb": "^4.11.0",
         "node-fetch": "^2.6.7",
         "p-limit": "^2.3.0",
@@ -34709,6 +34710,14 @@
         "sinon-stub-promise": "^4.0.0"
       }
     },
+    "services/third-party-datastore/node_modules/brace-expansion": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
+      "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==",
+      "dependencies": {
+        "balanced-match": "^1.0.0"
+      }
+    },
     "services/third-party-datastore/node_modules/bson": {
       "version": "4.7.0",
       "resolved": "https://registry.npmjs.org/bson/-/bson-4.7.0.tgz",
@@ -34769,6 +34778,20 @@
         "yallist": "^2.1.2"
       }
     },
+    "services/third-party-datastore/node_modules/minimatch": {
+      "version": "7.4.2",
+      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-7.4.2.tgz",
+      "integrity": "sha512-xy4q7wou3vUoC9k1xGTXc+awNdGaGVHtFUaey8tiX4H1QRc04DZ/rmDFwNm2EBsuYEhAZ6SgMmYf3InGY6OauA==",
+      "dependencies": {
+        "brace-expansion": "^2.0.1"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/isaacs"
+      }
+    },
     "services/third-party-datastore/node_modules/mongodb": {
       "version": "4.11.0",
       "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-4.11.0.tgz",
@@ -35210,6 +35233,7 @@
         "mathjax": "^2.7.9",
         "mathjax-3": "npm:mathjax@^3.2.2",
         "method-override": "^2.3.3",
+        "minimatch": "^7.4.2",
         "minimist": "^1.2.7",
         "mmmagic": "^0.5.3",
         "moment": "^2.29.4",
@@ -36405,6 +36429,18 @@
         "eslint": "^3 || ^4 || ^5 || ^6 || ^7 || ^8"
       }
     },
+    "services/web/node_modules/eslint-plugin-jsx-a11y/node_modules/minimatch": {
+      "version": "3.1.2",
+      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
+      "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
+      "dev": true,
+      "dependencies": {
+        "brace-expansion": "^1.1.7"
+      },
+      "engines": {
+        "node": "*"
+      }
+    },
     "services/web/node_modules/eslint-plugin-react": {
       "version": "7.28.0",
       "resolved": "https://registry.npmjs.org/eslint-plugin-react/-/eslint-plugin-react-7.28.0.tgz",
@@ -36457,6 +36493,18 @@
         "node": ">=0.10.0"
       }
     },
+    "services/web/node_modules/eslint-plugin-react/node_modules/minimatch": {
+      "version": "3.1.2",
+      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
+      "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
+      "dev": true,
+      "dependencies": {
+        "brace-expansion": "^1.1.7"
+      },
+      "engines": {
+        "node": "*"
+      }
+    },
     "services/web/node_modules/events": {
       "version": "3.3.0",
       "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz",
@@ -36690,6 +36738,18 @@
         "webpack": "^5.0.0"
       }
     },
+    "services/web/node_modules/karma-webpack/node_modules/minimatch": {
+      "version": "3.1.2",
+      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
+      "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
+      "dev": true,
+      "dependencies": {
+        "brace-expansion": "^1.1.7"
+      },
+      "engines": {
+        "node": "*"
+      }
+    },
     "services/web/node_modules/karma-webpack/node_modules/webpack-merge": {
       "version": "4.2.2",
       "resolved": "https://registry.npmjs.org/webpack-merge/-/webpack-merge-4.2.2.tgz",
@@ -36832,6 +36892,28 @@
         "url": "https://opencollective.com/webpack"
       }
     },
+    "services/web/node_modules/minimatch": {
+      "version": "7.4.2",
+      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-7.4.2.tgz",
+      "integrity": "sha512-xy4q7wou3vUoC9k1xGTXc+awNdGaGVHtFUaey8tiX4H1QRc04DZ/rmDFwNm2EBsuYEhAZ6SgMmYf3InGY6OauA==",
+      "dependencies": {
+        "brace-expansion": "^2.0.1"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/isaacs"
+      }
+    },
+    "services/web/node_modules/minimatch/node_modules/brace-expansion": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
+      "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==",
+      "dependencies": {
+        "balanced-match": "^1.0.0"
+      }
+    },
     "services/web/node_modules/mongodb": {
       "version": "4.13.0",
       "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-4.13.0.tgz",
@@ -44467,6 +44549,7 @@
         "express": "^4.18.2",
         "lodash": "^4.17.21",
         "lru-cache": "^4.1.5",
+        "minimatch": "*",
         "mocha": "^10.2.0",
         "mongodb": "^4.11.0",
         "nock": "0.15.2",
@@ -44479,6 +44562,14 @@
         "sinon-stub-promise": "^4.0.0"
       },
       "dependencies": {
+        "brace-expansion": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
+          "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==",
+          "requires": {
+            "balanced-match": "^1.0.0"
+          }
+        },
         "bson": {
           "version": "4.7.0",
           "resolved": "https://registry.npmjs.org/bson/-/bson-4.7.0.tgz",
@@ -44516,6 +44607,14 @@
             "yallist": "^2.1.2"
           }
         },
+        "minimatch": {
+          "version": "7.4.2",
+          "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-7.4.2.tgz",
+          "integrity": "sha512-xy4q7wou3vUoC9k1xGTXc+awNdGaGVHtFUaey8tiX4H1QRc04DZ/rmDFwNm2EBsuYEhAZ6SgMmYf3InGY6OauA==",
+          "requires": {
+            "brace-expansion": "^2.0.1"
+          }
+        },
         "mongodb": {
           "version": "4.11.0",
           "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-4.11.0.tgz",
@@ -44958,6 +45057,7 @@
         "mensch": "^0.3.4",
         "method-override": "^2.3.3",
         "mini-css-extract-plugin": "^2.6.0",
+        "minimatch": "^7.4.2",
         "minimist": "^1.2.7",
         "mmmagic": "^0.5.3",
         "mocha": "^10.2.0",
@@ -45772,6 +45872,17 @@
             "jsx-ast-utils": "^3.2.1",
             "language-tags": "^1.0.5",
             "minimatch": "^3.0.4"
+          },
+          "dependencies": {
+            "minimatch": {
+              "version": "3.1.2",
+              "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
+              "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
+              "dev": true,
+              "requires": {
+                "brace-expansion": "^1.1.7"
+              }
+            }
           }
         },
         "eslint-plugin-react": {
@@ -45804,6 +45915,15 @@
               "requires": {
                 "esutils": "^2.0.2"
               }
+            },
+            "minimatch": {
+              "version": "3.1.2",
+              "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
+              "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
+              "dev": true,
+              "requires": {
+                "brace-expansion": "^1.1.7"
+              }
             }
           }
         },
@@ -45976,6 +46096,15 @@
             "webpack-merge": "^4.1.5"
           },
           "dependencies": {
+            "minimatch": {
+              "version": "3.1.2",
+              "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
+              "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
+              "dev": true,
+              "requires": {
+                "brace-expansion": "^1.1.7"
+              }
+            },
             "webpack-merge": {
               "version": "4.2.2",
               "resolved": "https://registry.npmjs.org/webpack-merge/-/webpack-merge-4.2.2.tgz",
@@ -46083,6 +46212,24 @@
             }
           }
         },
+        "minimatch": {
+          "version": "7.4.2",
+          "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-7.4.2.tgz",
+          "integrity": "sha512-xy4q7wou3vUoC9k1xGTXc+awNdGaGVHtFUaey8tiX4H1QRc04DZ/rmDFwNm2EBsuYEhAZ6SgMmYf3InGY6OauA==",
+          "requires": {
+            "brace-expansion": "^2.0.1"
+          },
+          "dependencies": {
+            "brace-expansion": {
+              "version": "2.0.1",
+              "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
+              "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==",
+              "requires": {
+                "balanced-match": "^1.0.0"
+              }
+            }
+          }
+        },
         "mongodb": {
           "version": "4.13.0",
           "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-4.13.0.tgz",

services/web/app/src/Features/Uploads/FileTypeManager.js

@@ -3,41 +3,17 @@ const Path = require('path')
 const isUtf8 = require('utf-8-validate')
 const { promisifyAll } = require('../../util/promises')
 const Settings = require('@overleaf/settings')
+const Minimatch = require('minimatch').Minimatch
+
+const fileIgnoreMatcher = new Minimatch(Settings.fileIgnorePattern, {
+  nocase: true, // make the whole path matching case-insensitive
+  // (previously we were only matching the extension case-insensitively but it seems safer to match the whole path)
+  dot: true, // allows matching on paths containing a dot e.g. /.git/foo/bar.txt
+})
 
 const FileTypeManager = {
   TEXT_EXTENSIONS: new Set(Settings.textExtensions.map(ext => `.${ext}`)),
 
-  IGNORE_EXTENSIONS: new Set([
-    '.dvi',
-    '.aux',
-    '.log',
-    '.toc',
-    '.out',
-    '.pdfsync',
-    '.synctex', // synctex.gz is handled by the .gz extension
-    '.synctex(busy)',
-    '.fdb_latexmk',
-    '.fls',
-    // Index and glossary files
-    '.nlo',
-    '.ind',
-    '.glo',
-    '.gls',
-    '.glg',
-    // Bibtex
-    '.bbl',
-    '.blg',
-    // Misc/bad
-    '.doc',
-    '.docx',
-    '.gz',
-    '.swp',
-  ]),
-
-  IGNORE_FILENAMES: new Set(['.gitignore']),
-
-  IGNORE_FOLDERS: new Set(['__MACOSX', '.git', '.texpadtmp', '.R']),
-
   MAX_TEXT_FILE_SIZE: 1 * 1024 * 1024, // 1 MB
 
   isDirectory(path, callback) {
@@ -120,26 +96,10 @@ const FileTypeManager = {
     return true
   },
 
+  // FIXME: we can convert this to a synchronous function if we want to
   shouldIgnore(path, callback) {
-    const basename = Path.basename(path)
+    // use minimatch file matching to check if the path should be ignored
-    const extension = Path.extname(basename).toLowerCase()
+    const ignore = fileIgnoreMatcher.match(path)
-    const folders = path.split('/')
-    let ignore = false
-    if (basename.startsWith('.') && basename !== '.latexmkrc') {
-      ignore = true
-    }
-    if (FileTypeManager.IGNORE_EXTENSIONS.has(extension)) {
-      ignore = true
-    } else if (FileTypeManager.IGNORE_FILENAMES.has(basename)) {
-      ignore = true
-    } else {
-      for (const folder of folders) {
-        if (FileTypeManager.IGNORE_FOLDERS.has(folder)) {
-          ignore = true
-          break
-        }
-      }
-    }
     callback(null, ignore)
   },
 }

services/web/config/settings.defaults.js

@@ -675,6 +675,10 @@ module.exports = {
     parseTextExtensions(process.env.ADDITIONAL_TEXT_EXTENSIONS)
   ),
 
+  fileIgnorePattern:
+    process.env.FILE_IGNORE_PATTERN ||
+    '**/{{__MACOSX,.git,.texpadtmp,.R}{,/**},.!(latexmkrc),*.{dvi,aux,log,toc,out,pdfsync,synctex,synctex(busy),fdb_latexmk,fls,nlo,ind,glo,gls,glg,bbl,blg,doc,docx,gz,swp}}',
+
   validRootDocExtensions: ['tex', 'Rtex', 'ltx'],
 
   emailConfirmationDisabled:

services/web/package.json

@@ -188,6 +188,7 @@
     "mathjax": "^2.7.9",
     "mathjax-3": "npm:mathjax@^3.2.2",
     "method-override": "^2.3.3",
+    "minimatch": "^7.4.2",
     "minimist": "^1.2.7",
     "mmmagic": "^0.5.3",
     "moment": "^2.29.4",

services/web/test/unit/src/Uploads/FileSystemImportManagerTests.js

@@ -3,6 +3,7 @@ const { expect } = require('chai')
 const mockFs = require('mock-fs')
 const SandboxedModule = require('sandboxed-module')
 const { ObjectId } = require('mongodb')
+const Settings = require('@overleaf/settings')
 
 const MODULE_PATH =
   '../../../../app/src/Features/Uploads/FileSystemImportManager.js'
@@ -27,6 +28,7 @@ describe('FileSystemImportManager', function () {
       requires: {
         '@overleaf/settings': {
           textExtensions: ['tex', 'txt'],
+          fileIgnorePattern: Settings.fileIgnorePattern, // use the real pattern from the default settings
         },
         '../Editor/EditorController': this.EditorController,
       },