Merge pull request #9915 from overleaf/jpa-project-list-api-query-optimizations

[web] optimize db queries of project list api endpoint

GitOrigin-RevId: e1e747858e95cf60003d68e6331dc41839389455

services/web/app/src/Features/Helpers/Mongo.js

@@ -33,6 +33,9 @@ function normalizeQuery(query) {
 }
 
 function normalizeMultiQuery(query) {
+  if (query instanceof Set) {
+    query = Array.from(query)
+  }
   if (Array.isArray(query)) {
     return { _id: { $in: query.map(id => _getObjectIdInstance(id)) } }
   } else {

services/web/app/src/Features/Project/ProjectListController.js

@@ -1,4 +1,5 @@
 const _ = require('lodash')
+const Metrics = require('@overleaf/metrics')
 const Settings = require('@overleaf/settings')
 const ProjectHelper = require('./ProjectHelper')
 const ProjectGetter = require('./ProjectGetter')
@@ -88,6 +89,10 @@ async function projectListReactPage(req, res, next) {
   let survey
 
   const userId = SessionManager.getLoggedInUserId(req.session)
+  const projectsBlobPending = _getProjects(userId).catch(err => {
+    logger.err({ err, userId }, 'projects listing in background failed')
+    return undefined
+  })
   const user = await User.findById(
     userId,
     'email emails features lastPrimaryEmailCheck signUpDate'
@@ -274,6 +279,14 @@ async function projectListReactPage(req, res, next) {
     delete req.session.saml
   }
 
+  const prefetchedProjectsBlob = await Promise.race([
+    projectsBlobPending,
+    Promise.resolve(undefined),
+  ])
+  Metrics.inc('project-list-prefetch-projects', 1, {
+    status: prefetchedProjectsBlob ? 'success' : 'too-slow',
+  })
+
   res.render('project/list-react', {
     title: 'your_projects',
     usersBestSubscription,
@@ -286,6 +299,7 @@ async function projectListReactPage(req, res, next) {
     survey,
     tags,
     portalTemplates,
+    prefetchedProjectsBlob,
   })
 }
 
@@ -317,14 +331,16 @@ async function _getProjects(
   sort = { by: 'lastUpdated', order: 'desc' },
   page = { size: 20 }
 ) {
-  const allProjects =
-    /** @type {AllUsersProjects} **/ await ProjectGetter.promises.findAllUsersProjects(
+  const [
+    /** @type {AllUsersProjects} **/ allProjects,
+    /** @type {Tag[]} **/ tags,
+  ] = await Promise.all([
+    ProjectGetter.promises.findAllUsersProjects(
       userId,
       'name lastUpdated lastUpdatedBy publicAccesLevel archived trashed owner_ref tokens'
-    )
-  const tags = /** @type {Tag[]} **/ await TagsHandler.promises.getAllTags(
-    userId
-  )
+    ),
+    TagsHandler.promises.getAllTags(userId),
+  ])
   const formattedProjects = _formatProjects(allProjects, userId)
   const filteredProjects = _applyFilters(
     formattedProjects,
@@ -475,20 +491,19 @@ async function _injectProjectUsers(projects) {
     }
   }
 
+  const projection = {
+    first_name: 1,
+    last_name: 1,
+    email: 1,
+  }
   const users = {}
-  for (const userId of userIds) {
-    const user = await UserGetter.promises.getUser(userId, {
-      first_name: 1,
-      last_name: 1,
-      email: 1,
-    })
-    if (user) {
-      users[userId] = {
-        id: userId,
-        email: user.email,
-        firstName: user.first_name,
-        lastName: user.last_name,
-      }
+  for (const user of await UserGetter.promises.getUsers(userIds, projection)) {
+    const userId = user._id.toString()
+    users[userId] = {
+      id: userId,
+      email: user.email,
+      firstName: user.first_name,
+      lastName: user.last_name,
     }
   }
   for (const project of projects) {

services/web/app/views/project/list-react.pug

@@ -17,6 +17,7 @@ block append meta
 	meta(name="ol-survey" data-type="json" content=survey)
 	meta(name="ol-tags" data-type="json" content=tags)
 	meta(name="ol-portalTemplates" data-type="json" content=portalTemplates)
+	meta(name="ol-prefetchedProjectsBlob" data-type="json" content=prefetchedProjectsBlob)
 
 block content
 	main.content.content-alt.project-list-react#project-list-root

services/web/frontend/js/features/project-list/context/project-list-context.tsx

@@ -103,14 +103,21 @@ type ProjectListProviderProps = {
 }
 
 export function ProjectListProvider({ children }: ProjectListProviderProps) {
-  const [loadedProjects, setLoadedProjects] = useState<Project[]>([])
+  const prefetchedProjectsBlob = getMeta('ol-prefetchedProjectsBlob')
+  const [loadedProjects, setLoadedProjects] = useState<Project[]>(
+    prefetchedProjectsBlob?.projects ?? []
+  )
   const [visibleProjects, setVisibleProjects] = useState<Project[]>([])
   const [maxVisibleProjects, setMaxVisibleProjects] =
     useState(MAX_PROJECT_PER_PAGE)
   const [hiddenProjectsCount, setHiddenProjectsCount] = useState(0)
   const [loadMoreCount, setLoadMoreCount] = useState<number>(0)
-  const [loadProgress, setLoadProgress] = useState(20)
-  const [totalProjectsCount, setTotalProjectsCount] = useState<number>(0)
+  const [loadProgress, setLoadProgress] = useState(
+    prefetchedProjectsBlob ? 100 : 20
+  )
+  const [totalProjectsCount, setTotalProjectsCount] = useState<number>(
+    prefetchedProjectsBlob?.totalSize ?? 0
+  )
   const [sort, setSort] = useState<Sort>({
     by: 'lastUpdated',
     order: 'desc',
@@ -131,10 +138,14 @@ export function ProjectListProvider({ children }: ProjectListProviderProps) {
     isIdle,
     error,
     runAsync,
-  } = useAsync<GetProjectsResponseBody>()
+  } = useAsync<GetProjectsResponseBody>({
+    status: prefetchedProjectsBlob ? 'resolved' : 'pending',
+    data: prefetchedProjectsBlob,
+  })
   const isLoading = isIdle ? true : loading
 
   useEffect(() => {
+    if (prefetchedProjectsBlob) return
     setLoadProgress(40)
     runAsync(getProjects({ by: 'lastUpdated', order: 'desc' }))
       .then(data => {
@@ -145,7 +156,7 @@ export function ProjectListProvider({ children }: ProjectListProviderProps) {
       .finally(() => {
         setLoadProgress(100)
       })
-  }, [runAsync])
+  }, [prefetchedProjectsBlob, runAsync])
 
   useEffect(() => {
     let filteredProjects = [...loadedProjects]