From cdbf8c1831fe7468e52e8376031811d04c5e6555 Mon Sep 17 00:00:00 2001 From: Alexandre Bourdin Date: Wed, 12 Oct 2022 16:30:38 +0200 Subject: [PATCH] Merge pull request #9915 from overleaf/jpa-project-list-api-query-optimizations [web] optimize db queries of project list api endpoint GitOrigin-RevId: e1e747858e95cf60003d68e6331dc41839389455 --- .../web/app/src/Features/Helpers/Mongo.js | 3 ++ .../Features/Project/ProjectListController.js | 53 ++++++++++++------- services/web/app/views/project/list-react.pug | 1 + .../context/project-list-context.tsx | 21 ++++++-- 4 files changed, 54 insertions(+), 24 deletions(-) diff --git a/services/web/app/src/Features/Helpers/Mongo.js b/services/web/app/src/Features/Helpers/Mongo.js index a6c102d989..fd5bd1da38 100644 --- a/services/web/app/src/Features/Helpers/Mongo.js +++ b/services/web/app/src/Features/Helpers/Mongo.js @@ -33,6 +33,9 @@ function normalizeQuery(query) { } function normalizeMultiQuery(query) { + if (query instanceof Set) { + query = Array.from(query) + } if (Array.isArray(query)) { return { _id: { $in: query.map(id => _getObjectIdInstance(id)) } } } else { diff --git a/services/web/app/src/Features/Project/ProjectListController.js b/services/web/app/src/Features/Project/ProjectListController.js index 52f2cfd2c0..fa4f9b243d 100644 --- a/services/web/app/src/Features/Project/ProjectListController.js +++ b/services/web/app/src/Features/Project/ProjectListController.js @@ -1,4 +1,5 @@ const _ = require('lodash') +const Metrics = require('@overleaf/metrics') const Settings = require('@overleaf/settings') const ProjectHelper = require('./ProjectHelper') const ProjectGetter = require('./ProjectGetter') @@ -88,6 +89,10 @@ async function projectListReactPage(req, res, next) { let survey const userId = SessionManager.getLoggedInUserId(req.session) + const projectsBlobPending = _getProjects(userId).catch(err => { + logger.err({ err, userId }, 'projects listing in background failed') + return undefined + }) const user = await User.findById( userId, 'email emails features lastPrimaryEmailCheck signUpDate' @@ -274,6 +279,14 @@ async function projectListReactPage(req, res, next) { delete req.session.saml } + const prefetchedProjectsBlob = await Promise.race([ + projectsBlobPending, + Promise.resolve(undefined), + ]) + Metrics.inc('project-list-prefetch-projects', 1, { + status: prefetchedProjectsBlob ? 'success' : 'too-slow', + }) + res.render('project/list-react', { title: 'your_projects', usersBestSubscription, @@ -286,6 +299,7 @@ async function projectListReactPage(req, res, next) { survey, tags, portalTemplates, + prefetchedProjectsBlob, }) } @@ -317,14 +331,16 @@ async function _getProjects( sort = { by: 'lastUpdated', order: 'desc' }, page = { size: 20 } ) { - const allProjects = - /** @type {AllUsersProjects} **/ await ProjectGetter.promises.findAllUsersProjects( + const [ + /** @type {AllUsersProjects} **/ allProjects, + /** @type {Tag[]} **/ tags, + ] = await Promise.all([ + ProjectGetter.promises.findAllUsersProjects( userId, 'name lastUpdated lastUpdatedBy publicAccesLevel archived trashed owner_ref tokens' - ) - const tags = /** @type {Tag[]} **/ await TagsHandler.promises.getAllTags( - userId - ) + ), + TagsHandler.promises.getAllTags(userId), + ]) const formattedProjects = _formatProjects(allProjects, userId) const filteredProjects = _applyFilters( formattedProjects, @@ -475,20 +491,19 @@ async function _injectProjectUsers(projects) { } } + const projection = { + first_name: 1, + last_name: 1, + email: 1, + } const users = {} - for (const userId of userIds) { - const user = await UserGetter.promises.getUser(userId, { - first_name: 1, - last_name: 1, - email: 1, - }) - if (user) { - users[userId] = { - id: userId, - email: user.email, - firstName: user.first_name, - lastName: user.last_name, - } + for (const user of await UserGetter.promises.getUsers(userIds, projection)) { + const userId = user._id.toString() + users[userId] = { + id: userId, + email: user.email, + firstName: user.first_name, + lastName: user.last_name, } } for (const project of projects) { diff --git a/services/web/app/views/project/list-react.pug b/services/web/app/views/project/list-react.pug index 460e71f626..aa8fa4817c 100644 --- a/services/web/app/views/project/list-react.pug +++ b/services/web/app/views/project/list-react.pug @@ -17,6 +17,7 @@ block append meta meta(name="ol-survey" data-type="json" content=survey) meta(name="ol-tags" data-type="json" content=tags) meta(name="ol-portalTemplates" data-type="json" content=portalTemplates) + meta(name="ol-prefetchedProjectsBlob" data-type="json" content=prefetchedProjectsBlob) block content main.content.content-alt.project-list-react#project-list-root diff --git a/services/web/frontend/js/features/project-list/context/project-list-context.tsx b/services/web/frontend/js/features/project-list/context/project-list-context.tsx index 9ca6264962..5a376344d2 100644 --- a/services/web/frontend/js/features/project-list/context/project-list-context.tsx +++ b/services/web/frontend/js/features/project-list/context/project-list-context.tsx @@ -103,14 +103,21 @@ type ProjectListProviderProps = { } export function ProjectListProvider({ children }: ProjectListProviderProps) { - const [loadedProjects, setLoadedProjects] = useState([]) + const prefetchedProjectsBlob = getMeta('ol-prefetchedProjectsBlob') + const [loadedProjects, setLoadedProjects] = useState( + prefetchedProjectsBlob?.projects ?? [] + ) const [visibleProjects, setVisibleProjects] = useState([]) const [maxVisibleProjects, setMaxVisibleProjects] = useState(MAX_PROJECT_PER_PAGE) const [hiddenProjectsCount, setHiddenProjectsCount] = useState(0) const [loadMoreCount, setLoadMoreCount] = useState(0) - const [loadProgress, setLoadProgress] = useState(20) - const [totalProjectsCount, setTotalProjectsCount] = useState(0) + const [loadProgress, setLoadProgress] = useState( + prefetchedProjectsBlob ? 100 : 20 + ) + const [totalProjectsCount, setTotalProjectsCount] = useState( + prefetchedProjectsBlob?.totalSize ?? 0 + ) const [sort, setSort] = useState({ by: 'lastUpdated', order: 'desc', @@ -131,10 +138,14 @@ export function ProjectListProvider({ children }: ProjectListProviderProps) { isIdle, error, runAsync, - } = useAsync() + } = useAsync({ + status: prefetchedProjectsBlob ? 'resolved' : 'pending', + data: prefetchedProjectsBlob, + }) const isLoading = isIdle ? true : loading useEffect(() => { + if (prefetchedProjectsBlob) return setLoadProgress(40) runAsync(getProjects({ by: 'lastUpdated', order: 'desc' })) .then(data => { @@ -145,7 +156,7 @@ export function ProjectListProvider({ children }: ProjectListProviderProps) { .finally(() => { setLoadProgress(100) }) - }, [runAsync]) + }, [prefetchedProjectsBlob, runAsync]) useEffect(() => { let filteredProjects = [...loadedProjects]