overleaf/services/web/app/src/Features/Downloads/ProjectZipStreamManager.js

let ProjectZipStreamManager
const archiver = require('archiver')
const async = require('async')
const logger = require('@overleaf/logger')
const ProjectEntityHandler = require('../Project/ProjectEntityHandler')
const ProjectGetter = require('../Project/ProjectGetter')
const FileStoreHandler = require('../FileStore/FileStoreHandler')

module.exports = ProjectZipStreamManager = {
  createZipStreamForMultipleProjects(projectIds, callback) {
    // We'll build up a zip file that contains multiple zip files
    const archive = archiver('zip')
    archive.on('error', err =>
      logger.err(
        { err, projectIds },
        'something went wrong building archive of project'
      )
    )
    callback(null, archive)

    const jobs = projectIds.map(projectId => cb => {
      ProjectGetter.getProject(projectId, { name: true }, (error, project) => {
        if (error) {
          return cb(error)
        }
        if (!project) {
          logger.log(
            { projectId },
            'cannot append project to zip stream: project not found'
          )
          return cb()
        }
        logger.log(
          { projectId, name: project.name },
          'appending project to zip stream'
        )
        ProjectZipStreamManager.createZipStreamForProject(
          projectId,
          (error, stream) => {
            if (error) {
              return cb(error)
            }
            archive.append(stream, { name: `${project.name}.zip` })
            stream.on('end', () => {
              logger.log({ projectId, name: project.name }, 'zip stream ended')
              cb()
            })
          }
        )
      })
    })

    async.series(jobs, () => {
      logger.log(
        { projectIds },
        'finished creating zip stream of multiple projects'
      )
      archive.finalize()
    })
  },

  createZipStreamForProject(projectId, callback) {
    const archive = archiver('zip')
    // return stream immediately before we start adding things to it
    archive.on('error', err =>
      logger.err(
        { err, projectId },
        'something went wrong building archive of project'
      )
    )
    callback(null, archive)
    this.addAllDocsToArchive(projectId, archive, error => {
      if (error) {
        logger.error(
          { err: error, projectId },
          'error adding docs to zip stream'
        )
      }
      this.addAllFilesToArchive(projectId, archive, error => {
        if (error) {
          logger.error(
            { err: error, projectId },
            'error adding files to zip stream'
          )
        }
        archive.finalize()
      })
    })
  },

  addAllDocsToArchive(projectId, archive, callback) {
    ProjectEntityHandler.getAllDocs(projectId, (error, docs) => {
      if (error) {
        return callback(error)
      }
      const jobs = Object.entries(docs).map(([path, doc]) => cb => {
        if (path[0] === '/') {
          path = path.slice(1)
        }
        logger.log({ projectId }, 'Adding doc')
        archive.append(doc.lines.join('\n'), { name: path })
        setImmediate(cb)
      })
      async.series(jobs, callback)
    })
  },

  addAllFilesToArchive(projectId, archive, callback) {
    ProjectEntityHandler.getAllFiles(projectId, (error, files) => {
      if (error) {
        return callback(error)
      }
      const jobs = Object.entries(files).map(([path, file]) => cb => {
        FileStoreHandler.getFileStream(
          projectId,
          file._id,
          {},
          (error, stream) => {
            if (error) {
              logger.warn(
                { err: error, projectId, file_id: file._id },
                'something went wrong adding file to zip archive'
              )
              return cb(error)
            }
            if (path[0] === '/') {
              path = path.slice(1)
            }
            archive.append(stream, { name: path })
            stream.on('end', () => cb())
          }
        )
      })
      async.parallelLimit(jobs, 5, callback)
    })
  },
}