overleaf/services/web/scripts/refresh_features.js

const { db, waitForDb } = require('../app/src/infrastructure/mongodb')
const minimist = require('minimist')
const _ = require('lodash')
const async = require('async')
const FeaturesUpdater = require('../app/src/Features/Subscription/FeaturesUpdater')
const FeaturesHelper = require('../app/src/Features/Subscription/FeaturesHelper')
const UserFeaturesUpdater = require('../app/src/Features/Subscription/UserFeaturesUpdater')
const AnalyticsManager = require('../app/src/Features/Analytics/AnalyticsManager')
const DropboxHandler = require('../modules/dropbox/app/src/DropboxHandler')
const { OError } = require('../app/src/Features/Errors/Errors')
const logger = require('@overleaf/logger')

const ScriptLogger = {
  checkedUsersCount: 0,
  mismatchUsersCount: 0,
  allDaysSinceLastLoggedIn: [],
  allMismatchReasons: {},

  recordMismatch: (user, mismatchReasons) => {
    const mismatchReasonsString = JSON.stringify(mismatchReasons)
    if (ScriptLogger.allMismatchReasons[mismatchReasonsString]) {
      ScriptLogger.allMismatchReasons[mismatchReasonsString].push(user._id)
    } else {
      ScriptLogger.allMismatchReasons[mismatchReasonsString] = [user._id]
    }

    ScriptLogger.mismatchUsersCount += 1

    if (user.lastLoggedIn) {
      const daysSinceLastLoggedIn =
        (new Date() - user.lastLoggedIn) / 1000 / 3600 / 24
      ScriptLogger.allDaysSinceLastLoggedIn.push(daysSinceLastLoggedIn)
    }
  },

  printProgress: () => {
    console.warn(
      `Users checked: ${ScriptLogger.checkedUsersCount}. Mismatches: ${ScriptLogger.mismatchUsersCount}`
    )
  },

  printSummary: () => {
    console.log('All Mismatch Reasons:', ScriptLogger.allMismatchReasons)
    console.log('Mismatch Users Count', ScriptLogger.mismatchUsersCount)
    console.log(
      'Average Last Logged In (Days):',
      _.sum(ScriptLogger.allDaysSinceLastLoggedIn) /
        ScriptLogger.allDaysSinceLastLoggedIn.length
    )
    console.log(
      'Recent Logged In (Last 7 Days):',
      _.filter(ScriptLogger.allDaysSinceLastLoggedIn, a => a < 7).length
    )
    console.log(
      'Recent Logged In (Last 30 Days):',
      _.filter(ScriptLogger.allDaysSinceLastLoggedIn, a => a < 30).length
    )
  },
}

const checkAndUpdateUser = (user, callback) =>
  FeaturesUpdater.computeFeatures(user._id, (error, freshFeatures) => {
    if (error) {
      return callback(error)
    }

    const mismatchReasons = FeaturesHelper.compareFeatures(
      user.features,
      freshFeatures
    )
    if (Object.keys(mismatchReasons).length === 0) {
      // features are matching; nothing else to do
      return callback()
    }

    ScriptLogger.recordMismatch(user, mismatchReasons)

    if (!COMMIT) {
      // not saving features; nothing else to do
      return callback()
    }

    const matchedFeatureSet = FeaturesHelper.getMatchedFeatureSet(freshFeatures)
    AnalyticsManager.setUserPropertyForUser(
      user._id,
      'feature-set',
      matchedFeatureSet
    )

    UserFeaturesUpdater.overrideFeatures(
      user._id,
      freshFeatures,
      (error, featuresChanged) => {
        if (error) {
          return callback(error)
        }
        if (
          mismatchReasons.dropbox !== undefined &&
          freshFeatures.dropbox === false
        ) {
          DropboxHandler.unlinkAccount(
            user._id,
            { sendEmail: false },
            error => {
              if (error) {
                return callback(
                  OError.tag(error, 'error unlinking dropbox', {
                    userId: user._id,
                  })
                )
              }
              logger.log({ userId: user._id }, 'Unlinked dropbox')
              callback(null, featuresChanged)
            }
          )
        } else {
          callback(null, featuresChanged)
        }
      }
    )
  })

const checkAndUpdateUsers = (users, callback) =>
  async.eachLimit(users, ASYNC_LIMIT, checkAndUpdateUser, callback)

const loopForUsers = (skip, callback) => {
  db.users
    .find({})
    .project({ features: 1, lastLoggedIn: 1 })
    .sort({ _id: 1 })
    .skip(skip)
    .limit(FETCH_LIMIT)
    .toArray((error, users) => {
      if (error) {
        return callback(error)
      }

      if (users.length === 0) {
        console.warn('DONE')
        return callback()
      }

      checkAndUpdateUsers(users, error => {
        if (error) {
          return callback(error)
        }
        ScriptLogger.checkedUsersCount += users.length
        retryCounter = 0
        ScriptLogger.printProgress()
        ScriptLogger.printSummary()
        loopForUsers(MONGO_SKIP + ScriptLogger.checkedUsersCount, callback)
      })
    })
}

let retryCounter = 0
const run = () =>
  loopForUsers(MONGO_SKIP + ScriptLogger.checkedUsersCount, error => {
    if (error) {
      if (retryCounter < 3) {
        console.error(error)
        retryCounter += 1
        console.warn(`RETRYING IN 60 SECONDS. (${retryCounter}/3)`)
        return setTimeout(run, 6000)
      }
      throw error
    }
    process.exit()
  })

let FETCH_LIMIT, ASYNC_LIMIT, COMMIT, MONGO_SKIP
const setup = () => {
  const argv = minimist(process.argv.slice(2))
  FETCH_LIMIT = argv.fetch ? argv.fetch : 100
  ASYNC_LIMIT = argv.async ? argv.async : 10
  MONGO_SKIP = argv.skip ? argv.skip : 0
  COMMIT = argv.commit !== undefined
  const FORCE = argv.force !== undefined
  if (!FORCE) {
    console.log(
      'NOTE: features can be automatically refreshed on login (using `featuresEpoch`)\n' +
        'Consider incrementing settings.featuresEpoch instead of running this script.\n' +
        'If you really need to run this script, use refresh_features.js --force.'
    )
    process.exit(1)
  }
  if (!COMMIT) {
    console.warn('Doing dry run without --commit')
  }
  if (MONGO_SKIP) {
    console.warn(`Skipping first ${MONGO_SKIP} records`)
  }
}

waitForDb().then(() => {
  setup()
  run()
})