const MockWebApi = require('./helpers/MockWebApi') const DocUpdaterClient = require('./helpers/DocUpdaterClient') const DocUpdaterApp = require('./helpers/DocUpdaterApp') const { promisify } = require('util') const { exec } = require('child_process') const { expect } = require('chai') const Settings = require('@overleaf/settings') const fs = require('fs') const Path = require('path') const rclient = require('@overleaf/redis-wrapper').createClient( Settings.redis.documentupdater ) describe('CheckRedisMongoSyncState', function () { beforeEach(function (done) { DocUpdaterApp.ensureRunning(done) }) beforeEach(async function () { await rclient.flushall() }) async function runScript(options) { let result try { result = await promisify(exec)( Object.entries(options) .map(([key, value]) => `${key}=${value}`) .concat(['node', 'scripts/check_redis_mongo_sync_state.js']) .join(' ') ) } catch (error) { // includes details like exit code, stdErr and stdOut return error } result.code = 0 return result } describe('without projects', function () { it('should work when in sync', async function () { const result = await runScript({}) expect(result.code).to.equal(0) expect(result.stdout).to.include('Processed 0 projects') expect(result.stdout).to.include( 'Found 0 projects with 0 out of sync docs' ) }) }) describe('with a project', function () { let projectId, docId beforeEach(function (done) { projectId = DocUpdaterClient.randomId() docId = DocUpdaterClient.randomId() MockWebApi.insertDoc(projectId, docId, { lines: ['mongo', 'lines'], version: 1, }) DocUpdaterClient.getDoc(projectId, docId, done) }) it('should work when in sync', async function () { const result = await runScript({}) expect(result.code).to.equal(0) expect(result.stdout).to.include('Processed 1 projects') expect(result.stdout).to.include( 'Found 0 projects with 0 out of sync docs' ) }) describe('with out of sync lines', function () { beforeEach(function () { MockWebApi.insertDoc(projectId, docId, { lines: ['updated', 'mongo', 'lines'], version: 1, }) }) it('should detect the out of sync state', async function () { const result = await runScript({}) expect(result.code).to.equal(1) expect(result.stdout).to.include('Processed 1 projects') expect(result.stdout).to.include( 'Found 1 projects with 1 out of sync docs' ) }) }) describe('with out of sync ranges', function () { beforeEach(function () { MockWebApi.insertDoc(projectId, docId, { lines: ['mongo', 'lines'], version: 1, ranges: { changes: ['FAKE CHANGE'] }, }) }) it('should detect the out of sync state', async function () { const result = await runScript({}) expect(result.code).to.equal(1) expect(result.stdout).to.include('Processed 1 projects') expect(result.stdout).to.include( 'Found 1 projects with 1 out of sync docs' ) }) }) describe('with out of sync version', function () { beforeEach(function () { MockWebApi.insertDoc(projectId, docId, { lines: ['mongo', 'lines'], version: 2, }) }) it('should detect the out of sync state', async function () { const result = await runScript({}) expect(result.code).to.equal(1) expect(result.stdout).to.include('Processed 1 projects') expect(result.stdout).to.include( 'Found 1 projects with 1 out of sync docs' ) }) it('should auto-fix the out of sync state', async function () { const result = await runScript({ AUTO_FIX_VERSION_MISMATCH: 'true', }) expect(result.code).to.equal(0) expect(result.stdout).to.include('Processed 1 projects') expect(result.stdout).to.include( 'Found 0 projects with 0 out of sync docs' ) }) }) describe('with a project', function () { let projectId2, docId2 beforeEach(function (done) { projectId2 = DocUpdaterClient.randomId() docId2 = DocUpdaterClient.randomId() MockWebApi.insertDoc(projectId2, docId2, { lines: ['mongo', 'lines'], version: 1, }) DocUpdaterClient.getDoc(projectId2, docId2, done) }) it('should work when in sync', async function () { const result = await runScript({}) expect(result.code).to.equal(0) expect(result.stdout).to.include('Processed 2 projects') expect(result.stdout).to.include( 'Found 0 projects with 0 out of sync docs' ) }) describe('with one out of sync', function () { beforeEach(function () { MockWebApi.insertDoc(projectId, docId, { lines: ['updated', 'mongo', 'lines'], version: 1, }) }) it('should detect one project out of sync', async function () { const result = await runScript({}) expect(result.code).to.equal(1) expect(result.stdout).to.include('Processed 2 projects') expect(result.stdout).to.include( 'Found 1 projects with 1 out of sync docs' ) }) it('should write differences to disk', async function () { const FOLDER = '/tmp/folder' await fs.promises.rm(FOLDER, { recursive: true, force: true }) const result = await runScript({ WRITE_CONTENT: 'true', FOLDER, }) expect(result.code).to.equal(1) expect(result.stdout).to.include('Processed 2 projects') expect(result.stdout).to.include( 'Found 1 projects with 1 out of sync docs' ) const dir = Path.join(FOLDER, projectId, docId) expect(await fs.promises.readdir(FOLDER)).to.deep.equal([projectId]) expect(await fs.promises.readdir(dir)).to.deep.equal([ 'mongo-snapshot.txt', 'redis-snapshot.txt', ]) expect( await fs.promises.readFile( Path.join(dir, 'mongo-snapshot.txt'), 'utf-8' ) ).to.equal('updated\nmongo\nlines') expect( await fs.promises.readFile( Path.join(dir, 'redis-snapshot.txt'), 'utf-8' ) ).to.equal('mongo\nlines') }) }) describe('with both out of sync', function () { beforeEach(function () { MockWebApi.insertDoc(projectId, docId, { lines: ['updated', 'mongo', 'lines'], version: 1, }) MockWebApi.insertDoc(projectId2, docId2, { lines: ['updated2', 'mongo', 'lines'], version: 1, }) }) it('should detect both projects out of sync', async function () { const result = await runScript({}) expect(result.code).to.equal(1) expect(result.stdout).to.include('Processed 2 projects') expect(result.stdout).to.include( 'Found 2 projects with 2 out of sync docs' ) }) }) }) }) describe('with more projects than the LIMIT', function () { for (let i = 0; i < 20; i++) { beforeEach(function (done) { const projectId = DocUpdaterClient.randomId() const docId = DocUpdaterClient.randomId() MockWebApi.insertDoc(projectId, docId, { lines: ['mongo', 'lines'], version: 1, }) DocUpdaterClient.getDoc(projectId, docId, done) }) } it('should flag limit', async function () { const result = await runScript({ LIMIT: '4' }) expect(result.code).to.equal(2) // A redis SCAN may return more than COUNT (aka LIMIT) entries. Match loosely. expect(result.stdout).to.match(/Processed \d+ projects/) expect(result.stderr).to.include( 'Found too many un-flushed projects (LIMIT=4). Please fix the reported projects first, then try again.' ) }) it('should continue with auto-flush', async function () { const result = await runScript({ LIMIT: '4', FLUSH_IN_SYNC_PROJECTS: 'true', }) expect(result.code).to.equal(0) expect(result.stdout).to.include('Processed 20 projects') }) }) })