From a51f61a5558a276585b1503168d7f8cf5b538c4a Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Tue, 21 Apr 2020 14:48:47 +0100 Subject: [PATCH] remove redis migration code --- .../app/coffee/RedisMigrationManager.coffee | 224 ------------ .../coffee/RedisMigrationManagerTests.coffee | 320 ------------------ 2 files changed, 544 deletions(-) delete mode 100644 services/document-updater/app/coffee/RedisMigrationManager.coffee delete mode 100644 services/document-updater/test/acceptance/coffee/RedisMigrationManagerTests.coffee diff --git a/services/document-updater/app/coffee/RedisMigrationManager.coffee b/services/document-updater/app/coffee/RedisMigrationManager.coffee deleted file mode 100644 index d11024fc94..0000000000 --- a/services/document-updater/app/coffee/RedisMigrationManager.coffee +++ /dev/null @@ -1,224 +0,0 @@ -logger = require "logger-sharelatex" -Settings = require "settings-sharelatex" -redis = require("redis-sharelatex") -LockManager = require("./LockManager") -metrics = require "./Metrics" -async = require("async") - -# The aim is to migrate the project history queues -# ProjectHistory:Ops:{project_id} from the existing redis to a new redis. -# -# This has to work in conjunction with changes in project history. -# -# The basic principles are: -# -# - project history is modified to read from an 'old' and 'new' queue. It reads -# from the 'old' queue first, and when that queue is empty it reads from the -# 'new' queue. -# - docupdater will migrate to writing to the 'new' queue when the 'old' queue -# is empty. -# -# Some facts about the update process: -# -# - project history has a lock on the project-id, so each queue is processed in -# isolation -# - docupdaters take a lock on the doc_id but not the project_id, therefore -# multiple docupdaters can be appending to the queue for a project at the same -# time (provided they updates for individual docs are in order this is -# acceptable) -# - as we want to do this without shutting down the site, we have to take into -# account that different versions of the code will be running while deploys -# are in progress. -# -# The migration has to be carried out with the following constraint: -# -# - a docupdater should never write to the "old" queue when there are updates in -# the "new" queue (there is a strict ordering on the versions, new > old) -# -# The deployment process for docupdater will be -# -# - add a project-level lock to the queuing in docupdater -# - use a per-project migration flag to determine when to write to the new redis -# - set the migration flag for projects with an empty queue in the old redis -# - when all docupdaters respect the flag, make a new deploy which starts to set -# the flag -# - when all docupdaters are setting the flag (and writing to the new redis), -# finish the migration by writing all data to the new redis -# -# Final stage -# -# When all the queues are migrated, remove the migration code and return to a -# single client pointing at the new redis. Delete the -# ProjectHistory:MigrationKey:* entries in the new redis. -# -# Rollback -# -# Under the scheme above a project should only ever have data in the old redis -# or the new redis, but never both at the same time. -# -# Two scenarios: -# -# Hard rollback -# -# If we want to roll back to the old redis immediately, we need to get the data -# out of the new queues and back into the old queues, before appending to the -# old queues again. The actions to do this are: -# -# - close the site -# - revert docupdater so it only writes to the original redis (there will now -# be some data in the new redis for some projects which we need to recover) -# - run a script to move the new queues back into the old redis -# - revert project history to only read from the original redis -# -# Graceful rollback -# -# If we are prepared to keep the new redis running, but not add new projects to -# it we can do the following: -# -# - deploy all docupdaters to update from the "switch" phase into the -# "rollback" phase (projects in the new redis will continue to send data -# there, project not yet migrated will continue to go to the old redis) -# - deploy project history with the "old queue" pointing to the new redis and -# the "new queue" to the old redis to clear the new queue before processing -# the new queue (i.e. add a rollback:true property in new_project_history in -# the project-history settings via the environment variable -# MIGRATION_PHASE="rollback"). -# - projects will now clear gradually from the new redis back to the old redis -# - get a list of all the projects in the new redis and flush them, which will -# cause the new queues to be cleared and the old redis to be used for those -# projects. - -getProjectId = (key) -> - key.match(/\{([0-9a-f]{24})\}/)[1] - -class Multi - constructor: (@migrationClient) -> - @command_list = [] - @queueKey = null - rpush: (args...) -> - @queueKey = args[0] - @updates_count = args.length - 1 - @command_list.push { command:'rpush', args: args} - setnx: (args...) -> - @command_list.push { command: 'setnx', args: args} - exec: (callback) -> - # decide which client to use - project_id = getProjectId(@queueKey) - # Put a lock around finding and updating the queue to avoid time-of-check to - # time-of-use problems. When running in the "switch" phase we need a lock to - # guarantee the order of operations. (Example: docupdater A sees an old - # queue at t=t0 and pushes onto it at t=t1, project history clears the queue - # between t0 and t1, and docupdater B sees the empty queue, sets the - # migration flag and pushes onto the new queue at t2. Without a lock it's - # possible to have t2 < t1 if docupdater A is slower than B - then there - # would be entries in the old and new queues, which we want to avoid.) - LockManager.getLock project_id, (error, lockValue) => - return callback(error) if error? - releaseLock = (args...) => - LockManager.releaseLock project_id, lockValue, (lockError) -> - return callback(lockError) if lockError? - callback(args...) - @migrationClient.findQueue @queueKey, (err, rclient) => - return releaseLock(err) if err? - # add metric for updates - dest = (if rclient == @migrationClient.rclient_new then "new" else "old") - metrics.count "migration", @updates_count, 1, {status: "#{@migrationClient.migration_phase}-#{dest}"} - multi = rclient.multi() - for entry in @command_list - multi[entry.command](entry.args...) - multi.exec releaseLock - -class MigrationClient - constructor: (@old_settings, @new_settings) -> - @rclient_old = redis.createClient(@old_settings) - @rclient_new = redis.createClient(@new_settings) - @new_key_schema = new_settings.key_schema - # check that migration phase is valid on startup - logger.warn {migration_phase: @getMigrationPhase()}, "running with RedisMigrationManager" - - getMigrationPhase: () -> - @migration_phase = @new_settings.migration_phase # FIXME: allow setting migration phase while running for testing - throw new Error("invalid migration phase") unless @migration_phase in ['prepare', 'switch', 'rollback'] - return @migration_phase - - getMigrationStatus: (key, migrationKey, callback) -> - async.series [ - (cb) => @rclient_new.exists migrationKey, cb - (cb) => @rclient_new.exists key, cb - (cb) => @rclient_old.exists key, cb - ], (err, result) -> - return callback(err) if err? - migrationKeyExists = result[0] > 0 - newQueueExists = result[1] > 0 - oldQueueExists = result[2] > 0 - callback(null, migrationKeyExists, newQueueExists, oldQueueExists) - - findQueue: (key, callback) -> - project_id = getProjectId(key) - migrationKey = @new_key_schema.projectHistoryMigrationKey({project_id}) - migration_phase = @getMigrationPhase() # allow setting migration phase while running for testing - @getMigrationStatus key, migrationKey, (err, migrationKeyExists, newQueueExists, oldQueueExists) => - return callback(err) if err? - # In all cases, if the migration key exists we must always write to the - # new redis, unless we are rolling back. - if migration_phase is "prepare" - # in this phase we prepare for the switch, when some docupdaters will - # start setting the migration flag. We monitor the migration key and - # write to the new redis if the key is present, but we do not set the - # migration key. At this point no writes will be going into the new - # redis. When all the docupdaters are in the "prepare" phase we can - # begin deploying the "switch" phase. - if migrationKeyExists - logger.debug {project_id}, "using new client because migration key exists" - return callback(null, @rclient_new) - else - logger.debug {project_id}, "using old client because migration key does not exist" - return callback(null, @rclient_old) - else if migration_phase is "switch" - # As we deploy the "switch" phase new docupdaters will set the migration - # flag for projects which have an empty queue in the old redis, and - # write updates into the new redis. Existing docupdaters still in the - # "prepare" phase will pick up the migration flag and write new updates - # into the new redis when appropriate. When this deploy is complete - # writes will be going into the new redis for projects with an empty - # queue in the old redis. We have to remain in the switch phase until - # all projects are flushed from the old redis. - if migrationKeyExists - logger.debug {project_id}, "using new client because migration key exists" - return callback(null, @rclient_new) - else - if oldQueueExists - logger.debug {project_id}, "using old client because old queue exists" - return callback(null, @rclient_old) - else - @rclient_new.setnx migrationKey, "NEW", (err) => - return callback(err) if err? - logger.debug {key: key}, "switching to new redis because old queue is empty" - return callback(null, @rclient_new) - else if migration_phase is "rollback" - # If we need to roll back gracefully we do the opposite of the "switch" - # phase. We use the new redis when the migration key is set and the - # queue exists in the new redis, but if the queue in the new redis is - # empty we delete the migration key and send further updates to the old - # redis. - if migrationKeyExists - if newQueueExists - logger.debug {project_id}, "using new client because migration key exists and new queue is present" - return callback(null, @rclient_new) - else - @rclient_new.del migrationKey, (err) => - return callback(err) if err? - logger.debug {key: key}, "switching to old redis in rollback phase because new queue is empty" - return callback(null, @rclient_old) - else - logger.debug {project_id}, "using old client because migration key does not exist" - return callback(null, @rclient_old) - else - logger.error {key: key, migration_phase: migration_phase}, "unknown migration phase" - callback(new Error('invalid migration phase')) - multi: () -> - new Multi(@) - -module.exports = RedisMigrationManager = - createClient: (args...) -> - new MigrationClient(args...) diff --git a/services/document-updater/test/acceptance/coffee/RedisMigrationManagerTests.coffee b/services/document-updater/test/acceptance/coffee/RedisMigrationManagerTests.coffee deleted file mode 100644 index 2684a4a3d8..0000000000 --- a/services/document-updater/test/acceptance/coffee/RedisMigrationManagerTests.coffee +++ /dev/null @@ -1,320 +0,0 @@ -sinon = require "sinon" -chai = require("chai") -chai.should() -expect = chai.expect -async = require "async" -Settings = require('settings-sharelatex') -rclient_old = require("redis-sharelatex").createClient(Settings.redis.project_history) -rclient_new = require("redis-sharelatex").createClient(Settings.redis.new_project_history) -rclient_du = require("redis-sharelatex").createClient(Settings.redis.documentupdater) -Keys = Settings.redis.documentupdater.key_schema -HistoryKeys = Settings.redis.history.key_schema -ProjectHistoryKeys = Settings.redis.project_history.key_schema -NewProjectHistoryKeys = Settings.redis.new_project_history.key_schema - -MockTrackChangesApi = require "./helpers/MockTrackChangesApi" -MockWebApi = require "./helpers/MockWebApi" -DocUpdaterClient = require "./helpers/DocUpdaterClient" -DocUpdaterApp = require "./helpers/DocUpdaterApp" - -describe "RedisMigrationManager", -> - before (done) -> - @lines = ["one", "two", "three"] - @version = 42 - @update = - doc: @doc_id - op: [{ - i: "one and a half\n" - p: 4 - }] - v: @version - DocUpdaterApp.ensureRunning(done) - - describe "when the migration phase is 'prepare' (default)", -> - - describe "when there is no migration flag", -> - before (done) -> - [@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()] - - MockWebApi.insertDoc @project_id, @doc_id, {lines: @lines, version: @version} - DocUpdaterClient.preloadDoc @project_id, @doc_id, (error) => - throw error if error? - sinon.spy MockWebApi, "getDocument" - DocUpdaterClient.sendUpdate @project_id, @doc_id, @update, (error) -> - throw error if error? - setTimeout done, 200 - return null - - after -> - MockWebApi.getDocument.restore() - - it "should push the applied updates to old redis", (done) -> - rclient_old.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => - JSON.parse(updates[0]).op.should.deep.equal @update.op - done() - return null - - it "should not push the applied updates to the new redis", (done) -> - rclient_new.exists ProjectHistoryKeys.projectHistoryOps({@project_id}), (error, result) => - result.should.equal 0 - done() - return null - - it "should not set the migration flag for the project", (done) -> - rclient_new.exists NewProjectHistoryKeys.projectHistoryMigrationKey({@project_id}), (error, result) => - result.should.equal 0 - done() - return null - - describe "when the migration flag is set for the project", -> - before (done) -> - [@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()] - - rclient_new.set NewProjectHistoryKeys.projectHistoryMigrationKey({@project_id}), '1', (error) => - throw error if error? - MockWebApi.insertDoc @project_id, @doc_id, {lines: @lines, version: @version} - DocUpdaterClient.preloadDoc @project_id, @doc_id, (error) => - throw error if error? - sinon.spy MockWebApi, "getDocument" - DocUpdaterClient.sendUpdate @project_id, @doc_id, @update, (error) -> - throw error if error? - setTimeout done, 200 - return null - - after (done) -> - MockWebApi.getDocument.restore() - rclient_new.del NewProjectHistoryKeys.projectHistoryMigrationKey({@project_id}), done - return null - - it "should push the applied updates to the new redis", (done) -> - rclient_new.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => - JSON.parse(updates[0]).op.should.deep.equal @update.op - done() - return null - - it "should not push the applied updates to the old redis", (done) -> - rclient_old.exists ProjectHistoryKeys.projectHistoryOps({@project_id}), (error, result) => - result.should.equal 0 - done() - return null - - it "should keep the migration flag for the project", (done) -> - rclient_new.exists NewProjectHistoryKeys.projectHistoryMigrationKey({@project_id}), (error, result) => - result.should.equal 1 - done() - return null - - describe "when the migration phase is 'switch'", -> - before -> - Settings.redis.new_project_history.migration_phase = 'switch' - - describe "when the old queue is empty", -> - before (done) -> - [@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()] - - MockWebApi.insertDoc @project_id, @doc_id, {lines: @lines, version: @version} - DocUpdaterClient.preloadDoc @project_id, @doc_id, (error) => - throw error if error? - sinon.spy MockWebApi, "getDocument" - DocUpdaterClient.sendUpdate @project_id, @doc_id, @update, (error) -> - throw error if error? - setTimeout done, 200 - return null - - after -> - MockWebApi.getDocument.restore() - - it "should push the applied updates to the new redis", (done) -> - rclient_new.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => - JSON.parse(updates[0]).op.should.deep.equal @update.op - done() - return null - - it "should not push the applied updates to the old redis", (done) -> - rclient_old.exists ProjectHistoryKeys.projectHistoryOps({@project_id}), (error, result) => - result.should.equal 0 - done() - return null - - it "should set the migration flag for the project", (done) -> - rclient_new.get NewProjectHistoryKeys.projectHistoryMigrationKey({@project_id}), (error, result) => - result.should.equal "NEW" - done() - return null - - describe "when the old queue is not empty", -> - before (done) -> - [@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()] - - MockWebApi.insertDoc @project_id, @doc_id, {lines: @lines, version: @version} - DocUpdaterClient.preloadDoc @project_id, @doc_id, (error) => - throw error if error? - sinon.spy MockWebApi, "getDocument" - rclient_old.rpush ProjectHistoryKeys.projectHistoryOps({@project_id}), JSON.stringify({op: "dummy-op"}), (error) => - throw error if error? - DocUpdaterClient.sendUpdate @project_id, @doc_id, @update, (error) -> - throw error if error? - setTimeout done, 200 - return null - - after -> - MockWebApi.getDocument.restore() - - it "should push the applied updates to the old redis", (done) -> - rclient_old.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => - JSON.parse(updates[0]).op.should.deep.equal "dummy-op" - JSON.parse(updates[1]).op.should.deep.equal @update.op - done() - return null - - it "should not push the applied updates to the new redis", (done) -> - rclient_new.exists ProjectHistoryKeys.projectHistoryOps({@project_id}), (error, result) => - result.should.equal 0 - done() - return null - - it "should not set the migration flag for the project", (done) -> - rclient_new.exists NewProjectHistoryKeys.projectHistoryMigrationKey({@project_id}), (error, result) => - result.should.equal 0 - done() - return null - - describe "when the migration flag is set for the project", -> - before (done) -> - [@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()] - - rclient_new.set NewProjectHistoryKeys.projectHistoryMigrationKey({@project_id}), '1', (error) => - throw error if error? - MockWebApi.insertDoc @project_id, @doc_id, {lines: @lines, version: @version} - DocUpdaterClient.preloadDoc @project_id, @doc_id, (error) => - throw error if error? - sinon.spy MockWebApi, "getDocument" - DocUpdaterClient.sendUpdate @project_id, @doc_id, @update, (error) -> - throw error if error? - setTimeout done, 200 - return null - - after (done) -> - MockWebApi.getDocument.restore() - rclient_new.del NewProjectHistoryKeys.projectHistoryMigrationKey({@project_id}), done - return null - - it "should push the applied updates to the new redis", (done) -> - rclient_new.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => - JSON.parse(updates[0]).op.should.deep.equal @update.op - done() - return null - - it "should not push the applied updates to the old redis", (done) -> - rclient_old.exists ProjectHistoryKeys.projectHistoryOps({@project_id}), (error, result) => - result.should.equal 0 - done() - return null - - it "should keep the migration flag for the project", (done) -> - rclient_new.exists NewProjectHistoryKeys.projectHistoryMigrationKey({@project_id}), (error, result) => - result.should.equal 1 - done() - return null - - describe "when the migration phase is 'rollback'", -> - before -> - Settings.redis.new_project_history.migration_phase = 'rollback' - - describe "when the old queue is empty", -> - before (done) -> - [@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()] - - MockWebApi.insertDoc @project_id, @doc_id, {lines: @lines, version: @version} - DocUpdaterClient.preloadDoc @project_id, @doc_id, (error) => - throw error if error? - sinon.spy MockWebApi, "getDocument" - DocUpdaterClient.sendUpdate @project_id, @doc_id, @update, (error) -> - throw error if error? - setTimeout done, 200 - return null - - after -> - MockWebApi.getDocument.restore() - - it "should push the applied updates to the old redis", (done) -> - rclient_old.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => - JSON.parse(updates[0]).op.should.deep.equal @update.op - done() - return null - - it "should not push the applied updates to the new redis", (done) -> - rclient_new.exists ProjectHistoryKeys.projectHistoryOps({@project_id}), (error, result) => - result.should.equal 0 - done() - return null - - describe "when the new queue is not empty", -> - before (done) -> - [@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()] - - MockWebApi.insertDoc @project_id, @doc_id, {lines: @lines, version: @version} - DocUpdaterClient.preloadDoc @project_id, @doc_id, (error) => - throw error if error? - sinon.spy MockWebApi, "getDocument" - rclient_new.rpush ProjectHistoryKeys.projectHistoryOps({@project_id}), JSON.stringify({op: "dummy-op"}), (error) => - throw error if error? - DocUpdaterClient.sendUpdate @project_id, @doc_id, @update, (error) -> - throw error if error? - setTimeout done, 200 - return null - - after -> - MockWebApi.getDocument.restore() - - it "should push the applied updates to the old redis", (done) -> - rclient_old.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => - JSON.parse(updates[0]).op.should.deep.equal @update.op - done() - return null - - it "should not push the applied updates to the new redis", (done) -> - rclient_new.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => - JSON.parse(updates[0]).op.should.deep.equal "dummy-op" - updates.length.should.equal 1 - done() - return null - - describe "when the migration flag is set for the project", -> - before (done) -> - [@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()] - - rclient_new.set NewProjectHistoryKeys.projectHistoryMigrationKey({@project_id}), '1', (error) => - throw error if error? - MockWebApi.insertDoc @project_id, @doc_id, {lines: @lines, version: @version} - DocUpdaterClient.preloadDoc @project_id, @doc_id, (error) => - throw error if error? - sinon.spy MockWebApi, "getDocument" - DocUpdaterClient.sendUpdate @project_id, @doc_id, @update, (error) -> - throw error if error? - setTimeout done, 200 - return null - - after (done) -> - MockWebApi.getDocument.restore() - rclient_new.del NewProjectHistoryKeys.projectHistoryMigrationKey({@project_id}), done - return null - - it "should push the applied updates to the old redis", (done) -> - rclient_old.lrange ProjectHistoryKeys.projectHistoryOps({@project_id}), 0, -1, (error, updates) => - JSON.parse(updates[0]).op.should.deep.equal @update.op - done() - return null - - it "should not push the applied updates to the new redis", (done) -> - rclient_new.exists ProjectHistoryKeys.projectHistoryOps({@project_id}), (error, result) => - result.should.equal 0 - done() - return null - - it "should delete the migration flag for the project", (done) -> - rclient_new.exists NewProjectHistoryKeys.projectHistoryMigrationKey({@project_id}), (error, result) => - result.should.equal 0 - done() - return null -