diff --git a/services/document-updater/app/coffee/RedisBackend.coffee b/services/document-updater/app/coffee/RedisBackend.coffee index 7d02ba72af..f72e530471 100644 --- a/services/document-updater/app/coffee/RedisBackend.coffee +++ b/services/document-updater/app/coffee/RedisBackend.coffee @@ -18,7 +18,7 @@ class Client } ) - monitorAndReconnect: () -> + monitorTcpAndReconnect: () -> for client in @clients if client.driver == "ioredis" @_monitorCluster(client.rclient) @@ -28,18 +28,18 @@ class Client # Nodes can come and go as the cluster moves/heals, so each heartbeat # we ask again for the currently known nodes. for node in rclient.nodes("all") - do (node) => - timer = setTimeout () => - logger.error {err: new Error("Node timed out, reconnecting"), key: node.options.key} - node.stream.destroy() - timer = null - , @HEARTBEAT_TIMEOUT - node.ping (err) -> - if !err? - clearTimeout timer - timer = null + @_checkNode(node) , @HEARTBEAT_INTERVAL - + + _checkNode: (node) -> + timer = setTimeout () -> + logger.error {err: new Error("Node timed out, reconnecting"), key: node.options.key} + # Discussion of application layer monitoring recommends this way of reconnecting at https://github.com/luin/ioredis/issues/275 + node.stream.destroy() + , @HEARTBEAT_TIMEOUT + node.ping (err) -> + if !err? + clearTimeout timer class MultiClient constructor: (@clients) -> diff --git a/services/document-updater/app/coffee/RedisManager.coffee b/services/document-updater/app/coffee/RedisManager.coffee index 7fe03f88d8..a23725653b 100644 --- a/services/document-updater/app/coffee/RedisManager.coffee +++ b/services/document-updater/app/coffee/RedisManager.coffee @@ -10,7 +10,7 @@ Errors = require "./Errors" # Make times easy to read minutes = 60 # seconds for Redis expire -rclient.monitorAndReconnect() +rclient.monitorTcpAndReconnect() module.exports = RedisManager = putDocInMemory : (project_id, doc_id, docLines, version, _callback)-> diff --git a/services/document-updater/config/settings.defaults.coffee b/services/document-updater/config/settings.defaults.coffee index df2c9758c6..6a2902036e 100755 --- a/services/document-updater/config/settings.defaults.coffee +++ b/services/document-updater/config/settings.defaults.coffee @@ -21,33 +21,31 @@ module.exports = host:"localhost" password:"" documentupdater: - port:"6379" - host:"localhost" - password:"" - key_schema: - blockingKey: ({doc_id}) -> "Blocking:#{doc_id}" - docLines: ({doc_id}) -> "doclines:#{doc_id}" - docOps: ({doc_id}) -> "DocOps:#{doc_id}" - docVersion: ({doc_id}) -> "DocVersion:#{doc_id}" - projectKey: ({doc_id}) -> "ProjectId:#{doc_id}" - pendingUpdates: ({doc_id}) -> "PendingUpdates:#{doc_id}" - docsInProject: ({project_id}) -> "DocsIn:#{project_id}" + # port:"6379" + # host:"localhost" + # password:"" + # key_schema: + # blockingKey: ({doc_id}) -> "Blocking:#{doc_id}" + # docLines: ({doc_id}) -> "doclines:#{doc_id}" + # docOps: ({doc_id}) -> "DocOps:#{doc_id}" + # docVersion: ({doc_id}) -> "DocVersion:#{doc_id}" + # projectKey: ({doc_id}) -> "ProjectId:#{doc_id}" + # docsInProject: ({project_id}) -> "DocsIn:#{project_id}" # To use Redis cluster, configure the backend as follows: - # [{ - # primary: true - # cluster: [{ - # port: "7000" - # host: "localhost" - # }] - # key_schema: - # blockingKey: ({doc_id}) -> "Blocking:{#{doc_id}}" - # docLines: ({doc_id}) -> "doclines:{#{doc_id}}" - # docOps: ({doc_id}) -> "DocOps:{#{doc_id}}" - # docVersion: ({doc_id}) -> "DocVersion:{#{doc_id}}" - # projectKey: ({doc_id}) -> "ProjectId:{#{doc_id}}" - # pendingUpdates: ({doc_id}) -> "PendingUpdates:{#{doc_id}}" - # docsInProject: ({project_id}) -> "DocsIn:{#{project_id}}" - # }] + [{ + primary: true + cluster: [{ + port: "7000" + host: "localhost" + }] + key_schema: + blockingKey: ({doc_id}) -> "Blocking:{#{doc_id}}" + docLines: ({doc_id}) -> "doclines:{#{doc_id}}" + docOps: ({doc_id}) -> "DocOps:{#{doc_id}}" + docVersion: ({doc_id}) -> "DocVersion:{#{doc_id}}" + projectKey: ({doc_id}) -> "ProjectId:{#{doc_id}}" + docsInProject: ({project_id}) -> "DocsIn:{#{project_id}}" + }] max_doc_length: 2 * 1024 * 1024 # 2mb diff --git a/services/document-updater/test/unit/coffee/RedisBackend/RedisBackendTests.coffee b/services/document-updater/test/unit/coffee/RedisBackend/RedisBackendTests.coffee index 263bc7deab..10ad599301 100644 --- a/services/document-updater/test/unit/coffee/RedisBackend/RedisBackendTests.coffee +++ b/services/document-updater/test/unit/coffee/RedisBackend/RedisBackendTests.coffee @@ -307,10 +307,10 @@ describe "RedisBackend", -> }, "error in redis backend") .should.equal true - describe "monitorAndReconnect", -> + describe "monitorTcpAndReconnect", -> beforeEach -> @client._monitorCluster = sinon.stub() - @client.monitorAndReconnect() + @client.monitorTcpAndReconnect() it "should monitor the cluster client", -> @client._monitorCluster diff --git a/services/document-updater/test/unit/coffee/RedisManager/RedisManagerTests.coffee b/services/document-updater/test/unit/coffee/RedisManager/RedisManagerTests.coffee index d88dafb9bb..9022163da6 100644 --- a/services/document-updater/test/unit/coffee/RedisManager/RedisManagerTests.coffee +++ b/services/document-updater/test/unit/coffee/RedisManager/RedisManagerTests.coffee @@ -10,7 +10,7 @@ describe "RedisManager", -> @rclient = auth: () -> exec: sinon.stub() - monitorAndReconnect: () -> + monitorTcpAndReconnect: () -> @rclient.multi = () => @rclient @RedisManager = SandboxedModule.require modulePath, requires: "./RedisBackend":