diff --git a/services/real-time/app/coffee/DocumentUpdaterController.coffee b/services/real-time/app/coffee/DocumentUpdaterController.coffee index b809bd9086..dcdd8d142c 100644 --- a/services/real-time/app/coffee/DocumentUpdaterController.coffee +++ b/services/real-time/app/coffee/DocumentUpdaterController.coffee @@ -23,7 +23,7 @@ module.exports = DocumentUpdaterController = return if message.op? if message._id? - EventLogger.checkEventOrder(message._id, message) + EventLogger.checkEventOrder("applied-ops", message._id, message) DocumentUpdaterController._applyUpdateFromDocumentUpdater(io, message.doc_id, message.op) else if message.error? DocumentUpdaterController._processErrorFromDocumentUpdater(io, message.doc_id, message.error, message) diff --git a/services/real-time/app/coffee/EventLogger.coffee b/services/real-time/app/coffee/EventLogger.coffee index 29e79740b9..9a52ccb842 100644 --- a/services/real-time/app/coffee/EventLogger.coffee +++ b/services/real-time/app/coffee/EventLogger.coffee @@ -1,4 +1,5 @@ logger = require 'logger-sharelatex' +metrics = require 'metrics-sharelatex' # keep track of message counters to detect duplicate and out of order events # messsage ids have the format "UNIQUEHOSTKEY-COUNTER" @@ -11,7 +12,7 @@ module.exports = EventLogger = MAX_STALE_TIME_IN_MS: 3600 * 1000 - checkEventOrder: (message_id, message) -> + checkEventOrder: (channel, message_id, message) -> return if typeof(message_id) isnt 'string' [key, count] = message_id.split("-", 2) count = parseInt(count, 10) @@ -20,12 +21,15 @@ module.exports = EventLogger = # store the last count in a hash for each host previous = EventLogger._storeEventCount(key, count) if !previous? || count == (previous + 1) + metrics.inc "event.#{channel}.valid" return # order is ok if (count == previous) - logger.error {key:key, previous: previous, count:count, message:message}, "duplicate event" + metrics.inc "event.#{channel}.duplicate" + # logger.error {key:key, previous: previous, count:count, message:message}, "duplicate event" return "duplicate" else - logger.error {key:key, previous: previous, count:count, message:message}, "events out of order" + metrics.inc "event.#{channel}.out-of-order" + # logger.error {key:key, previous: previous, count:count, message:message}, "events out of order" return # out of order _storeEventCount: (key, count) -> diff --git a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee index 13137805fd..eeedb25916 100644 --- a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee +++ b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee @@ -38,7 +38,7 @@ module.exports = WebsocketLoadBalancer = io.sockets.emit(message.message, message.payload...) else if message.room_id? if message._id? - EventLogger.checkEventOrder(message._id, message) + EventLogger.checkEventOrder("editor-events", message._id, message) io.sockets.in(message.room_id).emit(message.message, message.payload...) else if message.health_check? logger.debug {message}, "got health check message in editor events channel"