diff --git a/services/real-time/app/coffee/DocumentUpdaterController.coffee b/services/real-time/app/coffee/DocumentUpdaterController.coffee index 2611d484ad..c96282f49e 100644 --- a/services/real-time/app/coffee/DocumentUpdaterController.coffee +++ b/services/real-time/app/coffee/DocumentUpdaterController.coffee @@ -71,10 +71,16 @@ module.exports = DocumentUpdaterController = seen[client.id] = true if client.id == update.meta.source logger.log doc_id: doc_id, version: update.v, source: update.meta?.source, "distributing update to sender" - client.emit "otUpdateApplied", v: update.v, doc: update.doc + try + client.emit "otUpdateApplied", v: update.v, doc: update.doc + catch err + logger.warn client_id: client.id, doc_id: doc_id, err: err, "error sending update to sender" else if !update.dup # Duplicate ops should just be sent back to sending client for acknowledgement logger.log doc_id: doc_id, version: update.v, source: update.meta?.source, client_id: client.id, "distributing update to collaborator" - client.emit "otUpdateApplied", update + try + client.emit "otUpdateApplied", update + catch err + logger.warn client_id: client.id, doc_id: doc_id, err: err, "error sending update to collaborator" if Object.keys(seen).length < clientList.length metrics.inc "socket-io.duplicate-clients", 0.1 logger.log doc_id: doc_id, socketIoClients: (client.id for client in clientList), "discarded duplicate clients" @@ -82,7 +88,10 @@ module.exports = DocumentUpdaterController = _processErrorFromDocumentUpdater: (io, doc_id, error, message) -> for client in io.sockets.clients(doc_id) logger.warn err: error, doc_id: doc_id, client_id: client.id, "error from document updater, disconnecting client" - client.emit "otUpdateError", error, message - client.disconnect() + try + client.emit "otUpdateError", error, message + client.disconnect() + catch err + logger.warn client_id: client.id, doc_id: doc_id, err: err, cause: error, "error sending error to client" diff --git a/services/real-time/app/coffee/DrainManager.coffee b/services/real-time/app/coffee/DrainManager.coffee index 2590a96726..1a8fb73be1 100644 --- a/services/real-time/app/coffee/DrainManager.coffee +++ b/services/real-time/app/coffee/DrainManager.coffee @@ -30,7 +30,10 @@ module.exports = DrainManager = if !@RECONNECTED_CLIENTS[client.id] @RECONNECTED_CLIENTS[client.id] = true logger.log {client_id: client.id}, "Asking client to reconnect gracefully" - client.emit "reconnectGracefully" + try + client.emit "reconnectGracefully" + catch err + logger.warn client_id: client.id, err: err, "error asking client to reconnect gracefully" drainedCount++ haveDrainedNClients = (drainedCount == N) if haveDrainedNClients diff --git a/services/real-time/app/coffee/Router.coffee b/services/real-time/app/coffee/Router.coffee index fcdb0c93dc..d5a7472ab0 100644 --- a/services/real-time/app/coffee/Router.coffee +++ b/services/real-time/app/coffee/Router.coffee @@ -45,29 +45,45 @@ module.exports = Router = client?.on "error", (err) -> logger.err { clientErr: err }, "socket.io client error" if client.connected - client.emit("reconnectGracefully") - client.disconnect() + try + client.emit("reconnectGracefully") + client.disconnect() + catch error + logger.warn error: error, cause: err, client_id: client.id, "error telling client to reconnect after error" if settings.shutDownInProgress - client.emit("connectionRejected", {message: "retry"}) - client.disconnect() + try + client.emit("connectionRejected", {message: "retry"}) + client.disconnect() + catch error + logger.warn error: error, client_id: client.id, message: "retry", "error rejecting client connection" return if client? and error?.message?.match(/could not look up session by key/) logger.warn err: error, client: client?, session: session?, "invalid session" # tell the client to reauthenticate if it has an invalid session key - client.emit("connectionRejected", {message: "invalid session"}) - client.disconnect() + try + client.emit("connectionRejected", {message: "invalid session"}) + client.disconnect() + catch error + logger.warn error: error, client_id: client.id, message: "invalid session", "error rejecting client connection" return if error? logger.err err: error, client: client?, session: session?, "error when client connected" - client?.emit("connectionRejected", {message: "error"}) - client?.disconnect() + try + client?.emit("connectionRejected", {message: "error"}) + client?.disconnect() + catch error + logger.warn error: error, client_id: client?.id, message: "error", "error rejecting client connection" return # send positive confirmation that the client has a valid connection - client.emit("connectionAccepted") + try + client.emit("connectionAccepted") + catch error + logger.warn error: error, client_id: client.id, "error accepting client connection" + return metrics.inc('socket-io.connection') metrics.gauge('socket-io.clients', io.sockets.clients()?.length) diff --git a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee index 12b7ef812b..df81a04395 100644 --- a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee +++ b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee @@ -67,7 +67,10 @@ module.exports = WebsocketLoadBalancer = logger.error {err: error, channel}, "error parsing JSON" return if message.room_id == "all" - io.sockets.emit(message.message, message.payload...) + try + io.sockets.emit(message.message, message.payload...) + catch error + logger.warn message: message, error: error, "error sending message to clients" else if message.message is 'clientTracking.refresh' && message.room_id? clientList = io.sockets.clients(message.room_id) logger.log {channel:channel, message: message.message, room_id: message.room_id, message_id: message._id, socketIoClients: (client.id for client in clientList)}, "refreshing client list" @@ -99,7 +102,10 @@ module.exports = WebsocketLoadBalancer = if !seen[client.id] seen[client.id] = true if !(is_restricted_user && message.message not in RESTRICTED_USER_MESSAGE_TYPE_PASS_LIST) - client.emit(message.message, message.payload...) + try + client.emit(message.message, message.payload...) + catch error + console.log(message: message, client_id: client.id, error: error, "error sending message to client") cb() , (err) -> if err?