Add try/catch around all client emissions

services/real-time/app/coffee/DocumentUpdaterController.coffee

@@ -71,10 +71,16 @@ module.exports = DocumentUpdaterController =
 			seen[client.id] = true
 			if client.id == update.meta.source
 				logger.log doc_id: doc_id, version: update.v, source: update.meta?.source, "distributing update to sender"
-				client.emit "otUpdateApplied", v: update.v, doc: update.doc
+				try
+					client.emit "otUpdateApplied", v: update.v, doc: update.doc
+				catch err
+					logger.warn client_id: client.id, doc_id: doc_id, err: err, "error sending update to sender"
 			else if !update.dup # Duplicate ops should just be sent back to sending client for acknowledgement
 				logger.log doc_id: doc_id, version: update.v, source: update.meta?.source, client_id: client.id, "distributing update to collaborator"
-				client.emit "otUpdateApplied", update
+				try
+					client.emit "otUpdateApplied", update
+				catch err
+					logger.warn client_id: client.id, doc_id: doc_id, err: err, "error sending update to collaborator"
 		if Object.keys(seen).length < clientList.length
 			metrics.inc "socket-io.duplicate-clients", 0.1
 			logger.log doc_id: doc_id, socketIoClients: (client.id for client in clientList), "discarded duplicate clients"
@@ -82,7 +88,10 @@ module.exports = DocumentUpdaterController =
 	_processErrorFromDocumentUpdater: (io, doc_id, error, message) ->
 		for client in io.sockets.clients(doc_id)
 			logger.warn err: error, doc_id: doc_id, client_id: client.id, "error from document updater, disconnecting client"
-			client.emit "otUpdateError", error, message
+			try
-			client.disconnect()
+				client.emit "otUpdateError", error, message
+				client.disconnect()
+			catch err
+				logger.warn client_id: client.id, doc_id: doc_id, err: err, cause: error, "error sending error to client"
 
 

services/real-time/app/coffee/DrainManager.coffee

@@ -30,7 +30,10 @@ module.exports = DrainManager =
 			if !@RECONNECTED_CLIENTS[client.id]
 				@RECONNECTED_CLIENTS[client.id] = true
 				logger.log {client_id: client.id}, "Asking client to reconnect gracefully"
-				client.emit "reconnectGracefully"
+				try
+					client.emit "reconnectGracefully"
+				catch err
+					logger.warn client_id: client.id, err: err, "error asking client to reconnect gracefully"
 				drainedCount++
 			haveDrainedNClients = (drainedCount == N)
 			if haveDrainedNClients

services/real-time/app/coffee/Router.coffee

@@ -45,29 +45,45 @@ module.exports = Router =
 			client?.on "error", (err) ->
 				logger.err { clientErr: err }, "socket.io client error"
 				if client.connected
-					client.emit("reconnectGracefully")
+					try
-					client.disconnect()
+						client.emit("reconnectGracefully")
+						client.disconnect()
+					catch error
+						logger.warn error: error, cause: err, client_id: client.id, "error telling client to reconnect after error"
 
 			if settings.shutDownInProgress
-				client.emit("connectionRejected", {message: "retry"})
+				try
-				client.disconnect()
+					client.emit("connectionRejected", {message: "retry"})
+					client.disconnect()
+				catch error
+					logger.warn error: error, client_id: client.id, message: "retry", "error rejecting client connection"
 				return
 
 			if client? and error?.message?.match(/could not look up session by key/)
 				logger.warn err: error, client: client?, session: session?, "invalid session"
 				# tell the client to reauthenticate if it has an invalid session key
-				client.emit("connectionRejected", {message: "invalid session"})
+				try
-				client.disconnect()
+					client.emit("connectionRejected", {message: "invalid session"})
+					client.disconnect()
+				catch error
+					logger.warn error: error, client_id: client.id, message: "invalid session", "error rejecting client connection"
 				return
 
 			if error?
 				logger.err err: error, client: client?, session: session?, "error when client connected"
-				client?.emit("connectionRejected", {message: "error"})
+				try
-				client?.disconnect()
+					client?.emit("connectionRejected", {message: "error"})
+					client?.disconnect()
+				catch error
+					logger.warn error: error, client_id: client?.id, message: "error", "error rejecting client connection"
 				return
 
 			# send positive confirmation that the client has a valid connection
-			client.emit("connectionAccepted")
+			try
+				client.emit("connectionAccepted")
+			catch error
+				logger.warn error: error, client_id: client.id, "error accepting client connection"
+				return
 
 			metrics.inc('socket-io.connection')
 			metrics.gauge('socket-io.clients', io.sockets.clients()?.length)

services/real-time/app/coffee/WebsocketLoadBalancer.coffee

@@ -67,7 +67,10 @@ module.exports = WebsocketLoadBalancer =
 				logger.error {err: error, channel}, "error parsing JSON"
 				return
 			if message.room_id == "all"
-				io.sockets.emit(message.message, message.payload...)
+				try
+					io.sockets.emit(message.message, message.payload...)
+				catch error
+					logger.warn message: message, error: error, "error sending message to clients"
 			else if message.message is 'clientTracking.refresh' && message.room_id?
 				clientList = io.sockets.clients(message.room_id)
 				logger.log {channel:channel, message: message.message, room_id: message.room_id, message_id: message._id, socketIoClients: (client.id for client in clientList)}, "refreshing client list"
@@ -99,7 +102,10 @@ module.exports = WebsocketLoadBalancer =
 							if !seen[client.id]
 								seen[client.id] = true
 								if !(is_restricted_user && message.message not in RESTRICTED_USER_MESSAGE_TYPE_PASS_LIST)
-									client.emit(message.message, message.payload...)
+									try
+										client.emit(message.message, message.payload...)
+									catch error
+										console.log(message: message, client_id: client.id, error: error, "error sending message to client")
 							cb()
 					, (err) ->
 						if err?