From baa0418fb54fb8f158267f8e8b5f248232dc0a8f Mon Sep 17 00:00:00 2001 From: Max Wu Date: Mon, 26 Feb 2018 16:43:29 +0800 Subject: [PATCH 1/9] Remove and replace all note id compression in LZString with base64url Signed-off-by: Max Wu --- lib/models/note.js | 27 +++++++++++++++++++++++++++ lib/realtime.js | 3 +-- lib/response.js | 11 +++++------ package.json | 1 + 4 files changed, 34 insertions(+), 8 deletions(-) diff --git a/lib/models/note.js b/lib/models/note.js index 484f1a8c6..e199a3dbd 100644 --- a/lib/models/note.js +++ b/lib/models/note.js @@ -3,6 +3,7 @@ var fs = require('fs') var path = require('path') var LZString = require('lz-string') +var base64url = require('base64url') var md = require('markdown-it')() var metaMarked = require('meta-marked') var cheerio = require('cheerio') @@ -114,6 +115,22 @@ module.exports = function (sequelize, DataTypes) { return false } }, + encodeNoteId: function (id) { + // remove dashes in UUID and encode in url-safe base64 + return base64url.encode(id.replace(/-/g, '')) + }, + decodeNoteId: function (encodedId) { + // decode from url-safe base64 + let id = base64url.decode(encodedId) + // add dashes between the UUID string parts + let idParts = [] + idParts.push(id.substr(0, 8)) + idParts.push(id.substr(8, 4)) + idParts.push(id.substr(12, 4)) + idParts.push(id.substr(16, 4)) + idParts.push(id.substr(20, 12)) + return idParts.join('-') + }, checkNoteIdValid: function (id) { var uuidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i var result = id.match(uuidRegex) @@ -190,6 +207,16 @@ module.exports = function (sequelize, DataTypes) { return _callback(err, null) }) }, + parseNoteIdByBase64Url: function (_callback) { + // try to parse note id by base64url + try { + var id = Note.decodeNoteId(noteId) + if (id && Note.checkNoteIdValid(id)) { return callback(null, id) } else { return _callback(null, null) } + } catch (err) { + return _callback(err, null) + } + }, + // parse note id by LZString is deprecated, here for compability parseNoteIdByLZString: function (_callback) { // try to parse note id by LZString Base64 try { diff --git a/lib/realtime.js b/lib/realtime.js index d6ba62b25..5ee9f8fde 100644 --- a/lib/realtime.js +++ b/lib/realtime.js @@ -5,7 +5,6 @@ var cookie = require('cookie') var cookieParser = require('cookie-parser') var url = require('url') var async = require('async') -var LZString = require('lz-string') var randomcolor = require('randomcolor') var Chance = require('chance') var chance = new Chance() @@ -703,7 +702,7 @@ function operationCallback (socket, operation) { } function updateHistory (userId, note, time) { - var noteId = note.alias ? note.alias : LZString.compressToBase64(note.id) + var noteId = note.alias ? note.alias : models.Note.encodeNoteId(note.id) if (note.server) history.updateHistory(userId, noteId, note.server.document, time) } diff --git a/lib/response.js b/lib/response.js index 41e8c3364..25b9fafc4 100644 --- a/lib/response.js +++ b/lib/response.js @@ -3,7 +3,6 @@ // external modules var fs = require('fs') var markdownpdf = require('markdown-pdf') -var LZString = require('lz-string') var shortId = require('shortid') var querystring = require('querystring') var request = require('request') @@ -124,7 +123,7 @@ function newNote (req, res, next) { alias: req.alias ? req.alias : null, content: req.body ? req.body : '' }).then(function (note) { - return res.redirect(config.serverurl + '/' + LZString.compressToBase64(note.id)) + return res.redirect(config.serverurl + '/' + models.Note.encodeNoteId(note.id)) }).catch(function (err) { logger.error(err) return response.errorInternalError(res) @@ -179,7 +178,7 @@ function showNote (req, res, next) { findNote(req, res, function (note) { // force to use note id var noteId = req.params.noteId - var id = LZString.compressToBase64(note.id) + var id = models.Note.encodeNoteId(note.id) if ((note.alias && noteId !== note.alias) || (!note.alias && noteId !== id)) { return res.redirect(config.serverurl + '/' + (note.alias || id)) } return responseHackMD(res, note) }) @@ -321,7 +320,7 @@ function actionPDF (req, res, note) { function actionGist (req, res, note) { var data = { client_id: config.github.clientID, - redirect_uri: config.serverurl + '/auth/github/callback/' + LZString.compressToBase64(note.id) + '/gist', + redirect_uri: config.serverurl + '/auth/github/callback/' + models.Note.encodeNoteId(note.id) + '/gist', scope: 'gist', state: shortId.generate() } @@ -418,7 +417,7 @@ function publishNoteActions (req, res, next) { var action = req.params.action switch (action) { case 'edit': - res.redirect(config.serverurl + '/' + (note.alias ? note.alias : LZString.compressToBase64(note.id))) + res.redirect(config.serverurl + '/' + (note.alias ? note.alias : models.Note.encodeNoteId(note.id))) break default: res.redirect(config.serverurl + '/s/' + note.shortid) @@ -432,7 +431,7 @@ function publishSlideActions (req, res, next) { var action = req.params.action switch (action) { case 'edit': - res.redirect(config.serverurl + '/' + (note.alias ? note.alias : LZString.compressToBase64(note.id))) + res.redirect(config.serverurl + '/' + (note.alias ? note.alias : models.Note.encodeNoteId(note.id))) break default: res.redirect(config.serverurl + '/p/' + note.shortid) diff --git a/package.json b/package.json index ba8b05d75..beb01122b 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ "Idle.Js": "git+https://github.com/shawnmclean/Idle.js", "async": "^2.1.4", "aws-sdk": "^2.7.20", + "base64url": "^2.0.0", "blueimp-md5": "^2.6.0", "body-parser": "^1.15.2", "bootstrap": "^3.3.7", From 44298baa935916c61d8402122ed5801b1d973acd Mon Sep 17 00:00:00 2001 From: Max Wu Date: Mon, 26 Feb 2018 16:46:59 +0800 Subject: [PATCH 2/9] Add migration for LZString compressed note id in history Signed-off-by: Max Wu --- lib/history.js | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/lib/history.js b/lib/history.js index f46ff49f6..f3d4440e3 100644 --- a/lib/history.js +++ b/lib/history.js @@ -1,6 +1,7 @@ 'use strict' // history // external modules +var LZString = require('lz-string') // core var config = require('./config') @@ -27,7 +28,26 @@ function getHistory (userid, callback) { } var history = {} if (user.history) { - history = parseHistoryToObject(JSON.parse(user.history)) + history = JSON.parse(user.history) + // migrate LZString encoded note id to base64url encoded note id + for (let i = 0, l = history.length; i < l; i++) { + let item = history[i] + // try to parse in base64url + let id = models.Note.decodeNoteId(item.id) + if (!id || !models.Note.checkNoteIdValid(id)) { + // try to parse in LZString if it can't be parsed in base64url + try { + id = LZString.decompressFromBase64(item.id) + } catch (err) { + id = null + } + if (id && models.Note.checkNoteIdValid(id)) { + // replace the note id to base64url encoded note id + history[i].id = models.Note.encodeNoteId(id) + } + } + } + history = parseHistoryToObject(history) } if (config.debug) { logger.info('read history success: ' + user.id) From fe429e9ac17b73638835b2ec1c5033043c5f9942 Mon Sep 17 00:00:00 2001 From: Max Wu Date: Tue, 27 Feb 2018 20:57:31 +0800 Subject: [PATCH 3/9] Update to use buffer in encode/decode note id Signed-off-by: Max Wu --- lib/models/note.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/models/note.js b/lib/models/note.js index e199a3dbd..119d72c34 100644 --- a/lib/models/note.js +++ b/lib/models/note.js @@ -117,11 +117,13 @@ module.exports = function (sequelize, DataTypes) { }, encodeNoteId: function (id) { // remove dashes in UUID and encode in url-safe base64 - return base64url.encode(id.replace(/-/g, '')) + let str = id.replace(/-/g, '') + let hexStr = Buffer.from(str, 'hex') + return base64url.encode(hexStr) }, decodeNoteId: function (encodedId) { // decode from url-safe base64 - let id = base64url.decode(encodedId) + let id = base64url.toBuffer(encodedId).toString('hex') // add dashes between the UUID string parts let idParts = [] idParts.push(id.substr(0, 8)) From d08c9522c0dd414a6fed1671064701160d233603 Mon Sep 17 00:00:00 2001 From: Max Wu Date: Sat, 3 Mar 2018 16:25:30 +0800 Subject: [PATCH 4/9] Update to migrate note url in the history of browser storage and cookie Signed-off-by: Max Wu --- public/js/history.js | 15 +++++++++++++++ public/js/utils.js | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) create mode 100644 public/js/utils.js diff --git a/public/js/history.js b/public/js/history.js index e14b80d84..e7d289fb4 100644 --- a/public/js/history.js +++ b/public/js/history.js @@ -3,6 +3,12 @@ import store from 'store' import S from 'string' +import LZString from 'lz-string' + +import { + checkNoteIdValid, + encodeNoteId +} from './utils' import { checkIfAuth @@ -291,6 +297,15 @@ function parseToHistory (list, notehistory, callback) { else if (!list || !notehistory) callback(list, notehistory) else if (notehistory && notehistory.length > 0) { for (let i = 0; i < notehistory.length; i++) { + // migrate LZString encoded id to base64url encoded id + try { + let id = LZString.decompressFromBase64(notehistory[i].id) + if (id && checkNoteIdValid(id)) { + notehistory[i].id = encodeNoteId(id) + } + } catch (err) { + // na + } // parse time to timestamp and fromNow const timestamp = (typeof notehistory[i].time === 'number' ? moment(notehistory[i].time) : moment(notehistory[i].time, 'MMMM Do YYYY, h:mm:ss a')) notehistory[i].timestamp = timestamp.valueOf() diff --git a/public/js/utils.js b/public/js/utils.js new file mode 100644 index 000000000..91e7f133f --- /dev/null +++ b/public/js/utils.js @@ -0,0 +1,32 @@ +import base64url from 'base64url' + +let uuidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i + +export function checkNoteIdValid (id) { + let result = id.match(uuidRegex) + if (result && result.length === 1) { + return true + } else { + return false + } +} + +export function encodeNoteId (id) { + // remove dashes in UUID and encode in url-safe base64 + let str = id.replace(/-/g, '') + let hexStr = Buffer.from(str, 'hex') + return base64url.encode(hexStr) +} + +export function decodeNoteId (encodedId) { + // decode from url-safe base64 + let id = base64url.toBuffer(encodedId).toString('hex') + // add dashes between the UUID string parts + let idParts = [] + idParts.push(id.substr(0, 8)) + idParts.push(id.substr(8, 4)) + idParts.push(id.substr(12, 4)) + idParts.push(id.substr(16, 4)) + idParts.push(id.substr(20, 12)) + return idParts.join('-') +} From 16cb842b946d55668318c08cf2e0aed001b9f855 Mon Sep 17 00:00:00 2001 From: Max Wu Date: Sat, 10 Mar 2018 16:51:00 +0800 Subject: [PATCH 5/9] Improve history migration performance Signed-off-by: Max Wu --- lib/history.js | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/lib/history.js b/lib/history.js index f3d4440e3..c7d2472ca 100644 --- a/lib/history.js +++ b/lib/history.js @@ -31,20 +31,14 @@ function getHistory (userid, callback) { history = JSON.parse(user.history) // migrate LZString encoded note id to base64url encoded note id for (let i = 0, l = history.length; i < l; i++) { - let item = history[i] - // try to parse in base64url - let id = models.Note.decodeNoteId(item.id) - if (!id || !models.Note.checkNoteIdValid(id)) { - // try to parse in LZString if it can't be parsed in base64url - try { - id = LZString.decompressFromBase64(item.id) - } catch (err) { - id = null - } + try { + let id = LZString.decompressFromBase64(history[i].id) if (id && models.Note.checkNoteIdValid(id)) { - // replace the note id to base64url encoded note id history[i].id = models.Note.encodeNoteId(id) } + } catch (err) { + // most error here comes from LZString, ignore + logger.error(err) } } history = parseHistoryToObject(history) From c7657ae81e23102cedd31543ee111d2736dc3b22 Mon Sep 17 00:00:00 2001 From: Max Wu Date: Sat, 10 Mar 2018 16:52:24 +0800 Subject: [PATCH 6/9] Fix parseNoteId order to fix some edge case that LZString note url could be parsed by base64url note url and thus return wrong note id Signed-off-by: Max Wu --- lib/models/note.js | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/models/note.js b/lib/models/note.js index 119d72c34..dc4d187b3 100644 --- a/lib/models/note.js +++ b/lib/models/note.js @@ -209,15 +209,6 @@ module.exports = function (sequelize, DataTypes) { return _callback(err, null) }) }, - parseNoteIdByBase64Url: function (_callback) { - // try to parse note id by base64url - try { - var id = Note.decodeNoteId(noteId) - if (id && Note.checkNoteIdValid(id)) { return callback(null, id) } else { return _callback(null, null) } - } catch (err) { - return _callback(err, null) - } - }, // parse note id by LZString is deprecated, here for compability parseNoteIdByLZString: function (_callback) { // try to parse note id by LZString Base64 @@ -228,6 +219,15 @@ module.exports = function (sequelize, DataTypes) { return _callback(err, null) } }, + parseNoteIdByBase64Url: function (_callback) { + // try to parse note id by base64url + try { + var id = Note.decodeNoteId(noteId) + if (id && Note.checkNoteIdValid(id)) { return callback(null, id) } else { return _callback(null, null) } + } catch (err) { + return _callback(err, null) + } + }, parseNoteIdByShortId: function (_callback) { // try to parse note id by shortId try { From 5e975cbe690048e144fb6bd99c5b239a3e764445 Mon Sep 17 00:00:00 2001 From: Max Wu Date: Sun, 11 Mar 2018 02:52:24 +0800 Subject: [PATCH 7/9] Fix to log instead of throwing error on parse note id Signed-off-by: Max Wu --- lib/models/note.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/models/note.js b/lib/models/note.js index dc4d187b3..d615bcf7e 100644 --- a/lib/models/note.js +++ b/lib/models/note.js @@ -216,7 +216,8 @@ module.exports = function (sequelize, DataTypes) { var id = LZString.decompressFromBase64(noteId) if (id && Note.checkNoteIdValid(id)) { return callback(null, id) } else { return _callback(null, null) } } catch (err) { - return _callback(err, null) + logger.error(err) + return _callback(null, null) } }, parseNoteIdByBase64Url: function (_callback) { @@ -225,7 +226,8 @@ module.exports = function (sequelize, DataTypes) { var id = Note.decodeNoteId(noteId) if (id && Note.checkNoteIdValid(id)) { return callback(null, id) } else { return _callback(null, null) } } catch (err) { - return _callback(err, null) + logger.error(err) + return _callback(null, null) } }, parseNoteIdByShortId: function (_callback) { From dfd833dbe2c21ea6ccdc185b717c59894950816d Mon Sep 17 00:00:00 2001 From: Max Wu Date: Sun, 11 Mar 2018 02:55:54 +0800 Subject: [PATCH 8/9] Update to show log on migrate LZString type note url in history Signed-off-by: Max Wu --- public/js/history.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/js/history.js b/public/js/history.js index e7d289fb4..a65753602 100644 --- a/public/js/history.js +++ b/public/js/history.js @@ -304,7 +304,7 @@ function parseToHistory (list, notehistory, callback) { notehistory[i].id = encodeNoteId(id) } } catch (err) { - // na + logger.error(err) } // parse time to timestamp and fromNow const timestamp = (typeof notehistory[i].time === 'number' ? moment(notehistory[i].time) : moment(notehistory[i].time, 'MMMM Do YYYY, h:mm:ss a')) From 8bfe51940f2eff035394b7713cbbce5b9b446842 Mon Sep 17 00:00:00 2001 From: Max Wu Date: Sun, 11 Mar 2018 03:00:36 +0800 Subject: [PATCH 9/9] Fix typo Signed-off-by: Max Wu --- public/js/history.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/js/history.js b/public/js/history.js index a65753602..713228187 100644 --- a/public/js/history.js +++ b/public/js/history.js @@ -304,7 +304,7 @@ function parseToHistory (list, notehistory, callback) { notehistory[i].id = encodeNoteId(id) } } catch (err) { - logger.error(err) + console.error(err) } // parse time to timestamp and fromNow const timestamp = (typeof notehistory[i].time === 'number' ? moment(notehistory[i].time) : moment(notehistory[i].time, 'MMMM Do YYYY, h:mm:ss a'))