Run database migrations automatically on startup

Instead of using sequelize-cli and ensure migrations by shellscript,
this patch automates database migrations properly to the umzug library.
The sequelize CLI becomes a dev dependencies as it's still useful for
generating migrations.

This should eliminate the need for crude generating of database config
files and alike. Instead we utilize the pre-configured sequelize
connection that CodiMD will use anyway.

Signed-off-by: Sheogorath <sheogorath@shivering-isles.com>

package.json

@@ -15,7 +15,7 @@
     "build": "yarn run build-backend && yarn run build-frontend",
     "build-backend": "tsc",
     "build-frontend": "webpack --config webpack.prod.js --progress --colors --bail",
-    "start": "sequelize db:migrate && node built/lib/app.js"
+    "start": "node built/lib/app.js"
   },
   "dependencies": {
     "@passport-next/passport-openid": "^1.0.0",
@@ -116,7 +116,6 @@
     "scrypt-kdf": "^2.0.1",
     "select2": "^3.5.2-browserify",
     "sequelize": "^5.21.1",
-    "sequelize-cli": "^5.5.1",
     "sequelize-typescript": "^1.1.0",
     "shortid": "2.2.8",
     "socket.io": "~2.1.1",
@@ -128,6 +127,7 @@
     "tedious": "^6.6.0",
     "toobusy-js": "^0.5.1",
     "turndown": "^5.0.1",
+    "umzug": "^2.3.0",
     "uuid": "^3.1.0",
     "validator": "^10.4.0",
     "velocity-animate": "^1.4.0",
@@ -221,6 +221,7 @@
     "mocha": "^5.2.0",
     "optimize-css-assets-webpack-plugin": "^5.0.3",
     "script-loader": "^0.7.2",
+    "sequelize-cli": "^5.5.1",
     "sinon": "^9.0.2",
     "string-loader": "^0.0.1",
     "style-loader": "^1.0.0",

src/lib/app.ts

@@ -24,7 +24,7 @@ import { config } from './config'
 import { addNonceToLocals, computeDirectives } from './csp'
 import { errors } from './errors'
 import { logger } from './logger'
-import { Revision, sequelize } from './models'
+import { Revision, sequelize, runMigrations } from './models'
 import { realtime, State } from './realtime'
 import { handleTermSignals } from './utils/functions'
 import { AuthRouter, BaseRouter, HistoryRouter, ImageRouter, NoteRouter, StatusRouter, UserRouter } from './web/'
@@ -289,7 +289,8 @@ function startListen (): void {
 }
 
 // sync db then start listen
-sequelize.authenticate().then(function () {
+sequelize.authenticate().then(async function () {
+  await runMigrations()
   sessionStore.sync()
   // check if realtime is ready
   if (realtime.isReady()) {

src/lib/migrations/20150504155329-create-users.js

@@ -1,6 +1,6 @@
 'use strict'
 module.exports = {
-  up: function (queryInterface, Sequelize) {
+  up: async function (queryInterface, Sequelize) {
     return queryInterface.createTable('Users', {
       id: {
         type: Sequelize.UUID,
@@ -18,7 +18,7 @@ module.exports = {
     })
   },
 
-  down: function (queryInterface, Sequelize) {
+  down: async function (queryInterface, Sequelize) {
     return queryInterface.dropTable('Users')
   }
 }

src/lib/migrations/20150508114741-create-notes.js

@@ -1,6 +1,6 @@
 'use strict'
 module.exports = {
-  up: function (queryInterface, Sequelize) {
+  up: async function (queryInterface, Sequelize) {
     return queryInterface.createTable('Notes', {
       id: {
         type: Sequelize.UUID,
@@ -15,7 +15,7 @@ module.exports = {
     })
   },
 
-  down: function (queryInterface, Sequelize) {
+  down: async function (queryInterface, Sequelize) {
     return queryInterface.dropTable('Notes')
   }
 }

src/lib/migrations/20150515125813-create-temp.js

@@ -1,6 +1,6 @@
 'use strict'
 module.exports = {
-  up: function (queryInterface, Sequelize) {
+  up: async function (queryInterface, Sequelize) {
     return queryInterface.createTable('Temp', {
       id: {
         type: Sequelize.STRING,
@@ -12,7 +12,7 @@ module.exports = {
     })
   },
 
-  down: function (queryInterface, Sequelize) {
+  down: async function (queryInterface, Sequelize) {
     return queryInterface.dropTable('Temp')
   }
 }

src/lib/migrations/20150702001020-update-to-0_3_1.js

@@ -1,6 +1,6 @@
 'use strict'
 module.exports = {
-  up: function (queryInterface, Sequelize) {
+  up: async function (queryInterface, Sequelize) {
     return queryInterface.addColumn('Notes', 'shortid', {
       type: Sequelize.STRING,
       defaultValue: '0000000000',
@@ -30,7 +30,7 @@ module.exports = {
     })
   },
 
-  down: function (queryInterface, Sequelize) {
+  down: async function (queryInterface, Sequelize) {
     return queryInterface.removeColumn('Notes', 'viewcount')
       .then(function () {
         return queryInterface.removeColumn('Notes', 'permission')

src/lib/migrations/20150915153700-change-notes-title-to-text.js

@@ -4,7 +4,7 @@ function isSQLite (sequelize) {
 }
 
 module.exports = {
-  up: function (queryInterface, Sequelize) {
+  up: async function (queryInterface, Sequelize) {
     return queryInterface.changeColumn('Notes', 'title', {
       type: Sequelize.TEXT
     }).then(function () {
@@ -15,7 +15,7 @@ module.exports = {
     })
   },
 
-  down: function (queryInterface, Sequelize) {
+  down: async function (queryInterface, Sequelize) {
     return queryInterface.changeColumn('Notes', 'title', {
       type: Sequelize.STRING
     }).then(function () {

src/lib/migrations/20160112220142-note-add-lastchange.js

@@ -1,6 +1,6 @@
 'use strict'
 module.exports = {
-  up: function (queryInterface, Sequelize) {
+  up: async function (queryInterface, Sequelize) {
     return queryInterface.addColumn('Notes', 'lastchangeuserId', {
       type: Sequelize.UUID
     }).then(function () {
@@ -17,7 +17,7 @@ module.exports = {
     })
   },
 
-  down: function (queryInterface, Sequelize) {
+  down: async function (queryInterface, Sequelize) {
     return queryInterface.removeColumn('Notes', 'lastchangeAt')
       .then(function () {
         return queryInterface.removeColumn('Notes', 'lastchangeuserId')

src/lib/migrations/20160420180355-note-add-alias.js

@@ -1,6 +1,6 @@
 'use strict'
 module.exports = {
-  up: function (queryInterface, Sequelize) {
+  up: async function (queryInterface, Sequelize) {
     return queryInterface.addColumn('Notes', 'alias', {
       type: Sequelize.STRING
     }).then(function () {
@@ -17,7 +17,7 @@ module.exports = {
     })
   },
 
-  down: function (queryInterface, Sequelize) {
+  down: async function (queryInterface, Sequelize) {
     return queryInterface.removeColumn('Notes', 'alias').then(function () {
       return queryInterface.removeIndex('Notes', ['alias'])
     })

src/lib/migrations/20160515114000-user-add-tokens.js

@@ -1,6 +1,6 @@
 'use strict'
 module.exports = {
-  up: function (queryInterface, Sequelize) {
+  up: async function (queryInterface, Sequelize) {
     return queryInterface.addColumn('Users', 'accessToken', Sequelize.STRING).then(function () {
       return queryInterface.addColumn('Users', 'refreshToken', Sequelize.STRING)
     }).catch(function (error) {
@@ -13,7 +13,7 @@ module.exports = {
     })
   },
 
-  down: function (queryInterface, Sequelize) {
+  down: async function (queryInterface, Sequelize) {
     return queryInterface.removeColumn('Users', 'accessToken').then(function () {
       return queryInterface.removeColumn('Users', 'refreshToken')
     })

src/lib/migrations/20160607060246-support-revision.js

@@ -1,6 +1,6 @@
 'use strict'
 module.exports = {
-  up: function (queryInterface, Sequelize) {
+  up: async function (queryInterface, Sequelize) {
     return queryInterface.addColumn('Notes', 'savedAt', Sequelize.DATE).then(function () {
       return queryInterface.createTable('Revisions', {
         id: {
@@ -25,7 +25,7 @@ module.exports = {
     })
   },
 
-  down: function (queryInterface, Sequelize) {
+  down: async function (queryInterface, Sequelize) {
     return queryInterface.dropTable('Revisions').then(function () {
       return queryInterface.removeColumn('Notes', 'savedAt')
     })

src/lib/migrations/20160703062241-support-authorship.js

@@ -1,6 +1,6 @@
 'use strict'
 module.exports = {
-  up: function (queryInterface, Sequelize) {
+  up: async function (queryInterface, Sequelize) {
     return queryInterface.addColumn('Notes', 'authorship', Sequelize.TEXT).then(function () {
       return queryInterface.addColumn('Revisions', 'authorship', Sequelize.TEXT)
     }).then(function () {
@@ -26,7 +26,7 @@ module.exports = {
     })
   },
 
-  down: function (queryInterface, Sequelize) {
+  down: async function (queryInterface, Sequelize) {
     return queryInterface.dropTable('Authors').then(function () {
       return queryInterface.removeColumn('Revisions', 'authorship')
     }).then(function () {

src/lib/migrations/20161009040430-support-delete-note.js

@@ -1,6 +1,6 @@
 'use strict'
 module.exports = {
-  up: function (queryInterface, Sequelize) {
+  up: async function (queryInterface, Sequelize) {
     return queryInterface.addColumn('Notes', 'deletedAt', Sequelize.DATE).catch(function (error) {
       if (error.message === 'SQLITE_ERROR: duplicate column name: deletedAt' || error.message === "ER_DUP_FIELDNAME: Duplicate column name 'deletedAt'" || error.message === 'column "deletedAt" of relation "Notes" already exists') {
         // eslint-disable-next-line no-console
@@ -11,7 +11,7 @@ module.exports = {
     })
   },
 
-  down: function (queryInterface, Sequelize) {
+  down: async function (queryInterface, Sequelize) {
     return queryInterface.removeColumn('Notes', 'deletedAt')
   }
 }

src/lib/migrations/20161201050312-support-email-signin.js

@@ -1,6 +1,6 @@
 'use strict'
 module.exports = {
-  up: function (queryInterface, Sequelize) {
+  up: async function (queryInterface, Sequelize) {
     return queryInterface.addColumn('Users', 'email', Sequelize.TEXT).then(function () {
       return queryInterface.addColumn('Users', 'password', Sequelize.TEXT).catch(function (error) {
         if (error.message === "ER_DUP_FIELDNAME: Duplicate column name 'password'" || error.message === 'column "password" of relation "Users" already exists') {
@@ -20,7 +20,7 @@ module.exports = {
     })
   },
 
-  down: function (queryInterface, Sequelize) {
+  down: async function (queryInterface, Sequelize) {
     return queryInterface.removeColumn('Users', 'email').then(function () {
       return queryInterface.removeColumn('Users', 'password')
     })

src/lib/migrations/20180306150303-fix-enum.js

@@ -1,11 +1,11 @@
 'use strict'
 
 module.exports = {
-  up: function (queryInterface, Sequelize) {
+  up: async function (queryInterface, Sequelize) {
     return queryInterface.changeColumn('Notes', 'permission', { type: Sequelize.ENUM('freely', 'editable', 'limited', 'locked', 'protected', 'private') })
   },
 
-  down: function (queryInterface, Sequelize) {
+  down: async function (queryInterface, Sequelize) {
     return queryInterface.changeColumn('Notes', 'permission', { type: Sequelize.ENUM('freely', 'editable', 'locked', 'private') })
   }
 }

src/lib/migrations/20180326103000-use-text-in-tokens.js

@@ -1,7 +1,7 @@
 'use strict'
 
 module.exports = {
-  up: function (queryInterface, Sequelize) {
+  up: async function (queryInterface, Sequelize) {
     return queryInterface.changeColumn('Users', 'accessToken', {
       type: Sequelize.TEXT
     }).then(function () {
@@ -11,7 +11,7 @@ module.exports = {
     })
   },
 
-  down: function (queryInterface, Sequelize) {
+  down: async function (queryInterface, Sequelize) {
     return queryInterface.changeColumn('Users', 'accessToken', {
       type: Sequelize.STRING
     }).then(function () {

src/lib/migrations/20180525153000-user-add-delete-token.js

@@ -1,13 +1,13 @@
 'use strict'
 module.exports = {
-  up: function (queryInterface, Sequelize) {
+  up: async function (queryInterface, Sequelize) {
     return queryInterface.addColumn('Users', 'deleteToken', {
       type: Sequelize.UUID,
       defaultValue: Sequelize.UUIDV4
     })
   },
 
-  down: function (queryInterface, Sequelize) {
+  down: async function (queryInterface, Sequelize) {
     return queryInterface.removeColumn('Users', 'deleteToken')
   }
 }

src/lib/models/index.ts

@@ -1,5 +1,6 @@
 import { Sequelize } from 'sequelize-typescript'
 import { cloneDeep } from 'lodash'
+import * as path from 'path'
 import { Author } from './author'
 import { Note } from './note'
 import { Revision } from './revision'
@@ -7,6 +8,8 @@ import { Temp } from './temp'
 import { User } from './user'
 import { logger } from '../logger'
 import { config } from '../config'
+import Umzug from 'umzug'
+import SequelizeTypes from 'sequelize'
 
 const dbconfig = cloneDeep(config.db)
 dbconfig.logging = config.debug ? (data): void => {
@@ -22,6 +25,36 @@ if (config.dbURL) {
   sequelize = new Sequelize(dbconfig.database, dbconfig.username, dbconfig.password, dbconfig)
 }
 
+const umzug = new Umzug({
+  migrations: {
+    path: path.resolve(__dirname, '..', 'migrations'),
+    params: [
+      sequelize.getQueryInterface(),
+      SequelizeTypes
+    ]
+  },
+  // Required wrapper function required to prevent winstion issue
+  // https://github.com/winstonjs/winston/issues/1577
+  logging: message => {
+    logger.info(message)
+  },
+  storage: 'sequelize',
+  storageOptions: {
+    sequelize: sequelize
+  }
+})
+
+export async function runMigrations(): Promise<void> {
+  // checks migrations and run them if they are not already applied
+  // exit in case of unsuccessful migrations
+  await umzug.up().catch(error => {
+    logger.error(error)
+    logger.error('Database migration failed. Exiting…')
+    process.exit(1)
+  })
+  logger.info('All migrations performed successfully')
+}
+
 sequelize.addModels([Author, Note, Revision, Temp, User])
 
 export { Author, Note, Revision, Temp, User }

yarn.lock

@@ -10533,7 +10533,7 @@ ultron@~1.1.0:
   resolved "https://registry.yarnpkg.com/ultron/-/ultron-1.1.1.tgz#9fe1536a10a664a65266a1e3ccf85fd36302bc9c"
   integrity sha512-UIEXBNeYmKptWH6z8ZnqTeS8fV74zG0/eRU9VGkpzz+LIJNs8W/zM/L+7ctCkRrgbNnnR0xxw4bKOr0cW0N0Og==
 
-umzug@^2.1.0:
+umzug@^2.1.0, umzug@^2.3.0:
   version "2.3.0"
   resolved "https://registry.yarnpkg.com/umzug/-/umzug-2.3.0.tgz#0ef42b62df54e216b05dcaf627830a6a8b84a184"
   integrity sha512-Z274K+e8goZK8QJxmbRPhl89HPO1K+ORFtm6rySPhFKfKc5GHhqdzD0SGhSWHkzoXasqJuItdhorSvY7/Cgflw==