From 59e87a8729b9a2e19f3b3b416e940d18ce41b26b Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Mon, 8 Jun 2015 18:35:24 -0300 Subject: [PATCH 1/5] initial version of texcount --- services/clsi/app.coffee | 1 + .../clsi/app/coffee/CompileController.coffee | 10 ++++ .../clsi/app/coffee/CompileManager.coffee | 50 ++++++++++++++++++- .../acceptance/coffee/SynctexTests.coffee | 17 +++++++ .../acceptance/coffee/helpers/Client.coffee | 9 ++++ 5 files changed, 86 insertions(+), 1 deletion(-) diff --git a/services/clsi/app.coffee b/services/clsi/app.coffee index 9083bb4bf3..8e451a7650 100644 --- a/services/clsi/app.coffee +++ b/services/clsi/app.coffee @@ -36,6 +36,7 @@ app.delete "/project/:project_id", CompileController.clearCache app.get "/project/:project_id/sync/code", CompileController.syncFromCode app.get "/project/:project_id/sync/pdf", CompileController.syncFromPdf +app.get "/project/:project_id/wordcount", CompileController.wordcount ForbidSymlinks = require "./app/js/StaticServerForbidSymlinks" diff --git a/services/clsi/app/coffee/CompileController.coffee b/services/clsi/app/coffee/CompileController.coffee index 71368be737..ce107f8eab 100644 --- a/services/clsi/app/coffee/CompileController.coffee +++ b/services/clsi/app/coffee/CompileController.coffee @@ -66,3 +66,13 @@ module.exports = CompileController = res.send JSON.stringify { code: codePositions } + + wordcount: (req, res, next = (error) ->) -> + file = req.query.file + project_id = req.params.project_id + + CompileManager.wordcount project_id, file, (error, result) -> + return next(error) if error? + res.send JSON.stringify { + texcount: result + } diff --git a/services/clsi/app/coffee/CompileManager.coffee b/services/clsi/app/coffee/CompileManager.coffee index ac0c241149..5bbd3ab2e1 100644 --- a/services/clsi/app/coffee/CompileManager.coffee +++ b/services/clsi/app/coffee/CompileManager.coffee @@ -107,4 +107,52 @@ module.exports = CompileManager = line: parseInt(line, 10) column: parseInt(column, 10) } - return results \ No newline at end of file + return results + + _parseWordcountFromOutput: (output) -> + results = { + encode: "" + textWords: 0 + headWords: 0 + outside: 0 + headers: 0 + elements: 0 + mathInline: 0 + mathDisplay: 0 + } + for line in output.split("\n") + [data, info] = line.split(":") + if data.indexOf("Encoding") > -1 + results['encode'] = info.trim() + if data.indexOf("in text") > -1 + results['textWords'] = parseInt(info, 10) + if data.indexOf("in head") > -1 + results['headWords'] = parseInt(info, 10) + if data.indexOf("outside") > -1 + results['outside'] = parseInt(info, 10) + if data.indexOf("of head") > -1 + results['headers'] = parseInt(info, 10) + if data.indexOf("float") > -1 + results['elements'] = parseInt(info, 10) + if data.indexOf("inlines") > -1 + results['mathInline'] = parseInt(info, 10) + if data.indexOf("displayed") > -1 + results['mathDisplay'] = parseInt(info, 10) + + return results + + wordcount: (project_id, file_name, callback = (error, pdfPositions) ->) -> + base_dir = Settings.path.synctexBaseDir(project_id) + file_path = base_dir + "/" + file_name + logger.log project_id: project_id, file_name: file_name, "try wordcount" + CompileManager._runWordcount [file_path], (error, stdout) -> + return callback(error) if error? + logger.log project_id: project_id, file_name: file_name, stdout: stdout, "wordcount output" + callback null, CompileManager._parseWordcountFromOutput(stdout) + + _runWordcount: (args, callback = (error, stdout) ->) -> + bin_path = Path.resolve("texcount") + seconds = 1000 + child_process.execFile "texcount", args, timeout: 10 * seconds, (error, stdout, stderr) -> + return callback(error) if error? + callback(null, stdout) \ No newline at end of file diff --git a/services/clsi/test/acceptance/coffee/SynctexTests.coffee b/services/clsi/test/acceptance/coffee/SynctexTests.coffee index 330eaff22e..8979b983e5 100644 --- a/services/clsi/test/acceptance/coffee/SynctexTests.coffee +++ b/services/clsi/test/acceptance/coffee/SynctexTests.coffee @@ -36,3 +36,20 @@ describe "Syncing", -> ) done() + describe "wordcount file", -> + it "should return wordcount info", (done) -> + Client.wordcount @project_id, "main.tex", (error, result) -> + throw error if error? + expect(result).to.deep.equal( + texcount: { + encode: "ascii" + textWords: 2 + headWords: 0 + outside: 0 + headers: 0 + elements: 0 + mathInline: 0 + mathDisplay: 0 + } + ) + done() diff --git a/services/clsi/test/acceptance/coffee/helpers/Client.coffee b/services/clsi/test/acceptance/coffee/helpers/Client.coffee index 7025b79cce..3e1a5b86d0 100644 --- a/services/clsi/test/acceptance/coffee/helpers/Client.coffee +++ b/services/clsi/test/acceptance/coffee/helpers/Client.coffee @@ -90,3 +90,12 @@ module.exports = Client = @compile project_id, req, callback + wordcount: (project_id, file, callback = (error, pdfPositions) ->) -> + request.get { + url: "#{@host}/project/#{project_id}/wordcount" + qs: { + file: file + } + }, (error, response, body) -> + return callback(error) if error? + callback null, JSON.parse(body) From 32c0a89ae1fb798a73dfdcbfc079fa3c31d9e29d Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Mon, 8 Jun 2015 19:27:47 -0300 Subject: [PATCH 2/5] add unit test --- .../clsi/app/coffee/CompileManager.coffee | 2 -- .../unit/coffee/CompileControllerTests.coffee | 24 ++++++++++++++ .../unit/coffee/CompileManagerTests.coffee | 31 ++++++++++++++++++- 3 files changed, 54 insertions(+), 3 deletions(-) diff --git a/services/clsi/app/coffee/CompileManager.coffee b/services/clsi/app/coffee/CompileManager.coffee index 5bbd3ab2e1..f689ae8833 100644 --- a/services/clsi/app/coffee/CompileManager.coffee +++ b/services/clsi/app/coffee/CompileManager.coffee @@ -144,14 +144,12 @@ module.exports = CompileManager = wordcount: (project_id, file_name, callback = (error, pdfPositions) ->) -> base_dir = Settings.path.synctexBaseDir(project_id) file_path = base_dir + "/" + file_name - logger.log project_id: project_id, file_name: file_name, "try wordcount" CompileManager._runWordcount [file_path], (error, stdout) -> return callback(error) if error? logger.log project_id: project_id, file_name: file_name, stdout: stdout, "wordcount output" callback null, CompileManager._parseWordcountFromOutput(stdout) _runWordcount: (args, callback = (error, stdout) ->) -> - bin_path = Path.resolve("texcount") seconds = 1000 child_process.execFile "texcount", args, timeout: 10 * seconds, (error, stdout, stderr) -> return callback(error) if error? diff --git a/services/clsi/test/unit/coffee/CompileControllerTests.coffee b/services/clsi/test/unit/coffee/CompileControllerTests.coffee index 3298f472e6..e0307d20c9 100644 --- a/services/clsi/test/unit/coffee/CompileControllerTests.coffee +++ b/services/clsi/test/unit/coffee/CompileControllerTests.coffee @@ -189,3 +189,27 @@ describe "CompileController", -> ) .should.equal true + describe "wordcount", -> + beforeEach -> + @file = "main.tex" + @project_id = "mock-project-id" + @req.params = + project_id: @project_id + @req.query = + file: @file + @res.send = sinon.stub() + + @CompileManager.wordcount = sinon.stub().callsArgWith(2, null, @texcount = ["mock-texcount"]) + @CompileController.wordcount @req, @res, @next + + it "should return the word count of a file", -> + @CompileManager.wordcount + .calledWith(@project_id, @file) + .should.equal true + + it "should return the texcount info", -> + @res.send + .calledWith(JSON.stringify + texcount: @texcount + ) + .should.equal true diff --git a/services/clsi/test/unit/coffee/CompileManagerTests.coffee b/services/clsi/test/unit/coffee/CompileManagerTests.coffee index d5644b6bb9..eae49627c1 100644 --- a/services/clsi/test/unit/coffee/CompileManagerTests.coffee +++ b/services/clsi/test/unit/coffee/CompileManagerTests.coffee @@ -171,4 +171,33 @@ describe "CompileManager", -> line: @line column: @column }]) - .should.equal true \ No newline at end of file + .should.equal true + + describe "wordcount", -> + beforeEach -> + @file_name = "main.tex" + @child_process.execFile = sinon.stub() + @child_process.execFile.callsArgWith(3, null, @stdout = "Encoding: ascii\nWords in text: 2", "") + @Settings.path.synctexBaseDir = (project_id) => "#{@Settings.path.compilesDir}/#{@project_id}" + @CompileManager.wordcount @project_id, @file_name, @callback + + it "should execute the texcount binary", -> + file_path = "#{@Settings.path.compilesDir}/#{@project_id}/#{@file_name}" + @child_process.execFile + .calledWith("texcount", [file_path], timeout: 10000) + .should.equal true + + it "should call the callback with the parsed output", -> + @callback + .calledWith(null, { + encode: "ascii" + textWords: 2 + headWords: 0 + outside: 0 + headers: 0 + elements: 0 + mathInline: 0 + mathDisplay: 0 + }) + .should.equal true + From d3326656488313b7ca421755c7af7f7476b526db Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Tue, 8 Sep 2015 10:19:46 -0300 Subject: [PATCH 3/5] move texcount to docker --- .../clsi/app/coffee/CompileManager.coffee | 19 +++++++------- .../unit/coffee/CompileManagerTests.coffee | 25 +++++++++++++------ 2 files changed, 26 insertions(+), 18 deletions(-) diff --git a/services/clsi/app/coffee/CompileManager.coffee b/services/clsi/app/coffee/CompileManager.coffee index f689ae8833..0392ccef2c 100644 --- a/services/clsi/app/coffee/CompileManager.coffee +++ b/services/clsi/app/coffee/CompileManager.coffee @@ -7,6 +7,8 @@ Path = require "path" logger = require "logger-sharelatex" Metrics = require "./Metrics" child_process = require "child_process" +CommandRunner = require(Settings.clsi?.commandRunner or "./CommandRunner") +fs = require("fs") module.exports = CompileManager = doCompile: (request, callback = (error, outputFiles) ->) -> @@ -142,15 +144,12 @@ module.exports = CompileManager = return results wordcount: (project_id, file_name, callback = (error, pdfPositions) ->) -> - base_dir = Settings.path.synctexBaseDir(project_id) - file_path = base_dir + "/" + file_name - CompileManager._runWordcount [file_path], (error, stdout) -> - return callback(error) if error? - logger.log project_id: project_id, file_name: file_name, stdout: stdout, "wordcount output" - callback null, CompileManager._parseWordcountFromOutput(stdout) + file_path = "$COMPILE_DIR/" + file_name + command = [ "texcount", file_path, "-out=" + file_path + ".wc"] + directory = Path.join(Settings.path.compilesDir, project_id) + timeout = 10 * 1000 - _runWordcount: (args, callback = (error, stdout) ->) -> - seconds = 1000 - child_process.execFile "texcount", args, timeout: 10 * seconds, (error, stdout, stderr) -> + CommandRunner.run project_id, command, directory, timeout, (error) -> return callback(error) if error? - callback(null, stdout) \ No newline at end of file + stdout = fs.readFileSync(directory + "/" + file_name + ".wc", "utf-8") + callback null, CompileManager._parseWordcountFromOutput(stdout) diff --git a/services/clsi/test/unit/coffee/CompileManagerTests.coffee b/services/clsi/test/unit/coffee/CompileManagerTests.coffee index eae49627c1..10ab2eac43 100644 --- a/services/clsi/test/unit/coffee/CompileManagerTests.coffee +++ b/services/clsi/test/unit/coffee/CompileManagerTests.coffee @@ -16,6 +16,8 @@ describe "CompileManager", -> "settings-sharelatex": @Settings = { path: compilesDir: "/compiles/dir" } "logger-sharelatex": @logger = { log: sinon.stub() } "child_process": @child_process = {} + "./CommandRunner": @CommandRunner = {} + "fs": @fs = {} @callback = sinon.stub() describe "doCompile", -> @@ -175,16 +177,24 @@ describe "CompileManager", -> describe "wordcount", -> beforeEach -> + @CommandRunner.run = sinon.stub().callsArg(4) + @fs.readFileSync = sinon.stub().returns @stdout = "Encoding: ascii\nWords in text: 2" + @callback = sinon.stub() + + @project_id = "project-id-123" + @timeout = 10 * 1000 @file_name = "main.tex" - @child_process.execFile = sinon.stub() - @child_process.execFile.callsArgWith(3, null, @stdout = "Encoding: ascii\nWords in text: 2", "") - @Settings.path.synctexBaseDir = (project_id) => "#{@Settings.path.compilesDir}/#{@project_id}" + @Settings.path.compilesDir = "/local/compile/directory" + @CompileManager.wordcount @project_id, @file_name, @callback - it "should execute the texcount binary", -> - file_path = "#{@Settings.path.compilesDir}/#{@project_id}/#{@file_name}" - @child_process.execFile - .calledWith("texcount", [file_path], timeout: 10000) + it "should run the texcount command", -> + @directory = "#{@Settings.path.compilesDir}/#{@project_id}" + @file_path = "$COMPILE_DIR/#{@file_name}" + @command =[ "texcount", @file_path, "-out=" + @file_path + ".wc"] + + @CommandRunner.run + .calledWith(@project_id, @command, @directory, @timeout) .should.equal true it "should call the callback with the parsed output", -> @@ -200,4 +210,3 @@ describe "CompileManager", -> mathDisplay: 0 }) .should.equal true - From ec338f8c10427cabb6bbb4826408dff2893e1464 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Wed, 9 Sep 2015 12:43:32 +0100 Subject: [PATCH 4/5] add -inc to word count use -inc to word count included files also moved private function to bottom --- .../clsi/app/coffee/CompileController.coffee | 2 +- .../clsi/app/coffee/CompileManager.coffee | 24 +++++++++---------- .../unit/coffee/CompileManagerTests.coffee | 2 +- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/services/clsi/app/coffee/CompileController.coffee b/services/clsi/app/coffee/CompileController.coffee index ce107f8eab..f7d46b1305 100644 --- a/services/clsi/app/coffee/CompileController.coffee +++ b/services/clsi/app/coffee/CompileController.coffee @@ -68,7 +68,7 @@ module.exports = CompileController = } wordcount: (req, res, next = (error) ->) -> - file = req.query.file + file = req.query.file || "main.tex" project_id = req.params.project_id CompileManager.wordcount project_id, file, (error, result) -> diff --git a/services/clsi/app/coffee/CompileManager.coffee b/services/clsi/app/coffee/CompileManager.coffee index 0392ccef2c..b4a8357bdb 100644 --- a/services/clsi/app/coffee/CompileManager.coffee +++ b/services/clsi/app/coffee/CompileManager.coffee @@ -111,6 +111,18 @@ module.exports = CompileManager = } return results + wordcount: (project_id, file_name, callback = (error, pdfPositions) ->) -> + logger.log project_id:project_id, file_name:file_name, "running wordcount" + file_path = "$COMPILE_DIR/" + file_name + command = [ "texcount", '-inc', file_path, "-out=" + file_path + ".wc"] + directory = Path.join(Settings.path.compilesDir, project_id) + timeout = 10 * 1000 + + CommandRunner.run project_id, command, directory, timeout, (error) -> + return callback(error) if error? + stdout = fs.readFileSync(directory + "/" + file_name + ".wc", "utf-8") + callback null, CompileManager._parseWordcountFromOutput(stdout) + _parseWordcountFromOutput: (output) -> results = { encode: "" @@ -140,16 +152,4 @@ module.exports = CompileManager = results['mathInline'] = parseInt(info, 10) if data.indexOf("displayed") > -1 results['mathDisplay'] = parseInt(info, 10) - return results - - wordcount: (project_id, file_name, callback = (error, pdfPositions) ->) -> - file_path = "$COMPILE_DIR/" + file_name - command = [ "texcount", file_path, "-out=" + file_path + ".wc"] - directory = Path.join(Settings.path.compilesDir, project_id) - timeout = 10 * 1000 - - CommandRunner.run project_id, command, directory, timeout, (error) -> - return callback(error) if error? - stdout = fs.readFileSync(directory + "/" + file_name + ".wc", "utf-8") - callback null, CompileManager._parseWordcountFromOutput(stdout) diff --git a/services/clsi/test/unit/coffee/CompileManagerTests.coffee b/services/clsi/test/unit/coffee/CompileManagerTests.coffee index 10ab2eac43..d6678b9407 100644 --- a/services/clsi/test/unit/coffee/CompileManagerTests.coffee +++ b/services/clsi/test/unit/coffee/CompileManagerTests.coffee @@ -191,7 +191,7 @@ describe "CompileManager", -> it "should run the texcount command", -> @directory = "#{@Settings.path.compilesDir}/#{@project_id}" @file_path = "$COMPILE_DIR/#{@file_name}" - @command =[ "texcount", @file_path, "-out=" + @file_path + ".wc"] + @command =[ "texcount", '-inc', @file_path, "-out=" + @file_path + ".wc"] @CommandRunner.run .calledWith(@project_id, @command, @directory, @timeout) From 4ee50b7239e7325b9fcb74dc411474e2e1e51788 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Wed, 9 Sep 2015 13:52:45 +0100 Subject: [PATCH 5/5] - fixed bug with texcount returning wrong data for nauty lines - improved acceptence test for word count to use nauty lines --- .../clsi/app/coffee/CompileManager.coffee | 6 +- .../acceptance/coffee/SynctexTests.coffee | 18 - .../acceptance/coffee/WordcountTests.coffee | 34 + .../acceptance/fixtures/naugty_strings.txt | 626 ++++++++++++++++++ .../unit/coffee/CompileManagerTests.coffee | 2 +- 5 files changed, 664 insertions(+), 22 deletions(-) create mode 100644 services/clsi/test/acceptance/coffee/WordcountTests.coffee create mode 100644 services/clsi/test/acceptance/fixtures/naugty_strings.txt diff --git a/services/clsi/app/coffee/CompileManager.coffee b/services/clsi/app/coffee/CompileManager.coffee index b4a8357bdb..296522f6b4 100644 --- a/services/clsi/app/coffee/CompileManager.coffee +++ b/services/clsi/app/coffee/CompileManager.coffee @@ -146,10 +146,10 @@ module.exports = CompileManager = results['outside'] = parseInt(info, 10) if data.indexOf("of head") > -1 results['headers'] = parseInt(info, 10) - if data.indexOf("float") > -1 + if data.indexOf("Number of floats/tables/figures") > -1 results['elements'] = parseInt(info, 10) - if data.indexOf("inlines") > -1 + if data.indexOf("Number of math inlines") > -1 results['mathInline'] = parseInt(info, 10) - if data.indexOf("displayed") > -1 + if data.indexOf("Number of math displayed") > -1 results['mathDisplay'] = parseInt(info, 10) return results diff --git a/services/clsi/test/acceptance/coffee/SynctexTests.coffee b/services/clsi/test/acceptance/coffee/SynctexTests.coffee index 8979b983e5..02b2397897 100644 --- a/services/clsi/test/acceptance/coffee/SynctexTests.coffee +++ b/services/clsi/test/acceptance/coffee/SynctexTests.coffee @@ -35,21 +35,3 @@ describe "Syncing", -> code: [ { file: 'main.tex', line: 3, column: -1 } ] ) done() - - describe "wordcount file", -> - it "should return wordcount info", (done) -> - Client.wordcount @project_id, "main.tex", (error, result) -> - throw error if error? - expect(result).to.deep.equal( - texcount: { - encode: "ascii" - textWords: 2 - headWords: 0 - outside: 0 - headers: 0 - elements: 0 - mathInline: 0 - mathDisplay: 0 - } - ) - done() diff --git a/services/clsi/test/acceptance/coffee/WordcountTests.coffee b/services/clsi/test/acceptance/coffee/WordcountTests.coffee new file mode 100644 index 0000000000..1789cfcdd0 --- /dev/null +++ b/services/clsi/test/acceptance/coffee/WordcountTests.coffee @@ -0,0 +1,34 @@ +Client = require "./helpers/Client" +request = require "request" +require("chai").should() +expect = require("chai").expect +path = require("path") +fs = require("fs") + +describe "Syncing", -> + before (done) -> + @request = + resources: [ + path: "main.tex" + content: fs.readFileSync(path.join(__dirname,"../fixtures/naugty_strings.txt"),"utf-8") + ] + @project_id = Client.randomId() + Client.compile @project_id, @request, (@error, @res, @body) => done() + + describe "wordcount file", -> + it "should return wordcount info", (done) -> + Client.wordcount @project_id, "main.tex", (error, result) -> + throw error if error? + expect(result).to.deep.equal( + texcount: { + encode: "utf8" + textWords: 2281 + headWords: 2 + outside: 0 + headers: 2 + elements: 0 + mathInline: 6 + mathDisplay: 0 + } + ) + done() diff --git a/services/clsi/test/acceptance/fixtures/naugty_strings.txt b/services/clsi/test/acceptance/fixtures/naugty_strings.txt new file mode 100644 index 0000000000..92eb1ddce6 --- /dev/null +++ b/services/clsi/test/acceptance/fixtures/naugty_strings.txt @@ -0,0 +1,626 @@ +\documentclass{article} +\usepackage[utf8]{inputenc} + +\title{eee} +\author{henry.oswald } +\date{September 2015} + +\usepackage{natbib} +\usepackage{graphicx} + +\begin{document} + +\maketitle + +\section{Introduction} + +Encoding: utf8 + +# Reserved Strings +# +# Strings which may be used elsewhere in code + +undefined +undef +null +NULL +(null) +nil +NIL +true +false +True +False +None +\ +\\ + +# Numeric Strings +# +# Strings which can be interpreted as numeric + +0 +1 +1.00 +$1.00 +1/2 +1E2 +1E02 +1E+02 +-1 +-1.00 +-$1.00 +-1/2 +-1E2 +-1E02 +-1E+02 +1/0 +0/0 +-2147483648/-1 +-9223372036854775808/-1 +0.00 +0..0 +. +0.0.0 +0,00 +0,,0 +, +0,0,0 +0.0/0 +1.0/0.0 +0.0/0.0 +1,0/0,0 +0,0/0,0 +--1 +- +-. +-, +999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999 +NaN +Infinity +-Infinity +0x0 +0xffffffff +0xffffffffffffffff +0xabad1dea +123456789012345678901234567890123456789 +1,000.00 +1 000.00 +1'000.00 +1,000,000.00 +1 000 000.00 +1'000'000.00 +1.000,00 +1 000,00 +1'000,00 +1.000.000,00 +1 000 000,00 +1'000'000,00 +01000 +08 +09 +2.2250738585072011e-308 + +# Special Characters +# +# Strings which contain common special ASCII characters (may need to be escaped) + +,./;'[]\-= +<>?:"{}|_+ +!@#$%^&*()`~ + +# Unicode Symbols +# +# Strings which contain common unicode symbols (e.g. smart quotes) + +Ω≈ç√∫˜µ≤≥÷ +åß∂ƒ©˙∆˚¬…æ +œ∑´®†¥¨ˆøπ“‘ +¡™£¢∞§¶•ªº–≠ +¸˛Ç◊ı˜Â¯˘¿ +ÅÍÎÏ˝ÓÔÒÚÆ☃ +Œ„´‰ˇÁ¨ˆØ∏”’ +`⁄€‹›fifl‡°·‚—± +⅛⅜⅝⅞ +ЁЂЃЄЅІЇЈЉЊЋЌЍЎЏАБВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯабвгдежзийклмнопрстуфхцчшщъыьэюя +٠١٢٣٤٥٦٧٨٩ + +# Unicode Subscript/Superscript +# +# Strings which contain unicode subscripts/superscripts; can cause rendering issues + +⁰⁴⁵ +₀₁₂ +⁰⁴⁵₀₁₂ + +# Quotation Marks +# +# Strings which contain misplaced quotation marks; can cause encoding errors + +' +" +'' +"" +'"' +"''''"'" +"'"'"''''" + +# Two-Byte Characters +# +# Strings which contain two-byte characters: can cause rendering issues or character-length issues + +田中さんにあげて下さい +パーティーへ行かないか +和製漢語 +部落格 +사회과학원 어학연구소 +찦차를 타고 온 펲시맨과 쑛다리 똠방각하 +社會科學院語學研究所 +울란바토르 +𠜎𠜱𠝹𠱓𠱸𠲖𠳏 + +# Japanese Emoticons +# +# Strings which consists of Japanese-style emoticons which are popular on the web + +ヽ༼ຈل͜ຈ༽ﾉ ヽ༼ຈل͜ຈ༽ﾉ +(｡◕ ∀ ◕｡) +｀ｨ(´∀｀∩ +__ﾛ(,_,*) +・(￣∀￣)・:*: +ﾟ･✿ヾ╲(｡◕‿◕｡)╱✿･ﾟ +,。・:*:・゜’( ☻ ω ☻ )。・:*:・゜’ +(╯°□°）╯︵ ┻━┻) +(ﾉಥ益ಥ）ﾉ ┻━┻ +( ͡° ͜ʖ ͡°) + +# Emoji +# +# Strings which contain Emoji; should be the same behavior as two-byte characters, but not always + +😍 +👩🏽 +👾 🙇 💁 🙅 🙆 🙋 🙎 🙍 +🐵 🙈 🙉 🙊 +❤️ 💔 💌 💕 💞 💓 💗 💖 💘 💝 💟 💜 💛 💚 💙 +✋🏿 💪🏿 👐🏿 🙌🏿 👏🏿 🙏🏿 +🚾 🆒 🆓 🆕 🆖 🆗 🆙 🏧 +0️⃣ 1️⃣ 2️⃣ 3️⃣ 4️⃣ 5️⃣ 6️⃣ 7️⃣ 8️⃣ 9️⃣ 🔟 + +# Unicode Numbers +# +# Strings which contain unicode numbers; if the code is localized, it should see the input as numeric + +１２３ +١٢٣ + +# Right-To-Left Strings +# +# Strings which contain text that should be rendered RTL if possible (e.g. Arabic, Hebrew) + +ثم نفس سقطت وبالتحديد،, جزيرتي باستخدام أن دنو. إذ هنا؟ الستار وتنصيب كان. أهّل ايطاليا، بريطانيا-فرنسا قد أخذ. سليمان، إتفاقية بين ما, يذكر الحدود أي بعد, معاملة بولندا، الإطلاق عل إيو. +בְּרֵאשִׁית, בָּרָא אֱלֹהִים, אֵת הַשָּׁמַיִם, וְאֵת הָאָרֶץ +הָיְתָהtestالصفحات التّحول +﷽ +ﷺ + +# Unicode Spaces +# +# Strings which contain unicode space characters with special properties (c.f. https://www.cs.tut.fi/~jkorpela/chars/spaces.html) + +​ +  +᠎ +　 + +␣ +␢ +␡ + +# Trick Unicode +# +# Strings which contain unicode with unusual properties (e.g. Right-to-left override) (c.f. http://www.unicode.org/charts/PDF/U2000.pdf) + +‪‪test‪ +‫test‫ +
test
 +test⁠test‫ +⁦test⁧ + +# Zalgo Text +# +# Strings which contain "corrupted" text. The corruption will not appear in non-HTML text, however. (via http://www.eeemo.net) + +Ṱ̺̺̕o͞ ̷i̲̬͇̪͙n̝̗͕v̟̜̘̦͟o̶̙̰̠kè͚̮̺̪̹̱̤ ̖t̝͕̳̣̻̪͞h̼͓̲̦̳̘̲e͇̣̰̦̬͎ ̢̼̻̱̘h͚͎͙̜̣̲ͅi̦̲̣̰̤v̻͍e̺̭̳̪̰-m̢iͅn̖̺̞̲̯̰d̵̼̟͙̩̼̘̳ ̞̥̱̳̭r̛̗̘e͙p͠r̼̞̻̭̗e̺̠̣͟s̘͇̳͍̝͉e͉̥̯̞̲͚̬͜ǹ̬͎͎̟̖͇̤t͍̬̤͓̼̭͘ͅi̪̱n͠g̴͉ ͏͉ͅc̬̟h͡a̫̻̯͘o̫̟̖͍̙̝͉s̗̦̲.̨̹͈̣ +̡͓̞ͅI̗̘̦͝n͇͇͙v̮̫ok̲̫̙͈i̖͙̭̹̠̞n̡̻̮̣̺g̲͈͙̭͙̬͎ ̰t͔̦h̞̲e̢̤ ͍̬̲͖f̴̘͕̣è͖ẹ̥̩l͖͔͚i͓͚̦͠n͖͍̗͓̳̮g͍ ̨o͚̪͡f̘̣̬ ̖̘͖̟͙̮c҉͔̫͖͓͇͖ͅh̵̤̣͚͔á̗̼͕ͅo̼̣̥s̱͈̺̖̦̻͢.̛̖̞̠̫̰ +̗̺͖̹̯͓Ṯ̤͍̥͇͈h̲́e͏͓̼̗̙̼̣͔ ͇̜̱̠͓͍ͅN͕͠e̗̱z̘̝̜̺͙p̤̺̹͍̯͚e̠̻̠͜r̨̤͍̺̖͔̖̖d̠̟̭̬̝͟i̦͖̩͓͔̤a̠̗̬͉̙n͚͜ ̻̞̰͚ͅh̵͉i̳̞v̢͇ḙ͎͟-҉̭̩̼͔m̤̭̫i͕͇̝̦n̗͙ḍ̟ ̯̲͕͞ǫ̟̯̰̲͙̻̝f ̪̰̰̗̖̭̘͘c̦͍̲̞͍̩̙ḥ͚a̮͎̟̙͜ơ̩̹͎s̤.̝̝ ҉Z̡̖̜͖̰̣͉̜a͖̰͙̬͡l̲̫̳͍̩g̡̟̼̱͚̞̬ͅo̗͜.̟ +̦H̬̤̗̤͝e͜ ̜̥̝̻͍̟́w̕h̖̯͓o̝͙̖͎̱̮ ҉̺̙̞̟͈W̷̼̭a̺̪͍į͈͕̭͙̯̜t̶̼̮s̘͙͖̕ ̠̫̠B̻͍͙͉̳ͅe̵h̵̬͇̫͙i̹͓̳̳̮͎̫̕n͟d̴̪̜̖ ̰͉̩͇͙̲͞ͅT͖̼͓̪͢h͏͓̮̻e̬̝̟ͅ ̤̹̝W͙̞̝͔͇͝ͅa͏͓͔̹̼̣l̴͔̰̤̟͔ḽ̫.͕ +Z̮̞̠͙͔ͅḀ̗̞͈̻̗Ḷ͙͎̯̹̞͓G̻O̭̗̮ + +# Unicode Upsidedown +# +# Strings which contain unicode with an "upsidedown" effect (via http://www.upsidedowntext.com) + +˙ɐnbᴉlɐ ɐuƃɐɯ ǝɹolop ʇǝ ǝɹoqɐl ʇn ʇunpᴉpᴉɔuᴉ ɹodɯǝʇ poɯsnᴉǝ op pǝs 'ʇᴉlǝ ƃuᴉɔsᴉdᴉpɐ ɹnʇǝʇɔǝsuoɔ 'ʇǝɯɐ ʇᴉs ɹolop ɯnsdᴉ ɯǝɹo˥ +00˙Ɩ$- + +# Unicode font +# +# Strings which contain bold/italic/etc. versions of normal characters + +Ｔｈｅ ｑｕｉｃｋ ｂｒｏｗｎ ｆｏｘ ｊｕｍｐｓ ｏｖｅｒ ｔｈｅ ｌａｚｙ ｄｏｇ +𝐓𝐡𝐞 𝐪𝐮𝐢𝐜𝐤 𝐛𝐫𝐨𝐰𝐧 𝐟𝐨𝐱 𝐣𝐮𝐦𝐩𝐬 𝐨𝐯𝐞𝐫 𝐭𝐡𝐞 𝐥𝐚𝐳𝐲 𝐝𝐨𝐠 +𝕿𝖍𝖊 𝖖𝖚𝖎𝖈𝖐 𝖇𝖗𝖔𝖜𝖓 𝖋𝖔𝖝 𝖏𝖚𝖒𝖕𝖘 𝖔𝖛𝖊𝖗 𝖙𝖍𝖊 𝖑𝖆𝖟𝖞 𝖉𝖔𝖌 +𝑻𝒉𝒆 𝒒𝒖𝒊𝒄𝒌 𝒃𝒓𝒐𝒘𝒏 𝒇𝒐𝒙 𝒋𝒖𝒎𝒑𝒔 𝒐𝒗𝒆𝒓 𝒕𝒉𝒆 𝒍𝒂𝒛𝒚 𝒅𝒐𝒈 +𝓣𝓱𝓮 𝓺𝓾𝓲𝓬𝓴 𝓫𝓻𝓸𝔀𝓷 𝓯𝓸𝔁 𝓳𝓾𝓶𝓹𝓼 𝓸𝓿𝓮𝓻 𝓽𝓱𝓮 𝓵𝓪𝔃𝔂 𝓭𝓸𝓰 +𝕋𝕙𝕖 𝕢𝕦𝕚𝕔𝕜 𝕓𝕣𝕠𝕨𝕟 𝕗𝕠𝕩 𝕛𝕦𝕞𝕡𝕤 𝕠𝕧𝕖𝕣 𝕥𝕙𝕖 𝕝𝕒𝕫𝕪 𝕕𝕠𝕘 +𝚃𝚑𝚎 𝚚𝚞𝚒𝚌𝚔 𝚋𝚛𝚘𝚠𝚗 𝚏𝚘𝚡 𝚓𝚞𝚖𝚙𝚜 𝚘𝚟𝚎𝚛 𝚝𝚑𝚎 𝚕𝚊𝚣𝚢 𝚍𝚘𝚐 +⒯⒣⒠ ⒬⒰⒤⒞⒦ ⒝⒭⒪⒲⒩ ⒡⒪⒳ ⒥⒰⒨⒫⒮ ⒪⒱⒠⒭ ⒯⒣⒠ ⒧⒜⒵⒴ ⒟⒪⒢ + +# Script Injection +# +# Strings which attempt to invoke a benign script injection; shows vulnerability to XSS + + +<script>alert('123');</script> + + +"> +'> +> + +< / script >< script >alert(123)< / script > + onfocus=JaVaSCript:alert(123) autofocus +" onfocus=JaVaSCript:alert(123) autofocus +' onfocus=JaVaSCript:alert(123) autofocus +＜script＞alert(123)＜/script＞ +ript>alert(123)ript> +--> +";alert(123);t=" +';alert(123);t=' +JavaSCript:alert(123) +;alert(123); +src=JaVaSCript:prompt(132) +">javascript:alert(1); +javascript:alert(1); +javascript:alert(1); +javascript:alert(1); +javascript:alert(1); +javascript:alert(1); +javascript:alert(1); +'`"><\x3Cscript>javascript:alert(1) +'`"><\x00script>javascript:alert(1) +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +`"'> +`"'> +`"'> +`"'> +`"'> +`"'> +`"'> +`"'> +`"'> +`"'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +XXX + + + +<a href=http://foo.bar/#x=`y></a><img alt="`><img src=x:x onerror=javascript:alert(1)></a>"> +<!--[if]><script>javascript:alert(1)</script --> +<!--[if<img src=x onerror=javascript:alert(1)//]> --> +<script src="/\%(jscript)s"></script> +<script src="\\%(jscript)s"></script> +<IMG """><SCRIPT>alert("XSS")</SCRIPT>"> +<IMG SRC=javascript:alert(String.fromCharCode(88,83,83))> +<IMG SRC=# onmouseover="alert('xxs')"> +<IMG SRC= onmouseover="alert('xxs')"> +<IMG onmouseover="alert('xxs')"> +<IMG SRC=javascript:alert('XSS')> +<IMG SRC=javascript:alert('XSS')> +<IMG SRC=javascript:alert('XSS')> +<IMG SRC="jav ascript:alert('XSS');"> +<IMG SRC="jav ascript:alert('XSS');"> +<IMG SRC="jav ascript:alert('XSS');"> +<IMG SRC="jav ascript:alert('XSS');"> +perl -e 'print "<IMG SRC=java\0script:alert(\"XSS\")>";' > out +<IMG SRC="  javascript:alert('XSS');"> +<SCRIPT/XSS SRC="http://ha.ckers.org/xss.js"></SCRIPT> +<BODY onload!#$%&()*~+-_.,:;?@[/|\]^`=alert("XSS")> +<SCRIPT/SRC="http://ha.ckers.org/xss.js"></SCRIPT> +<<SCRIPT>alert("XSS");//<</SCRIPT> +<SCRIPT SRC=http://ha.ckers.org/xss.js?< B > +<SCRIPT SRC=//ha.ckers.org/.j> +<IMG SRC="javascript:alert('XSS')" +<iframe src=http://ha.ckers.org/scriptlet.html < +\";alert('XSS');// +<plaintext> + +# SQL Injection +# +# Strings which can cause a SQL injection if inputs are not sanitized + +1;DROP TABLE users +1'; DROP TABLE users-- 1 +' OR 1=1 -- 1 +' OR '1'='1 + +# Server Code Injection +# +# Strings which can cause user to run code on server as a privileged user (c.f. https://news.ycombinator.com/item?id=7665153) + +- +-- +--version +--help +$USER +/dev/null; touch /tmp/blns.fail ; echo +`touch /tmp/blns.fail` +$(touch /tmp/blns.fail) +@{[system "touch /tmp/blns.fail"]} + +# Command Injection (Ruby) +# +# Strings which can call system commands within Ruby/Rails applications + +eval("puts 'hello world'") +System("ls -al /") +`ls -al /` +Kernel.exec("ls -al /") +Kernel.exit(1) +%x('ls -al /') + +# XXE Injection (XML) +# +# String which can reveal system files when parsed by a badly configured XML parser + +<?xml version="1.0" encoding="ISO-8859-1"?><!DOCTYPE foo [ <!ELEMENT foo ANY ><!ENTITY xxe SYSTEM "file:///etc/passwd" >]><foo>&xxe;</foo> + +# Unwanted Interpolation +# +# Strings which can be accidentally expanded into different strings if evaluated in the wrong context, e.g. used as a printf format string or via Perl or shell eval. Might expose sensitive data from the program doing the interpolation, or might just represent the wrong string. + +$HOME +$ENV{'HOME'} +%d +%s +%*.*s + +# File Inclusion +# +# Strings which can cause user to pull in files that should not be a part of a web server + +../../../../../../../../../../../etc/passwd%00 +../../../../../../../../../../../etc/hosts + +# Known CVEs and Vulnerabilities +# +# Strings that test for known vulnerabilities + +() { 0; }; touch /tmp/blns.shellshock1.fail; +() { _; } >_[$($())] { touch /tmp/blns.shellshock2.fail; } + +# MSDOS/Windows Special Filenames +# +# Strings which are reserved characters in MSDOS/Windows + +CON +PRN +AUX +CLOCK$ +NUL +A: +ZZ: +COM1 +LPT1 +LPT2 +LPT3 +COM2 +COM3 +COM4 + +# Scunthorpe Problem +# +# Innocuous strings which may be blocked by profanity filters (https://en.wikipedia.org/wiki/Scunthorpe_problem) + +Scunthorpe General Hospital +Penistone Community Church +Lightwater Country Park +Jimmy Clitheroe +Horniman Museum +shitake mushrooms +RomansInSussex.co.uk +http://www.cum.qc.ca/ +Craig Cockburn, Software Specialist +Linda Callahan +Dr. Herman I. Libshitz +magna cum laude +Super Bowl XXX +medieval erection of parapets +evaluate +mocha +expression +Arsenal canal +classic +Tyson Gay + +# Human injection +# +# Strings which may cause human to reinterpret worldview + +If you're reading this, you've been in a coma for almost 20 years now. We're trying a new technique. We don't know where this message will end up in your dream, but we hope it works. Please wake up, we miss you. + +# Terminal escape codes +# +# Strings which punish the fools who use cat/type on this file + +Roses are red, violets are blue. Hope you enjoy terminal hue +But now...for my greatest trick... +The quick brown fox... [Beeeep] + +# iOS Vulnerability +# +# Strings which crashed iMessage in iOS versions 8.3 and earlier + +Powerلُلُصّبُلُلصّبُررً ॣ ॣh ॣ ॣ冗 + + +\end{document} diff --git a/services/clsi/test/unit/coffee/CompileManagerTests.coffee b/services/clsi/test/unit/coffee/CompileManagerTests.coffee index d6678b9407..87f424b0f9 100644 --- a/services/clsi/test/unit/coffee/CompileManagerTests.coffee +++ b/services/clsi/test/unit/coffee/CompileManagerTests.coffee @@ -191,7 +191,7 @@ describe "CompileManager", -> it "should run the texcount command", -> @directory = "#{@Settings.path.compilesDir}/#{@project_id}" @file_path = "$COMPILE_DIR/#{@file_name}" - @command =[ "texcount", '-inc', @file_path, "-out=" + @file_path + ".wc"] + @command =[ "texcount", "-inc", @file_path, "-out=" + @file_path + ".wc"] @CommandRunner.run .calledWith(@project_id, @command, @directory, @timeout)