From f067c039fef59438a80f4265582d709ac89439b4 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Thu, 9 Mar 2017 18:51:54 +0000 Subject: [PATCH] always search for inS3 when getting a doc --- services/docstore/app/coffee/DocManager.coffee | 4 +++- .../test/acceptance/coffee/ArchiveDocsTests.coffee | 6 +++--- .../test/unit/coffee/DocManagerTests.coffee | 14 ++++++++++++++ 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/services/docstore/app/coffee/DocManager.coffee b/services/docstore/app/coffee/DocManager.coffee index 6cf679c6c1..875184ca4e 100644 --- a/services/docstore/app/coffee/DocManager.coffee +++ b/services/docstore/app/coffee/DocManager.coffee @@ -10,7 +10,9 @@ module.exports = DocManager = # collection (which is all that this collection contains). In future, we should # migrate this version property to be part of the docs collection, to guarantee # consitency between lines and version when writing/reading, and for a simpler schema. - getDoc: (project_id, doc_id, filter = { version: false }, callback = (error, doc) ->) -> + getDoc: (project_id, doc_id, filter = { version: false, inS3:true}, callback = (error, doc) ->) -> + if filter? and !filter.inS3? + filter.inS3 = true MongoManager.findDoc project_id, doc_id, filter, (err, doc)-> if err? return callback(err) diff --git a/services/docstore/test/acceptance/coffee/ArchiveDocsTests.coffee b/services/docstore/test/acceptance/coffee/ArchiveDocsTests.coffee index e8e25e3ab2..a7616fcd86 100644 --- a/services/docstore/test/acceptance/coffee/ArchiveDocsTests.coffee +++ b/services/docstore/test/acceptance/coffee/ArchiveDocsTests.coffee @@ -12,6 +12,7 @@ describe "Archiving", -> beforeEach (done) -> @callback = sinon.stub() @project_id = ObjectId() + @version = 42 @docs = [{ _id: ObjectId() lines: ["one", "two", "three"] @@ -23,9 +24,8 @@ describe "Archiving", -> }, { _id: ObjectId() lines: [ "", "undefined", "undef", "null", "NULL", "(null)", "nil", "NIL", "true", "false", "True", "False", "None", "\\", "\\\\", "0", "1", "1.00", "$1.00", "1/2", "1E2", "1E02", "1E+02", "-1", "-1.00", "-$1.00", "-1/2", "-1E2", "-1E02", "-1E+02", "1/0", "0/0", "-2147483648/-1", "-9223372036854775808/-1", "0.00", "0..0", ".", "0.0.0", "0,00", "0,,0", ",", "0,0,0", "0.0/0", "1.0/0.0", "0.0/0.0", "1,0/0,0", "0,0/0,0", "--1", "-", "-.", "-,", "999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999", "NaN", "Infinity", "-Infinity", "0x0", "0xffffffff", "0xffffffffffffffff", "0xabad1dea", "123456789012345678901234567890123456789", "1,000.00", "1 000.00", "1'000.00", "1,000,000.00", "1 000 000.00", "1'000'000.00", "1.000,00", "1 000,00", "1'000,00", "1.000.000,00", "1 000i̳̞v̢͇ḙ͎͟-҉̭̩̼͔m̤̭̫i͕͇̝̦n̗͙ḍ̟ ̯̲͕͞ǫ̟̯̰̲͙̻̝f ̪̰̰̗̖̭̘͘c̦͍̲̞͍̩̙ḥ͚a̮͎̟̙͜ơ̩̹͎s̤.̝̝ ҉Z̡̖̜͖̰̣͉̜a͖̰͙̬͡l̲̫̳͍̩g̡̟̼̱͚̞̬ͅo̗͜.̟", "̦H̬̤̗̤͝e͜ ̜̥̝̻͍̟́w̕h̖̯͓o̝͙̖͎̱̮ ҉̺̙̞̟͈W̷̼̭a̺̪͍į͈͕̭͙̯̜t̶̼̮s̘͙͖̕ ̠̫̠B̻͍͙͉̳ͅe̵h̵̬͇̫͙i̹͓̳̳̮͎̫̕n͟d̴̪̜̖ ̰͉̩͇͙̲͞ͅT͖̼͓̪͢h͏͓̮̻e̬̝̟ͅ ̤̹̝W͙̞̝͔͇͝ͅa͏͓͔̹̼̣l̴͔̰̤̟͔ḽ̫.͕", "Z̮̞̠͙͔ͅḀ̗̞͈̻̗Ḷ͙͎̯̹̞͓G̻O̭̗̮", "˙ɐnbᴉlɐ ɐuƃɐɯ ǝɹolop ʇǝ ǝɹoqɐl ʇn ʇunpᴉpᴉɔuᴉ ɹodɯǝʇ poɯsnᴉǝ op pǝs 'ʇᴉlǝ ƃuᴉɔsᴉdᴉpɐ ɹnʇǝʇɔǝsuoɔ 'ʇǝɯɐ ʇᴉs ɹolop ɯnsdᴉ ɯǝɹo˥", "00˙Ɩ$-", "Ｔｈｅ ｑｕｉｃｋ ｂｒｏｗｎ ｆｏｘ ｊｕｍｐｓ ｏｖｅｒ ｔｈｅ ｌａｚｙ ｄｏｇ", "𝐓𝐡𝐞 𝐪𝐮𝐢𝐜𝐤 𝐛𝐫𝐨𝐰𝐧 𝐟𝐨𝐱 𝐣𝐮𝐦𝐩𝐬 𝐨𝐯𝐞𝐫 𝐭𝐡𝐞 𝐥𝐚𝐳𝐲 𝐝𝐨𝐠", "𝕿𝖍𝖊 𝖖𝖚𝖎𝖈𝖐 𝖇𝖗𝖔𝖜𝖓 𝖋𝖔𝖝 𝖏𝖚𝖒𝖕𝖘 𝖔𝖛𝖊𝖗 𝖙𝖍𝖊 𝖑𝖆𝖟𝖞 𝖉𝖔𝖌", "𝑻𝒉𝒆 𝒒𝒖𝒊𝒄𝒌 𝒃𝒓𝒐𝒘𝒏 𝒇𝒐𝒙 𝒋𝒖𝒎𝒑𝒔 𝒐𝒗𝒆𝒓 𝒕𝒉𝒆 𝒍𝒂𝒛𝒚 𝒅𝒐𝒈", "𝓣𝓱𝓮 𝓺𝓾𝓲𝓬𝓴 𝓫𝓻𝓸𝔀𝓷 𝓯𝓸𝔁 𝓳𝓾𝓶𝓹𝓼 𝓸𝓿𝓮𝓻 𝓽𝓱𝓮 𝓵𝓪𝔃𝔂 𝓭𝓸𝓰", "𝕋𝕙𝕖 𝕢𝕦𝕚𝕔𝕜 𝕓𝕣𝕠𝕨𝕟 𝕗𝕠𝕩 𝕛𝕦𝕞𝕡𝕤 𝕠𝕧𝕖𝕣 𝕥𝕙𝕖 𝕝𝕒𝕫𝕪 𝕕𝕠𝕘", "𝚃𝚑𝚎 𝚚𝚞𝚒𝚌𝚔 𝚋𝚛𝚘𝚠𝚗 𝚏𝚘𝚡 𝚓𝚞𝚖𝚙𝚜 𝚘𝚟𝚎𝚛 𝚝𝚑𝚎 𝚕𝚊𝚣𝚢 𝚍𝚘𝚐", "⒯⒣⒠ ⒬⒰⒤⒞⒦ ⒝⒭⒪⒲⒩ ⒡⒪⒳ ⒥⒰⒨⒫⒮ ⒪⒱⒠⒭ ⒯⒣⒠ ⒧⒜⒵⒴ ⒟⒪⒢", "", "<script>alert('123');</script>", "", " ", "\">", "'>", ">", "", "< / script >< script >alert(123)< / script >", " onfocus=JaVaSCript:alert(123) autofocus ", "\" onfocus=JaVaSCript:alert(123) autofocus ", "' onfocus=JaVaSCript:alert(123) autofocus ", "＜script＞alert(123)＜/script＞", "ript>alert(123)ript>", "-->", "\";alert(123);t=\"", "';alert(123);t='", "JavaSCript:alert(123)", ";alert(123);", "src=JaVaSCript:prompt(132)", "\"><\\x3Cscript>javascript:alert(1) ", "'`\"><\\x00script>javascript:alert(1)", "ABC
DEF", "ABC
DEF", "ABC
DEF", "ABC
DEF", "ABC
DEF", "ABC
DEF", "ABC
DEF", "ABC
DEF", "ABC
DEF", "ABC
DEF", "ABC
DEF", "ABC
DEF", "ABC
DEF", "ABC
DEF", "ABC
DEF", "ABC
DEF", "ABC
DEF", "ABC
DEF", "ABC
DEF", "ABC
DEF", "ABC
DEF", "ABC
DEF", "ABC
DEF", "ABC
DEF", "ABC
DEF", "ABC
DEF", "ABC
DEF", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "`\"'>", "`\"'>", "`\"'>", "`\"'>", "`\"'>", "`\"'>", "`\"'>", "`\"'>", "`\"'>", "`\"'>", "\"`'>", "\"`'>", "\"`'>", "\"`'>", "\"`'>", "\"`'>", "\"`'>", "\"`'>", "\"`'>", "\"`'>", "\"`'>", "\"`'>", "\"`'>", "\"`'>", "\"`'>", "\"`'>", "\"`'>", "\"`'>", "\"`'>", "\"`'>", "\"`'>", "\"`'>", "\"`'>", "\"`'>", "\"`'>", "\"`'>", "\"`'>", "\"`'>", "\"`'>", "\"`'>", "\"`'>", "\"`'>", "\"`'>", "\"`'>", "\"`'>", "\"`'>", "\"`'>", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "XXX", "javascript:alert(1)\"` `>", "", "", "<a href=http://foo.bar/#x=`y></a><img alt=\"`><img src=x:x onerror=javascript:alert(1)></a>\">", "<!--[if]><script>javascript:alert(1)</script -->", "<!--[if<img src=x onerror=javascript:alert(1)//]> -->", "<script src=\"/\\%(jscript)s\"></script>", "<script src=\"\\\\%(jscript)s\"></script>", "<IMG \"\"\"><SCRIPT>alert(\"XSS\")</SCRIPT>\">", "<IMG SRC=javascript:alert(String.fromCharCode(88,83,83))>", "<IMG SRC=# onmouseover=\"alert('xxs')\">", "<IMG SRC= onmouseover=\"alert('xxs')\">", "<IMG onmouseover=\"alert('xxs')\">", "<IMG SRC=javascript:alert('XSS')>", "<IMG SRC=javascript:alert('XSS')>", "<IMG SRC=javascript:alert('XSS')>", "<IMG SRC=\"jav ascript:alert('XSS');\">", "<IMG SRC=\"jav ascript:alert('XSS');\">", "<IMG SRC=\"jav ascript:alert('XSS');\">", "<IMG SRC=\"jav ascript:alert('XSS');\">", "perl -e 'print \"<IMG SRC=java\\0script:alert(\\\"XSS\\\")>\";' > out", "<IMG SRC=\"  javascript:alert('XSS');\">", "<SCRIPT/XSS SRC=\"http://ha.ckers.org/xss.js\"></SCRIPT>", "<BODY onload!#$%&()*~+-_.,:;?@[/|\\]^`=alert(\"XSS\")>", "<SCRIPT/SRC=\"http://ha.ckers.org/xss.js\"></SCRIPT>", "<<SCRIPT>alert(\"XSS\");//<</SCRIPT>", "<SCRIPT SRC=http://ha.ckers.org/xss.js?< B >", "<SCRIPT SRC=//ha.ckers.org/.j>", "<IMG SRC=\"javascript:alert('XSS')\"", "<iframe src=http://ha.ckers.org/scriptlet.html <", "\\\";alert('XSS');//", "<plaintext>", "1;DROP TABLE users", "1'; DROP TABLE users-- 1", "' OR 1=1 -- 1", "' OR '1'='1", "-", "--", "--version", "--help", "$USER", "/dev/null; touch /tmp/blns.fail ; echo", "`touch /tmp/blns.fail`", "$(touch /tmp/blns.fail)", "@{[system \"touch /tmp/blns.fail\"]}", "eval(\"puts 'hello world'\")", "System(\"ls -al /\")", "`ls -al /`", "Kernel.exec(\"ls -al /\")", "Kernel.exit(1)", "%x('ls -al /')", "<?xml version=\"1.0\" encoding=\"ISO-8859-1\"?><!DOCTYPE foo [ <!ELEMENT foo ANY ><!ENTITY xxe SYSTEM \"file:///etc/passwd\" >]><foo>&xxe;</foo>", "$HOME", "$ENV{'HOME'}", "%d", "%s", "%*.*s", "../../../../../../../../../../../etc/passwd%00", "../../../../../../../../../../../etc/hosts", "() { 0; }; touch /tmp/blns.shellshock1.fail;", "() { _; } >_[$($())] { touch /tmp/blns.shellshock2.fail; }", "CON", "PRN", "AUX", "CLOCK$", "NUL", "A:", "ZZ:", "COM1", "LPT1", "LPT2", "LPT3", "COM2", "COM3", "COM4", "Scunthorpe General Hospital", "Penistone Community Church", "Lightwater Country Park", "Jimmy Clitheroe", "Horniman Museum", "shitake mushrooms", "RomansInSussex.co.uk", "http://www.cum.qc.ca/", "Craig Cockburn, Software Specialist", "Linda Callahan", "Dr. Herman I. Libshitz", "magna cum laude", "Super Bowl XXX", "medieval erection of parapets", "evaluate", "mocha", "expression", "Arsenal canal", "classic", "Tyson Gay", "If you're reading this, you've been in a coma for almost 20 years now. We're trying a new technique. We don't know where this message will end up in your dream, but we hope it works. Please wake up, we miss you.", "Roses are \u001b[0;31mred\u001b[0m, violets are \u001b[0;34mblue. Hope you enjoy terminal hue", "But now...\u001b[20Cfor my greatest trick...\u001b[8m", "The quic\b\b\b\b\b\bk brown fo\u0007\u0007\u0007\u0007\u0007\u0007\u0007\u0007\u0007\u0007\u0007x... [Beeeep]", "Powerلُلُصّبُلُلصّبُررً ॣ ॣh ॣ ॣ冗" ] - rev: 6 + version:@version }] - @version = 42 @ranges = [] jobs = for doc in @docs do (doc) => @@ -157,7 +157,7 @@ describe "Archiving", -> DocstoreClient.getDoc @project_id, @docs[0]._id, {}, (error, res, doc) => doc.lines.should.deep.equal @docs[0].lines doc.version.should.equal @docs[0].version - doc.ranges.should.deep.equal @docs[0].ranges + doc.ranges.should.deep.equal @ranges done() diff --git a/services/docstore/test/unit/coffee/DocManagerTests.coffee b/services/docstore/test/unit/coffee/DocManagerTests.coffee index 9b43bc05af..c07e6a4e4a 100644 --- a/services/docstore/test/unit/coffee/DocManagerTests.coffee +++ b/services/docstore/test/unit/coffee/DocManagerTests.coffee @@ -34,6 +34,20 @@ describe "DocManager", -> @MongoManager.findDoc = sinon.stub() @MongoManager.getDocVersion = sinon.stub().yields(null, @version) + describe "when using a filter", -> + beforeEach -> + @MongoManager.findDoc.yields(null, @doc) + + it "should always get inS3 even when filter is passed", (done)-> + @DocManager.getDoc @project_id, @doc_id, {version: true}, => + @MongoManager.findDoc.args[0][2].inS3.should.equal true + done() + + it "should always get inS3 even when no filter is passed", (done)-> + @DocManager.getDoc @project_id, @doc_id, undefined, => + @MongoManager.findDoc.args[0][2].inS3.should.equal true + done() + describe "when the doc is in the doc collection", -> beforeEach -> @MongoManager.findDoc.yields(null, @doc)