github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2024-11-21 20:47:08 -05:00
Code Issues Projects Releases Packages Wiki Activity

merge multiple repositories into an existing monorepo

Browse source 
- merged using: 'monorepo_add.sh services-clsi:services/clsi'
- see https://github.com/shopsys/monorepo-tools
This commit is contained in:
Jakob Ackermann 2021-08-05 08:32:38 +01:00
commit de77f1ce67
No known key found for this signature in database
GPG key ID: 30C56800FCA3828A
221 changed files with 46318 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
services/clsi/.dockerignore Normal file
View file

@ -0,0 +1,11 @@
node_modules/*
gitrev
.git
.gitignore
.npm
.nvmrc
nodemon.json
cache/
compiles/
db/
output/

86
services/clsi/.eslintrc Normal file
View file

@ -0,0 +1,86 @@
// this file was auto-generated, do not edit it directly.
// instead run bin/update_build_scripts from
// https://github.com/sharelatex/sharelatex-dev-environment
{
"extends": [
"eslint:recommended",
"standard",
"prettier"
],
"parserOptions": {
"ecmaVersion": 2018
},
"plugins": [
"mocha",
"chai-expect",
"chai-friendly"
],
"env": {
"node": true,
"mocha": true
},
"rules": {
// TODO(das7pad): remove overrides after fixing all the violations manually (https://github.com/overleaf/issues/issues/3882#issuecomment-878999671)
// START of temporary overrides
"array-callback-return": "off",
"no-dupe-else-if": "off",
"no-var": "off",
"no-empty": "off",
"node/handle-callback-err": "off",
"no-loss-of-precision": "off",
"node/no-callback-literal": "off",
"node/no-path-concat": "off",
"prefer-regex-literals": "off",
// END of temporary overrides
// Swap the no-unused-expressions rule with a more chai-friendly one
"no-unused-expressions": 0,
"chai-friendly/no-unused-expressions": "error",
// Do not allow importing of implicit dependencies.
"import/no-extraneous-dependencies": "error"
},
"overrides": [
{
// Test specific rules
"files": ["test/**/*.js"],
"globals": {
"expect": true
},
"rules": {
// mocha-specific rules
"mocha/handle-done-callback": "error",
"mocha/no-exclusive-tests": "error",
"mocha/no-global-tests": "error",
"mocha/no-identical-title": "error",
"mocha/no-nested-tests": "error",
"mocha/no-pending-tests": "error",
"mocha/no-skipped-tests": "error",
"mocha/no-mocha-arrows": "error",
// chai-specific rules
"chai-expect/missing-assertion": "error",
"chai-expect/terminating-properties": "error",
// prefer-arrow-callback applies to all callbacks, not just ones in mocha tests.
// we don't enforce this at the top-level - just in tests to manage `this` scope
// based on mocha's context mechanism
"mocha/prefer-arrow-callback": "error"
}
},
{
// Backend specific rules
"files": ["app/**/*.js", "app.js", "index.js"],
"rules": {
// don't allow console.log in backend code
"no-console": "error",
// Do not allow importing of implicit dependencies.
"import/no-extraneous-dependencies": ["error", {
// Do not allow importing of devDependencies.
"devDependencies": false
}]
}
}
]
}

38
services/clsi/.github/ISSUE_TEMPLATE.md vendored Normal file
View file

@ -0,0 +1,38 @@
<!-- BUG REPORT TEMPLATE -->
## Steps to Reproduce
<!-- Describe the steps leading up to when / where you found the bug. -->
<!-- Screenshots may be helpful here. -->
1.
2.
3.
## Expected Behaviour
<!-- What should have happened when you completed the steps above? -->
## Observed Behaviour
<!-- What actually happened when you completed the steps above? -->
<!-- Screenshots may be helpful here. -->
## Context
<!-- How has this issue affected you? What were you trying to accomplish? -->
## Technical Info
<!-- Provide any technical details that may be applicable (or N/A if not applicable). -->
* URL:
* Browser Name and version:
* Operating System and version (desktop or mobile):
* Signed in as:
* Project and/or file:
## Analysis
<!--- Optionally, document investigation of / suggest a fix for the bug, e.g. 'comes from this line / commit' -->
## Who Needs to Know?
<!-- If you want to bring this to the attention of particular people, @-mention them below. -->
<!-- If a user reported this bug and should be notified when it is fixed, provide the Front conversation link. -->
-
-

48
services/clsi/.github/PULL_REQUEST_TEMPLATE.md vendored Normal file
View file

@ -0,0 +1,48 @@
<!-- ** This is an Overleaf public repository ** -->
<!-- Please review https://github.com/overleaf/overleaf/blob/master/CONTRIBUTING.md for guidance on what is expected of a contribution. -->
### Description
#### Screenshots
#### Related Issues / PRs
### Review
#### Potential Impact
#### Manual Testing Performed
- [ ]
- [ ]
#### Accessibility
### Deployment
#### Deployment Checklist
- [ ] Update documentation not included in the PR (if any)
- [ ]
#### Metrics and Monitoring
#### Who Needs to Know?

23
services/clsi/.github/dependabot.yml vendored Normal file
View file

@ -0,0 +1,23 @@
version: 2
updates:
- package-ecosystem: "npm"
directory: "/"
schedule:
interval: "daily"
pull-request-branch-name:
# Separate sections of the branch name with a hyphen
# Docker images use the branch name and do not support slashes in tags
# https://github.com/overleaf/google-ops/issues/822
# https://docs.github.com/en/github/administering-a-repository/configuration-options-for-dependency-updates#pull-request-branch-nameseparator
separator: "-"
# Block informal upgrades -- security upgrades use a separate queue.
# https://docs.github.com/en/github/administering-a-repository/configuration-options-for-dependency-updates#open-pull-requests-limit
open-pull-requests-limit: 0
# currently assign team-magma to all dependabot PRs - this may change in
# future if we reorganise teams
labels:
- "dependencies"
- "type:maintenance"

17
services/clsi/.gitignore vendored Normal file
View file

@ -0,0 +1,17 @@
**.swp
node_modules
test/acceptance/fixtures/tmp
compiles
output
.DS_Store
*~
cache
.vagrant
db.sqlite
db.sqlite-wal
db.sqlite-shm
config/*
npm-debug.log
# managed by dev-environment$ bin/update_build_scripts
.npmrc

3
services/clsi/.mocharc.json Normal file
View file

@ -0,0 +1,3 @@
{
"require": "test/setup.js"
}

1
services/clsi/.nvmrc Normal file
View file

@ -0,0 +1 @@
12.22.3

11
services/clsi/.prettierrc Normal file
View file

@ -0,0 +1,11 @@
# This file was auto-generated, do not edit it directly.
# Instead run bin/update_build_scripts from
# https://github.com/sharelatex/sharelatex-dev-environment
{
"arrowParens": "avoid",
"semi": false,
"singleQuote": true,
"trailingComma": "es5",
"tabWidth": 2,
"useTabs": false
}

35
services/clsi/.viminfo Normal file
View file

@ -0,0 +1,35 @@
# This viminfo file was generated by Vim 7.4.
# You may edit it if you're careful!
# Value of 'encoding' when this file was written
*encoding=latin1
# hlsearch on (H) or off (h):
~h
# Command Line History (newest to oldest):
:x
# Search String History (newest to oldest):
# Expression History (newest to oldest):
# Input Line History (newest to oldest):
# Input Line History (newest to oldest):
# Registers:
# File marks:
'0 1 0 ~/hello
# Jumplist (newest first):
-' 1 0 ~/hello
# History of marks within files (newest to oldest):
> ~/hello
" 1 0
^ 1 1
. 1 0
+ 1 0

28
services/clsi/Dockerfile Normal file
View file

@ -0,0 +1,28 @@
# This file was auto-generated, do not edit it directly.
# Instead run bin/update_build_scripts from
# https://github.com/sharelatex/sharelatex-dev-environment
FROM node:12.22.3 as base
WORKDIR /app
COPY install_deps.sh /app
RUN chmod 0755 ./install_deps.sh && ./install_deps.sh
ENTRYPOINT ["/bin/sh", "entrypoint.sh"]
COPY entrypoint.sh /app
FROM base as app
#wildcard as some files may not be in all repos
COPY package*.json npm-shrink*.json /app/
RUN npm ci --quiet
COPY . /app
FROM base
COPY --from=app /app /app
RUN mkdir -p cache compiles db output \
&& chown node:node cache compiles db output
CMD ["node", "--expose-gc", "app.js"]

661
services/clsi/LICENSE Normal file
View file

@ -0,0 +1,661 @@
GNU AFFERO GENERAL PUBLIC LICENSE
Version 3, 19 November 2007
Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
Everyone is permitted to copy and distribute verbatim copies
of this license document, but changing it is not allowed.
Preamble
The GNU Affero General Public License is a free, copyleft license for
software and other kinds of works, specifically designed to ensure
cooperation with the community in the case of network server software.
The licenses for most software and other practical works are designed
to take away your freedom to share and change the works. By contrast,
our General Public Licenses are intended to guarantee your freedom to
share and change all versions of a program--to make sure it remains free
software for all its users.
When we speak of free software, we are referring to freedom, not
price. Our General Public Licenses are designed to make sure that you
have the freedom to distribute copies of free software (and charge for
them if you wish), that you receive source code or can get it if you
want it, that you can change the software or use pieces of it in new
free programs, and that you know you can do these things.
Developers that use our General Public Licenses protect your rights
with two steps: (1) assert copyright on the software, and (2) offer
you this License which gives you legal permission to copy, distribute
and/or modify the software.
A secondary benefit of defending all users' freedom is that
improvements made in alternate versions of the program, if they
receive widespread use, become available for other developers to
incorporate. Many developers of free software are heartened and
encouraged by the resulting cooperation. However, in the case of
software used on network servers, this result may fail to come about.
The GNU General Public License permits making a modified version and
letting the public access it on a server without ever releasing its
source code to the public.
The GNU Affero General Public License is designed specifically to
ensure that, in such cases, the modified source code becomes available
to the community. It requires the operator of a network server to
provide the source code of the modified version running there to the
users of that server. Therefore, public use of a modified version, on
a publicly accessible server, gives the public access to the source
code of the modified version.
An older license, called the Affero General Public License and
published by Affero, was designed to accomplish similar goals. This is
a different license, not a version of the Affero GPL, but Affero has
released a new version of the Affero GPL which permits relicensing under
this license.
The precise terms and conditions for copying, distribution and
modification follow.
TERMS AND CONDITIONS
0. Definitions.
"This License" refers to version 3 of the GNU Affero General Public License.
"Copyright" also means copyright-like laws that apply to other kinds of
works, such as semiconductor masks.
"The Program" refers to any copyrightable work licensed under this
License. Each licensee is addressed as "you". "Licensees" and
"recipients" may be individuals or organizations.
To "modify" a work means to copy from or adapt all or part of the work
in a fashion requiring copyright permission, other than the making of an
exact copy. The resulting work is called a "modified version" of the
earlier work or a work "based on" the earlier work.
A "covered work" means either the unmodified Program or a work based
on the Program.
To "propagate" a work means to do anything with it that, without
permission, would make you directly or secondarily liable for
infringement under applicable copyright law, except executing it on a
computer or modifying a private copy. Propagation includes copying,
distribution (with or without modification), making available to the
public, and in some countries other activities as well.
To "convey" a work means any kind of propagation that enables other
parties to make or receive copies. Mere interaction with a user through
a computer network, with no transfer of a copy, is not conveying.
An interactive user interface displays "Appropriate Legal Notices"
to the extent that it includes a convenient and prominently visible
feature that (1) displays an appropriate copyright notice, and (2)
tells the user that there is no warranty for the work (except to the
extent that warranties are provided), that licensees may convey the
work under this License, and how to view a copy of this License. If
the interface presents a list of user commands or options, such as a
menu, a prominent item in the list meets this criterion.
1. Source Code.
The "source code" for a work means the preferred form of the work
for making modifications to it. "Object code" means any non-source
form of a work.
A "Standard Interface" means an interface that either is an official
standard defined by a recognized standards body, or, in the case of
interfaces specified for a particular programming language, one that
is widely used among developers working in that language.
The "System Libraries" of an executable work include anything, other
than the work as a whole, that (a) is included in the normal form of
packaging a Major Component, but which is not part of that Major
Component, and (b) serves only to enable use of the work with that
Major Component, or to implement a Standard Interface for which an
implementation is available to the public in source code form. A
"Major Component", in this context, means a major essential component
(kernel, window system, and so on) of the specific operating system
(if any) on which the executable work runs, or a compiler used to
produce the work, or an object code interpreter used to run it.
The "Corresponding Source" for a work in object code form means all
the source code needed to generate, install, and (for an executable
work) run the object code and to modify the work, including scripts to
control those activities. However, it does not include the work's
System Libraries, or general-purpose tools or generally available free
programs which are used unmodified in performing those activities but
which are not part of the work. For example, Corresponding Source
includes interface definition files associated with source files for
the work, and the source code for shared libraries and dynamically
linked subprograms that the work is specifically designed to require,
such as by intimate data communication or control flow between those
subprograms and other parts of the work.
The Corresponding Source need not include anything that users
can regenerate automatically from other parts of the Corresponding
Source.
The Corresponding Source for a work in source code form is that
same work.
2. Basic Permissions.
All rights granted under this License are granted for the term of
copyright on the Program, and are irrevocable provided the stated
conditions are met. This License explicitly affirms your unlimited
permission to run the unmodified Program. The output from running a
covered work is covered by this License only if the output, given its
content, constitutes a covered work. This License acknowledges your
rights of fair use or other equivalent, as provided by copyright law.
You may make, run and propagate covered works that you do not
convey, without conditions so long as your license otherwise remains
in force. You may convey covered works to others for the sole purpose
of having them make modifications exclusively for you, or provide you
with facilities for running those works, provided that you comply with
the terms of this License in conveying all material for which you do
not control copyright. Those thus making or running the covered works
for you must do so exclusively on your behalf, under your direction
and control, on terms that prohibit them from making any copies of
your copyrighted material outside their relationship with you.
Conveying under any other circumstances is permitted solely under
the conditions stated below. Sublicensing is not allowed; section 10
makes it unnecessary.
3. Protecting Users' Legal Rights From Anti-Circumvention Law.
No covered work shall be deemed part of an effective technological
measure under any applicable law fulfilling obligations under article
11 of the WIPO copyright treaty adopted on 20 December 1996, or
similar laws prohibiting or restricting circumvention of such
measures.
When you convey a covered work, you waive any legal power to forbid
circumvention of technological measures to the extent such circumvention
is effected by exercising rights under this License with respect to
the covered work, and you disclaim any intention to limit operation or
modification of the work as a means of enforcing, against the work's
users, your or third parties' legal rights to forbid circumvention of
technological measures.
4. Conveying Verbatim Copies.
You may convey verbatim copies of the Program's source code as you
receive it, in any medium, provided that you conspicuously and
appropriately publish on each copy an appropriate copyright notice;
keep intact all notices stating that this License and any
non-permissive terms added in accord with section 7 apply to the code;
keep intact all notices of the absence of any warranty; and give all
recipients a copy of this License along with the Program.
You may charge any price or no price for each copy that you convey,
and you may offer support or warranty protection for a fee.
5. Conveying Modified Source Versions.
You may convey a work based on the Program, or the modifications to
produce it from the Program, in the form of source code under the
terms of section 4, provided that you also meet all of these conditions:
a) The work must carry prominent notices stating that you modified
it, and giving a relevant date.
b) The work must carry prominent notices stating that it is
released under this License and any conditions added under section
7. This requirement modifies the requirement in section 4 to
"keep intact all notices".
c) You must license the entire work, as a whole, under this
License to anyone who comes into possession of a copy. This
License will therefore apply, along with any applicable section 7
additional terms, to the whole of the work, and all its parts,
regardless of how they are packaged. This License gives no
permission to license the work in any other way, but it does not
invalidate such permission if you have separately received it.
d) If the work has interactive user interfaces, each must display
Appropriate Legal Notices; however, if the Program has interactive
interfaces that do not display Appropriate Legal Notices, your
work need not make them do so.
A compilation of a covered work with other separate and independent
works, which are not by their nature extensions of the covered work,
and which are not combined with it such as to form a larger program,
in or on a volume of a storage or distribution medium, is called an
"aggregate" if the compilation and its resulting copyright are not
used to limit the access or legal rights of the compilation's users
beyond what the individual works permit. Inclusion of a covered work
in an aggregate does not cause this License to apply to the other
parts of the aggregate.
6. Conveying Non-Source Forms.
You may convey a covered work in object code form under the terms
of sections 4 and 5, provided that you also convey the
machine-readable Corresponding Source under the terms of this License,
in one of these ways:
a) Convey the object code in, or embodied in, a physical product
(including a physical distribution medium), accompanied by the
Corresponding Source fixed on a durable physical medium
customarily used for software interchange.
b) Convey the object code in, or embodied in, a physical product
(including a physical distribution medium), accompanied by a
written offer, valid for at least three years and valid for as
long as you offer spare parts or customer support for that product
model, to give anyone who possesses the object code either (1) a
copy of the Corresponding Source for all the software in the
product that is covered by this License, on a durable physical
medium customarily used for software interchange, for a price no
more than your reasonable cost of physically performing this
conveying of source, or (2) access to copy the
Corresponding Source from a network server at no charge.
c) Convey individual copies of the object code with a copy of the
written offer to provide the Corresponding Source. This
alternative is allowed only occasionally and noncommercially, and
only if you received the object code with such an offer, in accord
with subsection 6b.
d) Convey the object code by offering access from a designated
place (gratis or for a charge), and offer equivalent access to the
Corresponding Source in the same way through the same place at no
further charge. You need not require recipients to copy the
Corresponding Source along with the object code. If the place to
copy the object code is a network server, the Corresponding Source
may be on a different server (operated by you or a third party)
that supports equivalent copying facilities, provided you maintain
clear directions next to the object code saying where to find the
Corresponding Source. Regardless of what server hosts the
Corresponding Source, you remain obligated to ensure that it is
available for as long as needed to satisfy these requirements.
e) Convey the object code using peer-to-peer transmission, provided
you inform other peers where the object code and Corresponding
Source of the work are being offered to the general public at no
charge under subsection 6d.
A separable portion of the object code, whose source code is excluded
from the Corresponding Source as a System Library, need not be
included in conveying the object code work.
A "User Product" is either (1) a "consumer product", which means any
tangible personal property which is normally used for personal, family,
or household purposes, or (2) anything designed or sold for incorporation
into a dwelling. In determining whether a product is a consumer product,
doubtful cases shall be resolved in favor of coverage. For a particular
product received by a particular user, "normally used" refers to a
typical or common use of that class of product, regardless of the status
of the particular user or of the way in which the particular user
actually uses, or expects or is expected to use, the product. A product
is a consumer product regardless of whether the product has substantial
commercial, industrial or non-consumer uses, unless such uses represent
the only significant mode of use of the product.
"Installation Information" for a User Product means any methods,
procedures, authorization keys, or other information required to install
and execute modified versions of a covered work in that User Product from
a modified version of its Corresponding Source. The information must
suffice to ensure that the continued functioning of the modified object
code is in no case prevented or interfered with solely because
modification has been made.
If you convey an object code work under this section in, or with, or
specifically for use in, a User Product, and the conveying occurs as
part of a transaction in which the right of possession and use of the
User Product is transferred to the recipient in perpetuity or for a
fixed term (regardless of how the transaction is characterized), the
Corresponding Source conveyed under this section must be accompanied
by the Installation Information. But this requirement does not apply
if neither you nor any third party retains the ability to install
modified object code on the User Product (for example, the work has
been installed in ROM).
The requirement to provide Installation Information does not include a
requirement to continue to provide support service, warranty, or updates
for a work that has been modified or installed by the recipient, or for
the User Product in which it has been modified or installed. Access to a
network may be denied when the modification itself materially and
adversely affects the operation of the network or violates the rules and
protocols for communication across the network.
Corresponding Source conveyed, and Installation Information provided,
in accord with this section must be in a format that is publicly
documented (and with an implementation available to the public in
source code form), and must require no special password or key for
unpacking, reading or copying.
7. Additional Terms.
"Additional permissions" are terms that supplement the terms of this
License by making exceptions from one or more of its conditions.
Additional permissions that are applicable to the entire Program shall
be treated as though they were included in this License, to the extent
that they are valid under applicable law. If additional permissions
apply only to part of the Program, that part may be used separately
under those permissions, but the entire Program remains governed by
this License without regard to the additional permissions.
When you convey a copy of a covered work, you may at your option
remove any additional permissions from that copy, or from any part of
it. (Additional permissions may be written to require their own
removal in certain cases when you modify the work.) You may place
additional permissions on material, added by you to a covered work,
for which you have or can give appropriate copyright permission.
Notwithstanding any other provision of this License, for material you
add to a covered work, you may (if authorized by the copyright holders of
that material) supplement the terms of this License with terms:
a) Disclaiming warranty or limiting liability differently from the
terms of sections 15 and 16 of this License; or
b) Requiring preservation of specified reasonable legal notices or
author attributions in that material or in the Appropriate Legal
Notices displayed by works containing it; or
c) Prohibiting misrepresentation of the origin of that material, or
requiring that modified versions of such material be marked in
reasonable ways as different from the original version; or
d) Limiting the use for publicity purposes of names of licensors or
authors of the material; or
e) Declining to grant rights under trademark law for use of some
trade names, trademarks, or service marks; or
f) Requiring indemnification of licensors and authors of that
material by anyone who conveys the material (or modified versions of
it) with contractual assumptions of liability to the recipient, for
any liability that these contractual assumptions directly impose on
those licensors and authors.
All other non-permissive additional terms are considered "further
restrictions" within the meaning of section 10. If the Program as you
received it, or any part of it, contains a notice stating that it is
governed by this License along with a term that is a further
restriction, you may remove that term. If a license document contains
a further restriction but permits relicensing or conveying under this
License, you may add to a covered work material governed by the terms
of that license document, provided that the further restriction does
not survive such relicensing or conveying.
If you add terms to a covered work in accord with this section, you
must place, in the relevant source files, a statement of the
additional terms that apply to those files, or a notice indicating
where to find the applicable terms.
Additional terms, permissive or non-permissive, may be stated in the
form of a separately written license, or stated as exceptions;
the above requirements apply either way.
8. Termination.
You may not propagate or modify a covered work except as expressly
provided under this License. Any attempt otherwise to propagate or
modify it is void, and will automatically terminate your rights under
this License (including any patent licenses granted under the third
paragraph of section 11).
However, if you cease all violation of this License, then your
license from a particular copyright holder is reinstated (a)
provisionally, unless and until the copyright holder explicitly and
finally terminates your license, and (b) permanently, if the copyright
holder fails to notify you of the violation by some reasonable means
prior to 60 days after the cessation.
Moreover, your license from a particular copyright holder is
reinstated permanently if the copyright holder notifies you of the
violation by some reasonable means, this is the first time you have
received notice of violation of this License (for any work) from that
copyright holder, and you cure the violation prior to 30 days after
your receipt of the notice.
Termination of your rights under this section does not terminate the
licenses of parties who have received copies or rights from you under
this License. If your rights have been terminated and not permanently
reinstated, you do not qualify to receive new licenses for the same
material under section 10.
9. Acceptance Not Required for Having Copies.
You are not required to accept this License in order to receive or
run a copy of the Program. Ancillary propagation of a covered work
occurring solely as a consequence of using peer-to-peer transmission
to receive a copy likewise does not require acceptance. However,
nothing other than this License grants you permission to propagate or
modify any covered work. These actions infringe copyright if you do
not accept this License. Therefore, by modifying or propagating a
covered work, you indicate your acceptance of this License to do so.
10. Automatic Licensing of Downstream Recipients.
Each time you convey a covered work, the recipient automatically
receives a license from the original licensors, to run, modify and
propagate that work, subject to this License. You are not responsible
for enforcing compliance by third parties with this License.
An "entity transaction" is a transaction transferring control of an
organization, or substantially all assets of one, or subdividing an
organization, or merging organizations. If propagation of a covered
work results from an entity transaction, each party to that
transaction who receives a copy of the work also receives whatever
licenses to the work the party's predecessor in interest had or could
give under the previous paragraph, plus a right to possession of the
Corresponding Source of the work from the predecessor in interest, if
the predecessor has it or can get it with reasonable efforts.
You may not impose any further restrictions on the exercise of the
rights granted or affirmed under this License. For example, you may
not impose a license fee, royalty, or other charge for exercise of
rights granted under this License, and you may not initiate litigation
(including a cross-claim or counterclaim in a lawsuit) alleging that
any patent claim is infringed by making, using, selling, offering for
sale, or importing the Program or any portion of it.
11. Patents.
A "contributor" is a copyright holder who authorizes use under this
License of the Program or a work on which the Program is based. The
work thus licensed is called the contributor's "contributor version".
A contributor's "essential patent claims" are all patent claims
owned or controlled by the contributor, whether already acquired or
hereafter acquired, that would be infringed by some manner, permitted
by this License, of making, using, or selling its contributor version,
but do not include claims that would be infringed only as a
consequence of further modification of the contributor version. For
purposes of this definition, "control" includes the right to grant
patent sublicenses in a manner consistent with the requirements of
this License.
Each contributor grants you a non-exclusive, worldwide, royalty-free
patent license under the contributor's essential patent claims, to
make, use, sell, offer for sale, import and otherwise run, modify and
propagate the contents of its contributor version.
In the following three paragraphs, a "patent license" is any express
agreement or commitment, however denominated, not to enforce a patent
(such as an express permission to practice a patent or covenant not to
sue for patent infringement). To "grant" such a patent license to a
party means to make such an agreement or commitment not to enforce a
patent against the party.
If you convey a covered work, knowingly relying on a patent license,
and the Corresponding Source of the work is not available for anyone
to copy, free of charge and under the terms of this License, through a
publicly available network server or other readily accessible means,
then you must either (1) cause the Corresponding Source to be so
available, or (2) arrange to deprive yourself of the benefit of the
patent license for this particular work, or (3) arrange, in a manner
consistent with the requirements of this License, to extend the patent
license to downstream recipients. "Knowingly relying" means you have
actual knowledge that, but for the patent license, your conveying the
covered work in a country, or your recipient's use of the covered work
in a country, would infringe one or more identifiable patents in that
country that you have reason to believe are valid.
If, pursuant to or in connection with a single transaction or
arrangement, you convey, or propagate by procuring conveyance of, a
covered work, and grant a patent license to some of the parties
receiving the covered work authorizing them to use, propagate, modify
or convey a specific copy of the covered work, then the patent license
you grant is automatically extended to all recipients of the covered
work and works based on it.
A patent license is "discriminatory" if it does not include within
the scope of its coverage, prohibits the exercise of, or is
conditioned on the non-exercise of one or more of the rights that are
specifically granted under this License. You may not convey a covered
work if you are a party to an arrangement with a third party that is
in the business of distributing software, under which you make payment
to the third party based on the extent of your activity of conveying
the work, and under which the third party grants, to any of the
parties who would receive the covered work from you, a discriminatory
patent license (a) in connection with copies of the covered work
conveyed by you (or copies made from those copies), or (b) primarily
for and in connection with specific products or compilations that
contain the covered work, unless you entered into that arrangement,
or that patent license was granted, prior to 28 March 2007.
Nothing in this License shall be construed as excluding or limiting
any implied license or other defenses to infringement that may
otherwise be available to you under applicable patent law.
12. No Surrender of Others' Freedom.
If conditions are imposed on you (whether by court order, agreement or
otherwise) that contradict the conditions of this License, they do not
excuse you from the conditions of this License. If you cannot convey a
covered work so as to satisfy simultaneously your obligations under this
License and any other pertinent obligations, then as a consequence you may
not convey it at all. For example, if you agree to terms that obligate you
to collect a royalty for further conveying from those to whom you convey
the Program, the only way you could satisfy both those terms and this
License would be to refrain entirely from conveying the Program.
13. Remote Network Interaction; Use with the GNU General Public License.
Notwithstanding any other provision of this License, if you modify the
Program, your modified version must prominently offer all users
interacting with it remotely through a computer network (if your version
supports such interaction) an opportunity to receive the Corresponding
Source of your version by providing access to the Corresponding Source
from a network server at no charge, through some standard or customary
means of facilitating copying of software. This Corresponding Source
shall include the Corresponding Source for any work covered by version 3
of the GNU General Public License that is incorporated pursuant to the
following paragraph.
Notwithstanding any other provision of this License, you have
permission to link or combine any covered work with a work licensed
under version 3 of the GNU General Public License into a single
combined work, and to convey the resulting work. The terms of this
License will continue to apply to the part which is the covered work,
but the work with which it is combined will remain governed by version
3 of the GNU General Public License.
14. Revised Versions of this License.
The Free Software Foundation may publish revised and/or new versions of
the GNU Affero General Public License from time to time. Such new versions
will be similar in spirit to the present version, but may differ in detail to
address new problems or concerns.
Each version is given a distinguishing version number. If the
Program specifies that a certain numbered version of the GNU Affero General
Public License "or any later version" applies to it, you have the
option of following the terms and conditions either of that numbered
version or of any later version published by the Free Software
Foundation. If the Program does not specify a version number of the
GNU Affero General Public License, you may choose any version ever published
by the Free Software Foundation.
If the Program specifies that a proxy can decide which future
versions of the GNU Affero General Public License can be used, that proxy's
public statement of acceptance of a version permanently authorizes you
to choose that version for the Program.
Later license versions may give you additional or different
permissions. However, no additional obligations are imposed on any
author or copyright holder as a result of your choosing to follow a
later version.
15. Disclaimer of Warranty.
THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
16. Limitation of Liability.
IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
SUCH DAMAGES.
17. Interpretation of Sections 15 and 16.
If the disclaimer of warranty and limitation of liability provided
above cannot be given local legal effect according to their terms,
reviewing courts shall apply local law that most closely approximates
an absolute waiver of all civil liability in connection with the
Program, unless a warranty or assumption of liability accompanies a
copy of the Program in return for a fee.
END OF TERMS AND CONDITIONS
How to Apply These Terms to Your New Programs
If you develop a new program, and you want it to be of the greatest
possible use to the public, the best way to achieve this is to make it
free software which everyone can redistribute and change under these terms.
To do so, attach the following notices to the program. It is safest
to attach them to the start of each source file to most effectively
state the exclusion of warranty; and each file should have at least
the "copyright" line and a pointer to where the full notice is found.
<one line to give the program's name and a brief idea of what it does.>
Copyright (C) <year> <name of author>
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
Also add information on how to contact you by electronic and paper mail.
If your software can interact with users remotely through a computer
network, you should also make sure that it provides a way for users to
get its source. For example, if your program is a web application, its
interface could display a "Source" link that leads users to an archive
of the code. There are many ways you could offer source, and different
solutions will be better for different programs; see section 13 for the
specific requirements.
You should also get your employer (if you work as a programmer) or school,
if any, to sign a "copyright disclaimer" for the program, if necessary.
For more information on this, and how to apply and follow the GNU AGPL, see
<http://www.gnu.org/licenses/>.

90
services/clsi/Makefile Normal file
View file

@ -0,0 +1,90 @@
# This file was auto-generated, do not edit it directly.
# Instead run bin/update_build_scripts from
# https://github.com/sharelatex/sharelatex-dev-environment
BUILD_NUMBER ?= local
BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD)
PROJECT_NAME = clsi
BUILD_DIR_NAME = $(shell pwd | xargs basename | tr -cd '[a-zA-Z0-9_.\-]')
DOCKER_COMPOSE_FLAGS ?= -f docker-compose.yml
DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \
BRANCH_NAME=$(BRANCH_NAME) \
PROJECT_NAME=$(PROJECT_NAME) \
MOCHA_GREP=${MOCHA_GREP} \
docker-compose ${DOCKER_COMPOSE_FLAGS}
DOCKER_COMPOSE_TEST_ACCEPTANCE = \
COMPOSE_PROJECT_NAME=test_acceptance_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE)
DOCKER_COMPOSE_TEST_UNIT = \
COMPOSE_PROJECT_NAME=test_unit_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE)
clean:
-docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
-docker rmi gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
-$(DOCKER_COMPOSE_TEST_UNIT) down --rmi local
-$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down --rmi local
format:
$(DOCKER_COMPOSE) run --rm test_unit npm run --silent format
format_fix:
$(DOCKER_COMPOSE) run --rm test_unit npm run --silent format:fix
lint:
$(DOCKER_COMPOSE) run --rm test_unit npm run --silent lint
test: format lint test_unit test_acceptance
test_unit:
ifneq (,$(wildcard test/unit))
$(DOCKER_COMPOSE_TEST_UNIT) run --rm test_unit
$(MAKE) test_unit_clean
endif
test_clean: test_unit_clean
test_unit_clean:
ifneq (,$(wildcard test/unit))
$(DOCKER_COMPOSE_TEST_UNIT) down -v -t 0
endif
test_acceptance: test_acceptance_clean test_acceptance_pre_run test_acceptance_run
$(MAKE) test_acceptance_clean
test_acceptance_debug: test_acceptance_clean test_acceptance_pre_run test_acceptance_run_debug
$(MAKE) test_acceptance_clean
test_acceptance_run:
ifneq (,$(wildcard test/acceptance))
$(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance
endif
test_acceptance_run_debug:
ifneq (,$(wildcard test/acceptance))
$(DOCKER_COMPOSE_TEST_ACCEPTANCE) run -p 127.0.0.9:19999:19999 --rm test_acceptance npm run test:acceptance -- --inspect=0.0.0.0:19999 --inspect-brk
endif
test_clean: test_acceptance_clean
test_acceptance_clean:
$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down -v -t 0
test_acceptance_pre_run:
ifneq (,$(wildcard test/acceptance/js/scripts/pre-run))
$(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance test/acceptance/js/scripts/pre-run
endif
build:
docker build --pull --tag ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \
--tag gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \
.
tar:
$(DOCKER_COMPOSE) up tar
publish:
docker push $(DOCKER_REPO)/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
.PHONY: clean test test_unit test_acceptance test_clean build publish

184
services/clsi/README.md Normal file
View file

@ -0,0 +1,184 @@
overleaf/clsi
===============
A web api for compiling LaTeX documents in the cloud
The Common LaTeX Service Interface (CLSI) provides a RESTful interface to traditional LaTeX tools (or, more generally, any command line tool for composing marked-up documents into a display format such as PDF or HTML). The CLSI listens on the following ports by default:
* TCP/3013 - the RESTful interface
* TCP/3048 - reports load information
* TCP/3049 - HTTP interface to control the CLSI service
These defaults can be modified in `config/settings.defaults.js`.
The provided `Dockerfile` builds a Docker image which has the Docker command line tools installed. The configuration in `docker-compose-config.yml` mounts the Docker socket, in order that the CLSI container can talk to the Docker host it is running in. This allows it to spin up `sibling containers` running an image with a TeX distribution installed to perform the actual compiles.
The CLSI can be configured through the following environment variables:
* `ALLOWED_COMPILE_GROUPS` - Space separated list of allowed compile groups
* `ALLOWED_IMAGES` - Space separated list of allowed Docker TeX Live images
* `CATCH_ERRORS` - Set to `true` to log uncaught exceptions
* `COMPILE_GROUP_DOCKER_CONFIGS` - JSON string of Docker configs for compile groups
* `COMPILES_HOST_DIR` - Working directory for LaTeX compiles
* `COMPILE_SIZE_LIMIT` - Sets the body-parser [limit](https://github.com/expressjs/body-parser#limit)
* `DOCKER_RUNNER` - Set to true to use sibling containers
* `DOCKER_RUNTIME` -
* `FILESTORE_DOMAIN_OVERRIDE` - The url for the filestore service e.g.`http://$FILESTORE_HOST:3009`
* `FILESTORE_PARALLEL_FILE_DOWNLOADS` - Number of parallel file downloads
* `FILESTORE_PARALLEL_SQL_QUERY_LIMIT` - Number of parallel SQL queries
* `LISTEN_ADDRESS` - The address for the RESTful service to listen on. Set to `0.0.0.0` to listen on all network interfaces
* `PROCESS_LIFE_SPAN_LIMIT_MS` - Process life span limit in milliseconds
* `SENTRY_DSN` - Sentry [Data Source Name](https://docs.sentry.io/product/sentry-basics/dsn-explainer/)
* `SMOKE_TEST` - Whether to run smoke tests
* `SQLITE_PATH` - Path to SQLite database
* `SYNCTEX_BIN_HOST_PATH` - Path to SyncTeX binary
* `TEXLIVE_IMAGE` - The TeX Live Docker image to use for sibling containers, e.g. `gcr.io/overleaf-ops/texlive-full:2017.1`
* `TEX_LIVE_IMAGE_NAME_OVERRIDE` - The name of the registry for the Docker image e.g. `gcr.io/overleaf-ops`
* `TEXLIVE_IMAGE_USER` - When using sibling containers, the user to run as in the TeX Live image. Defaults to `tex`
* `TEXLIVE_OPENOUT_ANY` - Sets the `openout_any` environment variable for TeX Live (see the `\openout` primitive [documentation](http://tug.org/texinfohtml/web2c.html#tex-invocation))
Further environment variables configure the [metrics module](https://github.com/overleaf/metrics-module)
Installation
------------
The CLSI can be installed and set up as part of the entire [Overleaf stack](https://github.com/overleaf/overleaf) (complete with front end editor and document storage), or it can be run as a standalone service. To run is as a standalone service, first checkout this repository:
$ git clone git@github.com:overleaf/clsi.git
Then build the Docker image:
$ docker build . -t overleaf/clsi
Then pull the TeX Live image:
$ docker pull texlive/texlive
Then start the Docker container:
$ docker run --rm \
-p 127.0.0.1:3013:3013 \
-e LISTEN_ADDRESS=0.0.0.0 \
-e DOCKER_RUNNER=true \
-e TEXLIVE_IMAGE=texlive/texlive \
-e TEXLIVE_IMAGE_USER=root \
-e COMPILES_HOST_DIR="$PWD/compiles" \
-v "$PWD/compiles:/app/compiles" \
-v "$PWD/cache:/app/cache" \
-v /var/run/docker.sock:/var/run/docker.sock \
--name clsi \
overleaf/clsi
Note: if you're running the CLSI in macOS you may need to use `-v /var/run/docker.sock.raw:/var/run/docker.sock` instead.
The CLSI should then be running at <http://localhost:3013>
Important note for Linux users
==============================
The Node application runs as user `node` in the CLSI, which has uid `1000`. As a consequence of this, the `compiles` folder gets created on your host with `uid` and `gid` set to `1000`.
```
ls -lnd compiles
drwxr-xr-x 2 1000 1000 4096 Mar 19 12:41 compiles
```
If there is a user/group on your host which also happens to have `uid` / `gid` `1000` then that user/group will have ownership of the compiles folder on your host.
LaTeX runs in the sibling containers as the user specified in the `TEXLIVE_IMAGE_USER` environment variable. In the example above this is set to `root`, which has uid `0`. This creates a problem with the above permissions, as the root user does not have permission to write to subfolders of `compiles`.
A quick fix is to give the `root` group ownership and read write permissions to `compiles`, with `setgid` set so that new subfolders also inherit this ownership:
```
sudo chown -R 1000:root compiles
sudo chmod -R g+w compiles
sudo chmod g+s compiles
```
Another solution is to create a `sharelatex` group and add both `root` and the user with `uid` `1000` to it. If the host does not have a user with that `uid`, you will need to create one first.
```
sudo useradd --uid 1000 host-node-user # If required
sudo groupadd sharelatex
sudo usermod -a -G sharelatex root
sudo usermod -a -G sharelatex $(id -nu 1000)
sudo chown -R 1000:sharelatex compiles
sudo chmod -R g+w compiles
sudo chmod g+s compiles
```
This is a facet of the way docker works on Linux. See this [upstream issue](https://github.com/moby/moby/issues/7198)
Config
------
The CLSI will use a SQLite database by default, but you can optionally set up a MySQL database and then fill in the database name, username and password in the config file at `config/settings.development.js`.
API
---
The CLSI is based on a JSON API.
#### Example Request
(Note that valid JSON should not contain any comments like the example below).
POST /project/<project-id>/compile
```json5
{
"compile": {
"options": {
// Which compiler to use. Can be latex, pdflatex, xelatex or lualatex
"compiler": "lualatex",
// How many seconds to wait before killing the process. Default is 60.
"timeout": 40
},
// The main file to run LaTeX on
"rootResourcePath": "main.tex",
// An array of files to include in the compilation. May have either the content
// passed directly, or a URL where it can be downloaded.
"resources": [
{
"path": "main.tex",
"content": "\\documentclass{article}\n\\begin{document}\nHello World\n\\end{document}"
}
// ,{
// "path": "image.png",
// "url": "www.example.com/image.png",
// "modified": 123456789 // Unix time since epoch
// }
]
}
}
```
With `curl`, if you place the above JSON in a file called `data.json`, the request would look like this:
``` shell
$ curl -X POST -H 'Content-Type: application/json' -d @data.json http://localhost:3013/project/<id>/compile
```
You can specify any project-id in the URL, and the files and LaTeX environment will be persisted between requests.
URLs will be downloaded and cached until provided with a more recent modified date.
#### Example Response
```json
{
"compile": {
"status": "success",
"outputFiles": [{
"type": "pdf",
"url": "http://localhost:3013/project/<project-id>/output/output.pdf"
}, {
"type": "log",
"url": "http://localhost:3013/project/<project-id>/output/output.log"
}]
}
}
```
License
-------
The code in this repository is released under the GNU AFFERO GENERAL PUBLIC LICENSE, version 3. A copy can be found in the `LICENSE` file.
Copyright (c) Overleaf, 2014-2021.

425
services/clsi/app.js Normal file
View file

@ -0,0 +1,425 @@
/*
* decaffeinate suggestions:
* DS102: Remove unnecessary code created because of implicit returns
* DS103: Rewrite code to no longer use __guard__
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
const tenMinutes = 10 * 60 * 1000
const Metrics = require('@overleaf/metrics')
Metrics.initialize('clsi')
const CompileController = require('./app/js/CompileController')
const ContentController = require('./app/js/ContentController')
const Settings = require('@overleaf/settings')
const logger = require('logger-sharelatex')
logger.initialize('clsi')
if ((Settings.sentry != null ? Settings.sentry.dsn : undefined) != null) {
logger.initializeErrorReporting(Settings.sentry.dsn)
}
const smokeTest = require('./test/smoke/js/SmokeTests')
const ContentTypeMapper = require('./app/js/ContentTypeMapper')
const Errors = require('./app/js/Errors')
const Path = require('path')
Metrics.open_sockets.monitor(logger)
Metrics.memory.monitor(logger)
const ProjectPersistenceManager = require('./app/js/ProjectPersistenceManager')
const OutputCacheManager = require('./app/js/OutputCacheManager')
const ContentCacheManager = require('./app/js/ContentCacheManager')
require('./app/js/db').sync()
const express = require('express')
const bodyParser = require('body-parser')
const app = express()
Metrics.injectMetricsRoute(app)
app.use(Metrics.http.monitor(logger))
// Compile requests can take longer than the default two
// minutes (including file download time), so bump up the
// timeout a bit.
const TIMEOUT = 10 * 60 * 1000
app.use(function (req, res, next) {
req.setTimeout(TIMEOUT)
res.setTimeout(TIMEOUT)
res.removeHeader('X-Powered-By')
return next()
})
app.param('project_id', function (req, res, next, projectId) {
if (projectId != null ? projectId.match(/^[a-zA-Z0-9_-]+$/) : undefined) {
return next()
} else {
return next(new Error('invalid project id'))
}
})
app.param('user_id', function (req, res, next, userId) {
if (userId != null ? userId.match(/^[0-9a-f]{24}$/) : undefined) {
return next()
} else {
return next(new Error('invalid user id'))
}
})
app.param('build_id', function (req, res, next, buildId) {
if (
buildId != null ? buildId.match(OutputCacheManager.BUILD_REGEX) : undefined
) {
return next()
} else {
return next(new Error(`invalid build id ${buildId}`))
}
})
app.param('contentId', function (req, res, next, contentId) {
if (
contentId != null
? contentId.match(OutputCacheManager.CONTENT_REGEX)
: undefined
) {
return next()
} else {
return next(new Error(`invalid content id ${contentId}`))
}
})
app.param('hash', function (req, res, next, hash) {
if (hash != null ? hash.match(ContentCacheManager.HASH_REGEX) : undefined) {
return next()
} else {
return next(new Error(`invalid hash ${hash}`))
}
})
app.post(
'/project/:project_id/compile',
bodyParser.json({ limit: Settings.compileSizeLimit }),
CompileController.compile
)
app.post('/project/:project_id/compile/stop', CompileController.stopCompile)
app.delete('/project/:project_id', CompileController.clearCache)
app.get('/project/:project_id/sync/code', CompileController.syncFromCode)
app.get('/project/:project_id/sync/pdf', CompileController.syncFromPdf)
app.get('/project/:project_id/wordcount', CompileController.wordcount)
app.get('/project/:project_id/status', CompileController.status)
app.post('/project/:project_id/status', CompileController.status)
// Per-user containers
app.post(
'/project/:project_id/user/:user_id/compile',
bodyParser.json({ limit: Settings.compileSizeLimit }),
CompileController.compile
)
app.post(
'/project/:project_id/user/:user_id/compile/stop',
CompileController.stopCompile
)
app.delete('/project/:project_id/user/:user_id', CompileController.clearCache)
app.get(
'/project/:project_id/user/:user_id/sync/code',
CompileController.syncFromCode
)
app.get(
'/project/:project_id/user/:user_id/sync/pdf',
CompileController.syncFromPdf
)
app.get(
'/project/:project_id/user/:user_id/wordcount',
CompileController.wordcount
)
const ForbidSymlinks = require('./app/js/StaticServerForbidSymlinks')
// create a static server which does not allow access to any symlinks
// avoids possible mismatch of root directory between middleware check
// and serving the files
const staticCompileServer = ForbidSymlinks(
express.static,
Settings.path.compilesDir,
{
setHeaders(res, path, stat) {
if (Path.basename(path) === 'output.pdf') {
// Calculate an etag in the same way as nginx
// https://github.com/tj/send/issues/65
const etag = (path, stat) =>
`"${Math.ceil(+stat.mtime / 1000).toString(16)}` +
'-' +
Number(stat.size).toString(16) +
'"'
res.set('Etag', etag(path, stat))
}
return res.set('Content-Type', ContentTypeMapper.map(path))
},
}
)
const staticOutputServer = ForbidSymlinks(
express.static,
Settings.path.outputDir,
{
setHeaders(res, path, stat) {
if (Path.basename(path) === 'output.pdf') {
// Calculate an etag in the same way as nginx
// https://github.com/tj/send/issues/65
const etag = (path, stat) =>
`"${Math.ceil(+stat.mtime / 1000).toString(16)}` +
'-' +
Number(stat.size).toString(16) +
'"'
res.set('Etag', etag(path, stat))
}
return res.set('Content-Type', ContentTypeMapper.map(path))
},
}
)
app.get(
'/project/:project_id/user/:user_id/build/:build_id/output/*',
function (req, res, next) {
// for specific build get the path from the OutputCacheManager (e.g. .clsi/buildId)
req.url =
`/${req.params.project_id}-${req.params.user_id}/` +
OutputCacheManager.path(req.params.build_id, `/${req.params[0]}`)
return staticOutputServer(req, res, next)
}
)
app.get(
'/project/:projectId/content/:contentId/:hash',
ContentController.getPdfRange
)
app.get(
'/project/:projectId/user/:userId/content/:contentId/:hash',
ContentController.getPdfRange
)
app.get(
'/project/:project_id/build/:build_id/output/*',
function (req, res, next) {
// for specific build get the path from the OutputCacheManager (e.g. .clsi/buildId)
req.url =
`/${req.params.project_id}/` +
OutputCacheManager.path(req.params.build_id, `/${req.params[0]}`)
return staticOutputServer(req, res, next)
}
)
app.get(
'/project/:project_id/user/:user_id/output/*',
function (req, res, next) {
// for specific user get the path to the top level file
logger.warn(
{ url: req.url },
'direct request for file in compile directory'
)
req.url = `/${req.params.project_id}-${req.params.user_id}/${req.params[0]}`
return staticCompileServer(req, res, next)
}
)
app.get('/project/:project_id/output/*', function (req, res, next) {
logger.warn({ url: req.url }, 'direct request for file in compile directory')
if (
(req.query != null ? req.query.build : undefined) != null &&
req.query.build.match(OutputCacheManager.BUILD_REGEX)
) {
// for specific build get the path from the OutputCacheManager (e.g. .clsi/buildId)
req.url =
`/${req.params.project_id}/` +
OutputCacheManager.path(req.query.build, `/${req.params[0]}`)
} else {
req.url = `/${req.params.project_id}/${req.params[0]}`
}
return staticCompileServer(req, res, next)
})
app.get('/oops', function (req, res, next) {
logger.error({ err: 'hello' }, 'test error')
return res.send('error\n')
})
app.get('/oops-internal', function (req, res, next) {
setTimeout(function () {
throw new Error('Test error')
}, 1)
})
app.get('/status', (req, res, next) => res.send('CLSI is alive\n'))
Settings.processTooOld = false
if (Settings.processLifespanLimitMs) {
Settings.processLifespanLimitMs +=
Settings.processLifespanLimitMs * (Math.random() / 10)
logger.info(
'Lifespan limited to ',
Date.now() + Settings.processLifespanLimitMs
)
setTimeout(() => {
logger.log('shutting down, process is too old')
Settings.processTooOld = true
}, Settings.processLifespanLimitMs)
}
function runSmokeTest() {
if (Settings.processTooOld) return
logger.log('running smoke tests')
smokeTest.triggerRun(err => {
if (err) logger.error({ err }, 'smoke tests failed')
setTimeout(runSmokeTest, 30 * 1000)
})
}
if (Settings.smokeTest) {
runSmokeTest()
}
app.get('/health_check', function (req, res) {
if (Settings.processTooOld) {
return res.status(500).json({ processTooOld: true })
}
smokeTest.sendLastResult(res)
})
app.get('/smoke_test_force', (req, res) => smokeTest.sendNewResult(res))
app.use(function (error, req, res, next) {
if (error instanceof Errors.NotFoundError) {
logger.log({ err: error, url: req.url }, 'not found error')
return res.sendStatus(404)
} else if (error.code === 'EPIPE') {
// inspect container returns EPIPE when shutting down
return res.sendStatus(503) // send 503 Unavailable response
} else {
logger.error({ err: error, url: req.url }, 'server error')
return res.sendStatus((error != null ? error.statusCode : undefined) || 500)
}
})
const net = require('net')
const os = require('os')
let STATE = 'up'
const loadTcpServer = net.createServer(function (socket) {
socket.on('error', function (err) {
if (err.code === 'ECONNRESET') {
// this always comes up, we don't know why
return
}
logger.err({ err }, 'error with socket on load check')
return socket.destroy()
})
if (STATE === 'up' && Settings.internal.load_balancer_agent.report_load) {
let availableWorkingCpus
const currentLoad = os.loadavg()[0]
// staging clis's have 1 cpu core only
if (os.cpus().length === 1) {
availableWorkingCpus = 1
} else {
availableWorkingCpus = os.cpus().length - 1
}
const freeLoad = availableWorkingCpus - currentLoad
let freeLoadPercentage = Math.round((freeLoad / availableWorkingCpus) * 100)
if (freeLoadPercentage <= 0) {
freeLoadPercentage = 1 // when its 0 the server is set to drain and will move projects to different servers
}
socket.write(`up, ${freeLoadPercentage}%\n`, 'ASCII')
return socket.end()
} else {
socket.write(`${STATE}\n`, 'ASCII')
return socket.end()
}
})
const loadHttpServer = express()
loadHttpServer.post('/state/up', function (req, res, next) {
STATE = 'up'
logger.info('getting message to set server to down')
return res.sendStatus(204)
})
loadHttpServer.post('/state/down', function (req, res, next) {
STATE = 'down'
logger.info('getting message to set server to down')
return res.sendStatus(204)
})
loadHttpServer.post('/state/maint', function (req, res, next) {
STATE = 'maint'
logger.info('getting message to set server to maint')
return res.sendStatus(204)
})
const port =
__guard__(
Settings.internal != null ? Settings.internal.clsi : undefined,
x => x.port
) || 3013
const host =
__guard__(
Settings.internal != null ? Settings.internal.clsi : undefined,
x1 => x1.host
) || 'localhost'
const loadTcpPort = Settings.internal.load_balancer_agent.load_port
const loadHttpPort = Settings.internal.load_balancer_agent.local_port
if (!module.parent) {
// Called directly
// handle uncaught exceptions when running in production
if (Settings.catchErrors) {
process.removeAllListeners('uncaughtException')
process.on('uncaughtException', error =>
logger.error({ err: error }, 'uncaughtException')
)
}
app.listen(port, host, error => {
if (error) {
logger.fatal({ error }, `Error starting CLSI on ${host}:${port}`)
} else {
logger.info(`CLSI starting up, listening on ${host}:${port}`)
}
})
loadTcpServer.listen(loadTcpPort, host, function (error) {
if (error != null) {
throw error
}
return logger.info(`Load tcp agent listening on load port ${loadTcpPort}`)
})
loadHttpServer.listen(loadHttpPort, host, function (error) {
if (error != null) {
throw error
}
return logger.info(`Load http agent listening on load port ${loadHttpPort}`)
})
}
module.exports = app
setInterval(() => {
ProjectPersistenceManager.refreshExpiryTimeout(() => {
ProjectPersistenceManager.clearExpiredProjects()
})
}, tenMinutes)
function __guard__(value, transform) {
return typeof value !== 'undefined' && value !== null
? transform(value)
: undefined
}

20
services/clsi/app/js/CommandRunner.js Normal file
View file

@ -0,0 +1,20 @@
// TODO: This file was created by bulk-decaffeinate.
// Sanity-check the conversion and remove this comment.
/*
* decaffeinate suggestions:
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
let commandRunnerPath
const Settings = require('@overleaf/settings')
const logger = require('logger-sharelatex')
if ((Settings.clsi != null ? Settings.clsi.dockerRunner : undefined) === true) {
commandRunnerPath = './DockerRunner'
} else {
commandRunnerPath = './LocalCommandRunner'
}
logger.info({ commandRunnerPath }, 'selecting command runner for clsi')
const CommandRunner = require(commandRunnerPath)
module.exports = CommandRunner

269
services/clsi/app/js/CompileController.js Normal file
View file

@ -0,0 +1,269 @@
/* eslint-disable
camelcase,
handle-callback-err,
no-unused-vars,
*/
// TODO: This file was created by bulk-decaffeinate.
// Fix any style issues and re-enable lint.
/*
* decaffeinate suggestions:
* DS101: Remove unnecessary use of Array.from
* DS102: Remove unnecessary code created because of implicit returns
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
let CompileController
const RequestParser = require('./RequestParser')
const CompileManager = require('./CompileManager')
const Settings = require('@overleaf/settings')
const Metrics = require('./Metrics')
const ProjectPersistenceManager = require('./ProjectPersistenceManager')
const logger = require('logger-sharelatex')
const Errors = require('./Errors')
function isImageNameAllowed(imageName) {
const ALLOWED_IMAGES =
Settings.clsi && Settings.clsi.docker && Settings.clsi.docker.allowedImages
return !ALLOWED_IMAGES || ALLOWED_IMAGES.includes(imageName)
}
module.exports = CompileController = {
compile(req, res, next) {
if (next == null) {
next = function (error) {}
}
const timer = new Metrics.Timer('compile-request')
return RequestParser.parse(req.body, function (error, request) {
if (error != null) {
return next(error)
}
request.project_id = req.params.project_id
if (req.params.user_id != null) {
request.user_id = req.params.user_id
}
return ProjectPersistenceManager.markProjectAsJustAccessed(
request.project_id,
function (error) {
if (error != null) {
return next(error)
}
return CompileManager.doCompileWithLock(
request,
function (error, outputFiles, stats, timings) {
let code, status
if (outputFiles == null) {
outputFiles = []
}
if (error instanceof Errors.AlreadyCompilingError) {
code = 423 // Http 423 Locked
status = 'compile-in-progress'
} else if (error instanceof Errors.FilesOutOfSyncError) {
code = 409 // Http 409 Conflict
status = 'retry'
} else if (error && error.code === 'EPIPE') {
// docker returns EPIPE when shutting down
code = 503 // send 503 Unavailable response
status = 'unavailable'
} else if (error != null ? error.terminated : undefined) {
status = 'terminated'
} else if (error != null ? error.validate : undefined) {
status = `validation-${error.validate}`
} else if (error != null ? error.timedout : undefined) {
status = 'timedout'
logger.log(
{ err: error, project_id: request.project_id },
'timeout running compile'
)
} else if (error != null) {
status = 'error'
code = 500
logger.warn(
{ err: error, project_id: request.project_id },
'error running compile'
)
} else {
let file
status = 'failure'
for (file of Array.from(outputFiles)) {
if (file.path === 'output.pdf' && file.size > 0) {
status = 'success'
}
}
if (status === 'failure') {
logger.warn(
{ project_id: request.project_id, outputFiles },
'project failed to compile successfully, no output.pdf generated'
)
}
// log an error if any core files are found
for (file of Array.from(outputFiles)) {
if (file.path === 'core') {
logger.error(
{ project_id: request.project_id, req, outputFiles },
'core file found in output'
)
}
}
}
if (error != null) {
outputFiles = error.outputFiles || []
}
timer.done()
return res.status(code || 200).send({
compile: {
status,
error: (error != null ? error.message : undefined) || error,
stats,
timings,
outputFiles: outputFiles.map(file => {
return {
url:
`${Settings.apis.clsi.url}/project/${request.project_id}` +
(request.user_id != null
? `/user/${request.user_id}`
: '') +
(file.build != null ? `/build/${file.build}` : '') +
`/output/${file.path}`,
...file,
}
}),
},
})
}
)
}
)
})
},
stopCompile(req, res, next) {
const { project_id, user_id } = req.params
return CompileManager.stopCompile(project_id, user_id, function (error) {
if (error != null) {
return next(error)
}
return res.sendStatus(204)
})
},
clearCache(req, res, next) {
if (next == null) {
next = function (error) {}
}
return ProjectPersistenceManager.clearProject(
req.params.project_id,
req.params.user_id,
function (error) {
if (error != null) {
return next(error)
}
return res.sendStatus(204)
}
)
}, // No content
syncFromCode(req, res, next) {
if (next == null) {
next = function (error) {}
}
const { file } = req.query
const line = parseInt(req.query.line, 10)
const column = parseInt(req.query.column, 10)
const { imageName } = req.query
const { project_id } = req.params
const { user_id } = req.params
if (imageName && !isImageNameAllowed(imageName)) {
return res.status(400).send('invalid image')
}
return CompileManager.syncFromCode(
project_id,
user_id,
file,
line,
column,
imageName,
function (error, pdfPositions) {
if (error != null) {
return next(error)
}
return res.json({
pdf: pdfPositions,
})
}
)
},
syncFromPdf(req, res, next) {
if (next == null) {
next = function (error) {}
}
const page = parseInt(req.query.page, 10)
const h = parseFloat(req.query.h)
const v = parseFloat(req.query.v)
const { imageName } = req.query
const { project_id } = req.params
const { user_id } = req.params
if (imageName && !isImageNameAllowed(imageName)) {
return res.status(400).send('invalid image')
}
return CompileManager.syncFromPdf(
project_id,
user_id,
page,
h,
v,
imageName,
function (error, codePositions) {
if (error != null) {
return next(error)
}
return res.json({
code: codePositions,
})
}
)
},
wordcount(req, res, next) {
if (next == null) {
next = function (error) {}
}
const file = req.query.file || 'main.tex'
const { project_id } = req.params
const { user_id } = req.params
const { image } = req.query
if (image && !isImageNameAllowed(image)) {
return res.status(400).send('invalid image')
}
logger.log({ image, file, project_id }, 'word count request')
return CompileManager.wordcount(
project_id,
user_id,
file,
image,
function (error, result) {
if (error != null) {
return next(error)
}
return res.json({
texcount: result,
})
}
)
},
status(req, res, next) {
if (next == null) {
next = function (error) {}
}
return res.send('OK')
},
}

761
services/clsi/app/js/CompileManager.js Normal file
View file

@ -0,0 +1,761 @@
/* eslint-disable
camelcase,
handle-callback-err,
no-return-assign,
no-undef,
no-unused-vars,
*/
// TODO: This file was created by bulk-decaffeinate.
// Fix any style issues and re-enable lint.
/*
* decaffeinate suggestions:
* DS101: Remove unnecessary use of Array.from
* DS102: Remove unnecessary code created because of implicit returns
* DS103: Rewrite code to no longer use __guard__
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
let CompileManager
const ResourceWriter = require('./ResourceWriter')
const LatexRunner = require('./LatexRunner')
const OutputFileFinder = require('./OutputFileFinder')
const OutputCacheManager = require('./OutputCacheManager')
const Settings = require('@overleaf/settings')
const Path = require('path')
const logger = require('logger-sharelatex')
const Metrics = require('./Metrics')
const child_process = require('child_process')
const DraftModeManager = require('./DraftModeManager')
const TikzManager = require('./TikzManager')
const LockManager = require('./LockManager')
const fs = require('fs')
const fse = require('fs-extra')
const os = require('os')
const async = require('async')
const Errors = require('./Errors')
const CommandRunner = require('./CommandRunner')
const { emitPdfStats } = require('./ContentCacheMetrics')
const getCompileName = function (project_id, user_id) {
if (user_id != null) {
return `${project_id}-${user_id}`
} else {
return project_id
}
}
const getCompileDir = (project_id, user_id) =>
Path.join(Settings.path.compilesDir, getCompileName(project_id, user_id))
const getOutputDir = (project_id, user_id) =>
Path.join(Settings.path.outputDir, getCompileName(project_id, user_id))
module.exports = CompileManager = {
doCompileWithLock(request, callback) {
if (callback == null) {
callback = function (error, outputFiles) {}
}
const compileDir = getCompileDir(request.project_id, request.user_id)
const lockFile = Path.join(compileDir, '.project-lock')
// use a .project-lock file in the compile directory to prevent
// simultaneous compiles
return fse.ensureDir(compileDir, function (error) {
if (error != null) {
return callback(error)
}
return LockManager.runWithLock(
lockFile,
releaseLock => CompileManager.doCompile(request, releaseLock),
callback
)
})
},
doCompile(request, callback) {
if (callback == null) {
callback = function (error, outputFiles) {}
}
const compileDir = getCompileDir(request.project_id, request.user_id)
const outputDir = getOutputDir(request.project_id, request.user_id)
const timerE2E = new Metrics.Timer('compile-e2e')
let timer = new Metrics.Timer('write-to-disk')
logger.log(
{ project_id: request.project_id, user_id: request.user_id },
'syncing resources to disk'
)
return ResourceWriter.syncResourcesToDisk(
request,
compileDir,
function (error, resourceList) {
// NOTE: resourceList is insecure, it should only be used to exclude files from the output list
if (error != null && error instanceof Errors.FilesOutOfSyncError) {
logger.warn(
{ project_id: request.project_id, user_id: request.user_id },
'files out of sync, please retry'
)
return callback(error)
} else if (error != null) {
logger.err(
{
err: error,
project_id: request.project_id,
user_id: request.user_id,
},
'error writing resources to disk'
)
return callback(error)
}
logger.log(
{
project_id: request.project_id,
user_id: request.user_id,
time_taken: Date.now() - timer.start,
},
'written files to disk'
)
const syncStage = timer.done()
const injectDraftModeIfRequired = function (callback) {
if (request.draft) {
return DraftModeManager.injectDraftMode(
Path.join(compileDir, request.rootResourcePath),
callback
)
} else {
return callback()
}
}
const createTikzFileIfRequired = callback =>
TikzManager.checkMainFile(
compileDir,
request.rootResourcePath,
resourceList,
function (error, needsMainFile) {
if (error != null) {
return callback(error)
}
if (needsMainFile) {
return TikzManager.injectOutputFile(
compileDir,
request.rootResourcePath,
callback
)
} else {
return callback()
}
}
)
// set up environment variables for chktex
const env = {}
if (Settings.texliveOpenoutAny && Settings.texliveOpenoutAny !== '') {
// override default texlive openout_any environment variable
env.openout_any = Settings.texliveOpenoutAny
}
// only run chktex on LaTeX files (not knitr .Rtex files or any others)
const isLaTeXFile =
request.rootResourcePath != null
? request.rootResourcePath.match(/\.tex$/i)
: undefined
if (request.check != null && isLaTeXFile) {
env.CHKTEX_OPTIONS = '-nall -e9 -e10 -w15 -w16'
env.CHKTEX_ULIMIT_OPTIONS = '-t 5 -v 64000'
if (request.check === 'error') {
env.CHKTEX_EXIT_ON_ERROR = 1
}
if (request.check === 'validate') {
env.CHKTEX_VALIDATE = 1
}
}
// apply a series of file modifications/creations for draft mode and tikz
return async.series(
[injectDraftModeIfRequired, createTikzFileIfRequired],
function (error) {
if (error != null) {
return callback(error)
}
timer = new Metrics.Timer('run-compile')
// find the image tag to log it as a metric, e.g. 2015.1 (convert . to - for graphite)
let tag =
__guard__(
__guard__(
request.imageName != null
? request.imageName.match(/:(.*)/)
: undefined,
x1 => x1[1]
),
x => x.replace(/\./g, '-')
) || 'default'
if (!request.project_id.match(/^[0-9a-f]{24}$/)) {
tag = 'other'
} // exclude smoke test
Metrics.inc('compiles')
Metrics.inc(`compiles-with-image.${tag}`)
const compileName = getCompileName(
request.project_id,
request.user_id
)
return LatexRunner.runLatex(
compileName,
{
directory: compileDir,
mainFile: request.rootResourcePath,
compiler: request.compiler,
timeout: request.timeout,
image: request.imageName,
flags: request.flags,
environment: env,
compileGroup: request.compileGroup,
},
function (error, output, stats, timings) {
// request was for validation only
let metric_key, metric_value
if (request.check === 'validate') {
const result = (error != null ? error.code : undefined)
? 'fail'
: 'pass'
error = new Error('validation')
error.validate = result
}
// request was for compile, and failed on validation
if (
request.check === 'error' &&
(error != null ? error.message : undefined) === 'exited'
) {
error = new Error('compilation')
error.validate = 'fail'
}
// compile was killed by user, was a validation, or a compile which failed validation
if (
(error != null ? error.terminated : undefined) ||
(error != null ? error.validate : undefined) ||
(error != null ? error.timedout : undefined)
) {
OutputFileFinder.findOutputFiles(
resourceList,
compileDir,
function (err, outputFiles) {
if (err != null) {
return callback(err)
}
error.outputFiles = outputFiles // return output files so user can check logs
return callback(error)
}
)
return
}
// compile completed normally
if (error != null) {
return callback(error)
}
Metrics.inc('compiles-succeeded')
stats = stats || {}
const object = stats || {}
for (metric_key in object) {
metric_value = object[metric_key]
Metrics.count(metric_key, metric_value)
}
timings = timings || {}
const object1 = timings || {}
for (metric_key in object1) {
metric_value = object1[metric_key]
Metrics.timing(metric_key, metric_value)
}
const loadavg =
typeof os.loadavg === 'function' ? os.loadavg() : undefined
if (loadavg != null) {
Metrics.gauge('load-avg', loadavg[0])
}
const ts = timer.done()
logger.log(
{
project_id: request.project_id,
user_id: request.user_id,
time_taken: ts,
stats,
timings,
loadavg,
},
'done compile'
)
if ((stats != null ? stats['latex-runs'] : undefined) > 0) {
Metrics.timing(
'run-compile-per-pass',
ts / stats['latex-runs']
)
}
if (
(stats != null ? stats['latex-runs'] : undefined) > 0 &&
(timings != null ? timings['cpu-time'] : undefined) > 0
) {
Metrics.timing(
'run-compile-cpu-time-per-pass',
timings['cpu-time'] / stats['latex-runs']
)
}
// Emit compile time.
timings.compile = ts
timer = new Metrics.Timer('process-output-files')
return OutputFileFinder.findOutputFiles(
resourceList,
compileDir,
function (error, outputFiles) {
if (error != null) {
return callback(error)
}
return OutputCacheManager.saveOutputFiles(
{ request, stats, timings },
outputFiles,
compileDir,
outputDir,
(err, newOutputFiles) => {
if (err) {
const { project_id: projectId, user_id: userId } =
request
logger.err(
{ projectId, userId, err },
'failed to save output files'
)
}
const outputStage = timer.done()
timings.sync = syncStage
timings.output = outputStage
// Emit e2e compile time.
timings.compileE2E = timerE2E.done()
if (stats['pdf-size']) {
emitPdfStats(stats, timings)
}
callback(null, newOutputFiles, stats, timings)
}
)
}
)
}
)
}
)
}
)
},
stopCompile(project_id, user_id, callback) {
if (callback == null) {
callback = function (error) {}
}
const compileName = getCompileName(project_id, user_id)
return LatexRunner.killLatex(compileName, callback)
},
clearProject(project_id, user_id, _callback) {
if (_callback == null) {
_callback = function (error) {}
}
const callback = function (error) {
_callback(error)
return (_callback = function () {})
}
const compileDir = getCompileDir(project_id, user_id)
const outputDir = getOutputDir(project_id, user_id)
return CompileManager._checkDirectory(compileDir, function (err, exists) {
if (err != null) {
return callback(err)
}
if (!exists) {
return callback()
} // skip removal if no directory present
const proc = child_process.spawn('rm', [
'-r',
'-f',
'--',
compileDir,
outputDir,
])
proc.on('error', callback)
let stderr = ''
proc.stderr.setEncoding('utf8').on('data', chunk => (stderr += chunk))
return proc.on('close', function (code) {
if (code === 0) {
return callback(null)
} else {
return callback(
new Error(`rm -r ${compileDir} ${outputDir} failed: ${stderr}`)
)
}
})
})
},
_findAllDirs(callback) {
if (callback == null) {
callback = function (error, allDirs) {}
}
const root = Settings.path.compilesDir
return fs.readdir(root, function (err, files) {
if (err != null) {
return callback(err)
}
const allDirs = Array.from(files).map(file => Path.join(root, file))
return callback(null, allDirs)
})
},
clearExpiredProjects(max_cache_age_ms, callback) {
if (callback == null) {
callback = function (error) {}
}
const now = Date.now()
// action for each directory
const expireIfNeeded = (checkDir, cb) =>
fs.stat(checkDir, function (err, stats) {
if (err != null) {
return cb()
} // ignore errors checking directory
const age = now - stats.mtime
const hasExpired = age > max_cache_age_ms
if (hasExpired) {
return fse.remove(checkDir, cb)
} else {
return cb()
}
})
// iterate over all project directories
return CompileManager._findAllDirs(function (error, allDirs) {
if (error != null) {
return callback()
}
return async.eachSeries(allDirs, expireIfNeeded, callback)
})
},
_checkDirectory(compileDir, callback) {
if (callback == null) {
callback = function (error, exists) {}
}
return fs.lstat(compileDir, function (err, stats) {
if ((err != null ? err.code : undefined) === 'ENOENT') {
return callback(null, false) // directory does not exist
} else if (err != null) {
logger.err(
{ dir: compileDir, err },
'error on stat of project directory for removal'
)
return callback(err)
} else if (!(stats != null ? stats.isDirectory() : undefined)) {
logger.err(
{ dir: compileDir, stats },
'bad project directory for removal'
)
return callback(new Error('project directory is not directory'))
} else {
return callback(null, true)
}
})
}, // directory exists
syncFromCode(
project_id,
user_id,
file_name,
line,
column,
imageName,
callback
) {
// If LaTeX was run in a virtual environment, the file path that synctex expects
// might not match the file path on the host. The .synctex.gz file however, will be accessed
// wherever it is on the host.
if (callback == null) {
callback = function (error, pdfPositions) {}
}
const compileName = getCompileName(project_id, user_id)
const base_dir = Settings.path.synctexBaseDir(compileName)
const file_path = base_dir + '/' + file_name
const compileDir = getCompileDir(project_id, user_id)
const synctex_path = `${base_dir}/output.pdf`
const command = ['code', synctex_path, file_path, line, column]
CompileManager._runSynctex(
project_id,
user_id,
command,
imageName,
function (error, stdout) {
if (error != null) {
return callback(error)
}
logger.log(
{ project_id, user_id, file_name, line, column, command, stdout },
'synctex code output'
)
return callback(
null,
CompileManager._parseSynctexFromCodeOutput(stdout)
)
}
)
},
syncFromPdf(project_id, user_id, page, h, v, imageName, callback) {
if (callback == null) {
callback = function (error, filePositions) {}
}
const compileName = getCompileName(project_id, user_id)
const compileDir = getCompileDir(project_id, user_id)
const base_dir = Settings.path.synctexBaseDir(compileName)
const synctex_path = `${base_dir}/output.pdf`
const command = ['pdf', synctex_path, page, h, v]
CompileManager._runSynctex(
project_id,
user_id,
command,
imageName,
function (error, stdout) {
if (error != null) {
return callback(error)
}
logger.log(
{ project_id, user_id, page, h, v, stdout },
'synctex pdf output'
)
return callback(
null,
CompileManager._parseSynctexFromPdfOutput(stdout, base_dir)
)
}
)
},
_checkFileExists(dir, filename, callback) {
if (callback == null) {
callback = function (error) {}
}
const file = Path.join(dir, filename)
return fs.stat(dir, function (error, stats) {
if ((error != null ? error.code : undefined) === 'ENOENT') {
return callback(new Errors.NotFoundError('no output directory'))
}
if (error != null) {
return callback(error)
}
return fs.stat(file, function (error, stats) {
if ((error != null ? error.code : undefined) === 'ENOENT') {
return callback(new Errors.NotFoundError('no output file'))
}
if (error != null) {
return callback(error)
}
if (!(stats != null ? stats.isFile() : undefined)) {
return callback(new Error('not a file'))
}
return callback()
})
})
},
_runSynctex(project_id, user_id, command, imageName, callback) {
if (callback == null) {
callback = function (error, stdout) {}
}
const seconds = 1000
command.unshift('/opt/synctex')
const directory = getCompileDir(project_id, user_id)
const timeout = 60 * 1000 // increased to allow for large projects
const compileName = getCompileName(project_id, user_id)
const compileGroup = 'synctex'
CompileManager._checkFileExists(directory, 'output.synctex.gz', error => {
if (error) {
return callback(error)
}
return CommandRunner.run(
compileName,
command,
directory,
imageName ||
(Settings.clsi && Settings.clsi.docker
? Settings.clsi.docker.image
: undefined),
timeout,
{},
compileGroup,
function (error, output) {
if (error != null) {
logger.err(
{ err: error, command, project_id, user_id },
'error running synctex'
)
return callback(error)
}
return callback(null, output.stdout)
}
)
})
},
_parseSynctexFromCodeOutput(output) {
const results = []
for (const line of Array.from(output.split('\n'))) {
const [node, page, h, v, width, height] = Array.from(line.split('\t'))
if (node === 'NODE') {
results.push({
page: parseInt(page, 10),
h: parseFloat(h),
v: parseFloat(v),
height: parseFloat(height),
width: parseFloat(width),
})
}
}
return results
},
_parseSynctexFromPdfOutput(output, base_dir) {
const results = []
for (let line of Array.from(output.split('\n'))) {
let column, file_path, node
;[node, file_path, line, column] = Array.from(line.split('\t'))
if (node === 'NODE') {
const file = file_path.slice(base_dir.length + 1)
results.push({
file,
line: parseInt(line, 10),
column: parseInt(column, 10),
})
}
}
return results
},
wordcount(project_id, user_id, file_name, image, callback) {
if (callback == null) {
callback = function (error, pdfPositions) {}
}
logger.log({ project_id, user_id, file_name, image }, 'running wordcount')
const file_path = `$COMPILE_DIR/${file_name}`
const command = [
'texcount',
'-nocol',
'-inc',
file_path,
`-out=${file_path}.wc`,
]
const compileDir = getCompileDir(project_id, user_id)
const timeout = 60 * 1000
const compileName = getCompileName(project_id, user_id)
const compileGroup = 'wordcount'
return fse.ensureDir(compileDir, function (error) {
if (error != null) {
logger.err(
{ error, project_id, user_id, file_name },
'error ensuring dir for sync from code'
)
return callback(error)
}
return CommandRunner.run(
compileName,
command,
compileDir,
image,
timeout,
{},
compileGroup,
function (error) {
if (error != null) {
return callback(error)
}
return fs.readFile(
compileDir + '/' + file_name + '.wc',
'utf-8',
function (err, stdout) {
if (err != null) {
// call it node_err so sentry doesn't use random path error as unique id so it can't be ignored
logger.err(
{ node_err: err, command, compileDir, project_id, user_id },
'error reading word count output'
)
return callback(err)
}
const results = CompileManager._parseWordcountFromOutput(stdout)
logger.log(
{ project_id, user_id, wordcount: results },
'word count results'
)
return callback(null, results)
}
)
}
)
})
},
_parseWordcountFromOutput(output) {
const results = {
encode: '',
textWords: 0,
headWords: 0,
outside: 0,
headers: 0,
elements: 0,
mathInline: 0,
mathDisplay: 0,
errors: 0,
messages: '',
}
for (const line of Array.from(output.split('\n'))) {
const [data, info] = Array.from(line.split(':'))
if (data.indexOf('Encoding') > -1) {
results.encode = info.trim()
}
if (data.indexOf('in text') > -1) {
results.textWords = parseInt(info, 10)
}
if (data.indexOf('in head') > -1) {
results.headWords = parseInt(info, 10)
}
if (data.indexOf('outside') > -1) {
results.outside = parseInt(info, 10)
}
if (data.indexOf('of head') > -1) {
results.headers = parseInt(info, 10)
}
if (data.indexOf('Number of floats/tables/figures') > -1) {
results.elements = parseInt(info, 10)
}
if (data.indexOf('Number of math inlines') > -1) {
results.mathInline = parseInt(info, 10)
}
if (data.indexOf('Number of math displayed') > -1) {
results.mathDisplay = parseInt(info, 10)
}
if (data === '(errors') {
// errors reported as (errors:123)
results.errors = parseInt(info, 10)
}
if (line.indexOf('!!! ') > -1) {
// errors logged as !!! message !!!
results.messages += line + '\n'
}
}
return results
},
}
function __guard__(value, transform) {
return typeof value !== 'undefined' && value !== null
? transform(value)
: undefined
}

273
services/clsi/app/js/ContentCacheManager.js Normal file
View file

@ -0,0 +1,273 @@
/**
* ContentCacheManager - maintains a cache of stream hashes from a PDF file
*/
const { callbackify } = require('util')
const fs = require('fs')
const crypto = require('crypto')
const Path = require('path')
const Settings = require('@overleaf/settings')
const OError = require('@overleaf/o-error')
const pLimit = require('p-limit')
const { parseXrefTable } = require('../lib/pdfjs/parseXrefTable')
const { TimedOutError } = require('./Errors')
/**
*
* @param {String} contentDir path to directory where content hash files are cached
* @param {String} filePath the pdf file to scan for streams
* @param {number} size the pdf size
* @param {number} compileTime
*/
async function update(contentDir, filePath, size, compileTime) {
const checkDeadline = getDeadlineChecker(compileTime)
const ranges = []
const newRanges = []
// keep track of hashes expire old ones when they reach a generation > N.
const tracker = await HashFileTracker.from(contentDir)
tracker.updateAge()
checkDeadline('after init HashFileTracker')
const rawTable = await parseXrefTable(filePath, size, checkDeadline)
rawTable.sort((a, b) => {
return a.offset - b.offset
})
rawTable.forEach((obj, idx) => {
obj.idx = idx
})
checkDeadline('after parsing')
const uncompressedObjects = []
for (const object of rawTable) {
if (!object.uncompressed) {
continue
}
const nextObject = rawTable[object.idx + 1]
if (!nextObject) {
// Ignore this possible edge case.
// The last object should be part of the xRef table.
continue
} else {
object.endOffset = nextObject.offset
}
const size = object.endOffset - object.offset
object.size = size
if (size < Settings.pdfCachingMinChunkSize) {
continue
}
uncompressedObjects.push({ object, idx: uncompressedObjects.length })
}
checkDeadline('after finding uncompressed')
const handle = await fs.promises.open(filePath)
try {
for (const { object, idx } of uncompressedObjects) {
let buffer = Buffer.alloc(object.size, 0)
const { bytesRead } = await handle.read(
buffer,
0,
object.size,
object.offset
)
checkDeadline('after read ' + idx)
if (bytesRead !== object.size) {
throw new OError('could not read full chunk', {
object,
bytesRead,
})
}
const idxObj = buffer.indexOf('obj')
if (idxObj > 100) {
throw new OError('objectId is too large', {
object,
idxObj,
})
}
const objectIdRaw = buffer.subarray(0, idxObj)
buffer = buffer.subarray(objectIdRaw.byteLength)
const hash = pdfStreamHash(buffer)
checkDeadline('after hash ' + idx)
const range = {
objectId: objectIdRaw.toString(),
start: object.offset + objectIdRaw.byteLength,
end: object.endOffset,
hash,
}
ranges.push(range)
// Optimization: Skip writing of duplicate streams.
if (tracker.track(range)) continue
await writePdfStream(contentDir, hash, buffer)
checkDeadline('after write ' + idx)
newRanges.push(range)
}
} finally {
await handle.close()
}
// NOTE: Bailing out below does not make sense.
// Let the next compile use the already written ranges.
const reclaimedSpace = await tracker.deleteStaleHashes(5)
await tracker.flush()
return [ranges, newRanges, reclaimedSpace]
}
function getStatePath(contentDir) {
return Path.join(contentDir, '.state.v0.json')
}
class HashFileTracker {
constructor(contentDir, { hashAge = [], hashSize = [] }) {
this.contentDir = contentDir
this.hashAge = new Map(hashAge)
this.hashSize = new Map(hashSize)
}
static async from(contentDir) {
const statePath = getStatePath(contentDir)
let state = {}
try {
const blob = await fs.promises.readFile(statePath)
state = JSON.parse(blob)
} catch (e) {}
return new HashFileTracker(contentDir, state)
}
track(range) {
const exists = this.hashAge.has(range.hash)
if (!exists) {
this.hashSize.set(range.hash, range.end - range.start)
}
this.hashAge.set(range.hash, 0)
return exists
}
updateAge() {
for (const [hash, age] of this.hashAge) {
this.hashAge.set(hash, age + 1)
}
return this
}
findStale(maxAge) {
const stale = []
for (const [hash, age] of this.hashAge) {
if (age > maxAge) {
stale.push(hash)
}
}
return stale
}
async flush() {
const statePath = getStatePath(this.contentDir)
const blob = JSON.stringify({
hashAge: Array.from(this.hashAge.entries()),
hashSize: Array.from(this.hashSize.entries()),
})
const atomicWrite = statePath + '~'
try {
await fs.promises.writeFile(atomicWrite, blob)
} catch (err) {
try {
await fs.promises.unlink(atomicWrite)
} catch (e) {}
throw err
}
try {
await fs.promises.rename(atomicWrite, statePath)
} catch (err) {
try {
await fs.promises.unlink(atomicWrite)
} catch (e) {}
throw err
}
}
async deleteStaleHashes(n) {
// delete any hash file older than N generations
const hashes = this.findStale(n)
let reclaimedSpace = 0
if (hashes.length === 0) {
return reclaimedSpace
}
await promiseMapWithLimit(10, hashes, async hash => {
await fs.promises.unlink(Path.join(this.contentDir, hash))
this.hashAge.delete(hash)
reclaimedSpace += this.hashSize.get(hash)
this.hashSize.delete(hash)
})
return reclaimedSpace
}
}
function pdfStreamHash(buffer) {
const hash = crypto.createHash('sha256')
hash.update(buffer)
return hash.digest('hex')
}
async function writePdfStream(dir, hash, buffer) {
const filename = Path.join(dir, hash)
const atomicWriteFilename = filename + '~'
if (Settings.enablePdfCachingDark) {
// Write an empty file in dark mode.
buffer = Buffer.alloc(0)
}
try {
await fs.promises.writeFile(atomicWriteFilename, buffer)
await fs.promises.rename(atomicWriteFilename, filename)
} catch (err) {
try {
await fs.promises.unlink(atomicWriteFilename)
} catch (_) {
throw err
}
}
}
function getDeadlineChecker(compileTime) {
const maxOverhead = Math.min(
// Adding 10s to a 40s compile time is OK.
// Adding 1s to a 3s compile time is OK.
Math.max(compileTime / 4, 1000),
// Adding 30s to a 120s compile time is not OK, limit to 10s.
Settings.pdfCachingMaxProcessingTime
)
const deadline = Date.now() + maxOverhead
let lastStage = { stage: 'start', now: Date.now() }
let completedStages = 0
return function (stage) {
const now = Date.now()
if (now > deadline) {
throw new TimedOutError(stage, {
completedStages,
lastStage: lastStage.stage,
diffToLastStage: now - lastStage.now,
})
}
completedStages++
lastStage = { stage, now }
}
}
function promiseMapWithLimit(concurrency, array, fn) {
const limit = pLimit(concurrency)
return Promise.all(array.map(x => limit(() => fn(x))))
}
module.exports = {
HASH_REGEX: /^[0-9a-f]{64}$/,
update: callbackify(update),
promises: {
update,
},
}

115
services/clsi/app/js/ContentCacheMetrics.js Normal file
View file

@ -0,0 +1,115 @@
const logger = require('logger-sharelatex')
const Metrics = require('./Metrics')
const os = require('os')
let CACHED_LOAD = {
expires: -1,
load: [0, 0, 0],
}
function getSystemLoad() {
if (CACHED_LOAD.expires < Date.now()) {
CACHED_LOAD = {
expires: Date.now() + 10 * 1000,
load: os.loadavg(),
}
}
return CACHED_LOAD.load
}
const ONE_MB = 1024 * 1024
function emitPdfStats(stats, timings) {
if (stats['pdf-caching-timed-out']) {
Metrics.inc('pdf-caching-timed-out')
}
if (timings['compute-pdf-caching']) {
emitPdfCachingStats(stats, timings)
} else {
// How much bandwidth will the pdf incur when downloaded in full?
Metrics.summary('pdf-bandwidth', stats['pdf-size'])
}
}
function emitPdfCachingStats(stats, timings) {
if (!stats['pdf-size']) return // double check
// How much extra time did we spent in PDF.js?
Metrics.timing('compute-pdf-caching', timings['compute-pdf-caching'])
// How large is the overhead of hashing up-front?
const fraction =
timings.compileE2E - timings['compute-pdf-caching'] !== 0
? timings.compileE2E /
(timings.compileE2E - timings['compute-pdf-caching'])
: 1
if (fraction > 1.5 && timings.compileE2E > 10 * 1000) {
logger.warn(
{
stats,
timings,
load: getSystemLoad(),
},
'slow pdf caching'
)
}
Metrics.summary('overhead-compute-pdf-ranges', fraction * 100 - 100)
// How does the hashing scale to pdf size in MB?
Metrics.timing(
'compute-pdf-caching-relative-to-pdf-size',
timings['compute-pdf-caching'] / (stats['pdf-size'] / ONE_MB)
)
if (stats['pdf-caching-total-ranges-size']) {
// How does the hashing scale to total ranges size in MB?
Metrics.timing(
'compute-pdf-caching-relative-to-total-ranges-size',
timings['compute-pdf-caching'] /
(stats['pdf-caching-total-ranges-size'] / ONE_MB)
)
// How fast is the hashing per range on average?
Metrics.timing(
'compute-pdf-caching-relative-to-ranges-count',
timings['compute-pdf-caching'] / stats['pdf-caching-n-ranges']
)
// How many ranges are new?
Metrics.summary(
'new-pdf-ranges-relative-to-total-ranges',
(stats['pdf-caching-n-new-ranges'] / stats['pdf-caching-n-ranges']) * 100
)
}
// How much content is cacheable?
Metrics.summary(
'cacheable-ranges-to-pdf-size',
(stats['pdf-caching-total-ranges-size'] / stats['pdf-size']) * 100
)
const sizeWhenDownloadedInFull =
// All of the pdf
stats['pdf-size'] -
// These ranges are potentially cached.
stats['pdf-caching-total-ranges-size'] +
// These ranges are not cached.
stats['pdf-caching-new-ranges-size']
// How much bandwidth can we save when downloading the pdf in full?
Metrics.summary(
'pdf-bandwidth-savings',
100 - (sizeWhenDownloadedInFull / stats['pdf-size']) * 100
)
// How much bandwidth will the pdf incur when downloaded in full?
Metrics.summary('pdf-bandwidth', sizeWhenDownloadedInFull)
// How much space do the ranges use?
// This will accumulate the ranges size over time, skipping already written ranges.
Metrics.summary(
'pdf-ranges-disk-size',
stats['pdf-caching-new-ranges-size'] - stats['pdf-caching-reclaimed-space']
)
}
module.exports = {
emitPdfStats,
}

24
services/clsi/app/js/ContentController.js Normal file
View file

@ -0,0 +1,24 @@
const Path = require('path')
const send = require('send')
const Settings = require('@overleaf/settings')
const OutputCacheManager = require('./OutputCacheManager')
const ONE_DAY_S = 24 * 60 * 60
const ONE_DAY_MS = ONE_DAY_S * 1000
function getPdfRange(req, res, next) {
const { projectId, userId, contentId, hash } = req.params
const perUserDir = userId ? `${projectId}-${userId}` : projectId
const path = Path.join(
Settings.path.outputDir,
perUserDir,
OutputCacheManager.CONTENT_SUBDIR,
contentId,
hash
)
res.setHeader('cache-control', `public, max-age=${ONE_DAY_S}`)
res.setHeader('expires', new Date(Date.now() + ONE_DAY_MS).toUTCString())
send(req, path).pipe(res)
}
module.exports = { getPdfRange }

38
services/clsi/app/js/ContentTypeMapper.js Normal file
View file

@ -0,0 +1,38 @@
/* eslint-disable
no-unused-vars,
*/
// TODO: This file was created by bulk-decaffeinate.
// Fix any style issues and re-enable lint.
let ContentTypeMapper
const Path = require('path')
// here we coerce html, css and js to text/plain,
// otherwise choose correct mime type based on file extension,
// falling back to octet-stream
module.exports = ContentTypeMapper = {
map(path) {
switch (Path.extname(path)) {
case '.txt':
case '.html':
case '.js':
case '.css':
case '.svg':
return 'text/plain'
case '.csv':
return 'text/csv'
case '.pdf':
return 'application/pdf'
case '.png':
return 'image/png'
case '.jpg':
case '.jpeg':
return 'image/jpeg'
case '.tiff':
return 'image/tiff'
case '.gif':
return 'image/gif'
default:
return 'application/octet-stream'
}
},
}

18
services/clsi/app/js/DbQueue.js Normal file
View file

@ -0,0 +1,18 @@
// TODO: This file was created by bulk-decaffeinate.
// Sanity-check the conversion and remove this comment.
/*
* decaffeinate suggestions:
* DS102: Remove unnecessary code created because of implicit returns
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
const async = require('async')
const Settings = require('@overleaf/settings')
const logger = require('logger-sharelatex')
const queue = async.queue(
(task, cb) => task(cb),
Settings.parallelSqlQueryLimit
)
queue.drain = () => logger.debug('all items have been processed')
module.exports = { queue }

113
services/clsi/app/js/DockerLockManager.js Normal file
View file

@ -0,0 +1,113 @@
/* eslint-disable
handle-callback-err,
*/
// TODO: This file was created by bulk-decaffeinate.
// Fix any style issues and re-enable lint.
/*
* decaffeinate suggestions:
* DS101: Remove unnecessary use of Array.from
* DS102: Remove unnecessary code created because of implicit returns
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
let LockManager
const logger = require('logger-sharelatex')
const LockState = {} // locks for docker container operations, by container name
module.exports = LockManager = {
MAX_LOCK_HOLD_TIME: 15000, // how long we can keep a lock
MAX_LOCK_WAIT_TIME: 10000, // how long we wait for a lock
LOCK_TEST_INTERVAL: 1000, // retry time
tryLock(key, callback) {
let lockValue
if (callback == null) {
callback = function (err, gotLock) {}
}
const existingLock = LockState[key]
if (existingLock != null) {
// the lock is already taken, check how old it is
const lockAge = Date.now() - existingLock.created
if (lockAge < LockManager.MAX_LOCK_HOLD_TIME) {
return callback(null, false) // we didn't get the lock, bail out
} else {
logger.error(
{ key, lock: existingLock, age: lockAge },
'taking old lock by force'
)
}
}
// take the lock
LockState[key] = lockValue = { created: Date.now() }
return callback(null, true, lockValue)
},
getLock(key, callback) {
let attempt
if (callback == null) {
callback = function (error, lockValue) {}
}
const startTime = Date.now()
return (attempt = () =>
LockManager.tryLock(key, function (error, gotLock, lockValue) {
if (error != null) {
return callback(error)
}
if (gotLock) {
return callback(null, lockValue)
} else if (Date.now() - startTime > LockManager.MAX_LOCK_WAIT_TIME) {
const e = new Error('Lock timeout')
e.key = key
return callback(e)
} else {
return setTimeout(attempt, LockManager.LOCK_TEST_INTERVAL)
}
}))()
},
releaseLock(key, lockValue, callback) {
if (callback == null) {
callback = function (error) {}
}
const existingLock = LockState[key]
if (existingLock === lockValue) {
// lockValue is an object, so we can test by reference
delete LockState[key] // our lock, so we can free it
return callback()
} else if (existingLock != null) {
// lock exists but doesn't match ours
logger.error(
{ key, lock: existingLock },
'tried to release lock taken by force'
)
return callback()
} else {
logger.error(
{ key, lock: existingLock },
'tried to release lock that has gone'
)
return callback()
}
},
runWithLock(key, runner, callback) {
if (callback == null) {
callback = function (error) {}
}
return LockManager.getLock(key, function (error, lockValue) {
if (error != null) {
return callback(error)
}
return runner((error1, ...args) =>
LockManager.releaseLock(key, lockValue, function (error2) {
error = error1 || error2
if (error != null) {
return callback(error)
}
return callback(null, ...Array.from(args))
})
)
})
},
}

625
services/clsi/app/js/DockerRunner.js Normal file
View file

@ -0,0 +1,625 @@
const Settings = require('@overleaf/settings')
const logger = require('logger-sharelatex')
const Docker = require('dockerode')
const dockerode = new Docker()
const crypto = require('crypto')
const async = require('async')
const LockManager = require('./DockerLockManager')
const fs = require('fs')
const Path = require('path')
const _ = require('lodash')
const ONE_HOUR_IN_MS = 60 * 60 * 1000
logger.info('using docker runner')
function usingSiblingContainers() {
return (
Settings != null &&
Settings.path != null &&
Settings.path.sandboxedCompilesHostDir != null
)
}
let containerMonitorTimeout
let containerMonitorInterval
const DockerRunner = {
run(
projectId,
command,
directory,
image,
timeout,
environment,
compileGroup,
callback
) {
if (usingSiblingContainers()) {
const _newPath = Settings.path.sandboxedCompilesHostDir
logger.log(
{ path: _newPath },
'altering bind path for sibling containers'
)
// Server Pro, example:
// '/var/lib/sharelatex/data/compiles/<project-id>'
// ... becomes ...
// '/opt/sharelatex_data/data/compiles/<project-id>'
directory = Path.join(
Settings.path.sandboxedCompilesHostDir,
Path.basename(directory)
)
}
const volumes = { [directory]: '/compile' }
command = command.map(arg =>
arg.toString().replace('$COMPILE_DIR', '/compile')
)
if (image == null) {
image = Settings.clsi.docker.image
}
if (
Settings.clsi.docker.allowedImages &&
!Settings.clsi.docker.allowedImages.includes(image)
) {
return callback(new Error('image not allowed'))
}
if (Settings.texliveImageNameOveride != null) {
const img = image.split('/')
image = `${Settings.texliveImageNameOveride}/${img[2]}`
}
const options = DockerRunner._getContainerOptions(
command,
image,
volumes,
timeout,
environment,
compileGroup
)
const fingerprint = DockerRunner._fingerprintContainer(options)
const name = `project-${projectId}-${fingerprint}`
options.name = name
// logOptions = _.clone(options)
// logOptions?.HostConfig?.SecurityOpt = "secomp used, removed in logging"
logger.log({ projectId }, 'running docker container')
DockerRunner._runAndWaitForContainer(
options,
volumes,
timeout,
(error, output) => {
if (error && error.statusCode === 500) {
logger.log(
{ err: error, projectId },
'error running container so destroying and retrying'
)
DockerRunner.destroyContainer(name, null, true, error => {
if (error != null) {
return callback(error)
}
DockerRunner._runAndWaitForContainer(
options,
volumes,
timeout,
callback
)
})
} else {
callback(error, output)
}
}
)
// pass back the container name to allow it to be killed
return name
},
kill(containerId, callback) {
logger.log({ containerId }, 'sending kill signal to container')
const container = dockerode.getContainer(containerId)
container.kill(error => {
if (
error != null &&
error.message != null &&
error.message.match(/Cannot kill container .* is not running/)
) {
logger.warn(
{ err: error, containerId },
'container not running, continuing'
)
error = null
}
if (error != null) {
logger.error({ err: error, containerId }, 'error killing container')
callback(error)
} else {
callback()
}
})
},
_runAndWaitForContainer(options, volumes, timeout, _callback) {
const callback = _.once(_callback)
const { name } = options
let streamEnded = false
let containerReturned = false
let output = {}
function callbackIfFinished() {
if (streamEnded && containerReturned) {
callback(null, output)
}
}
function attachStreamHandler(error, _output) {
if (error != null) {
return callback(error)
}
output = _output
streamEnded = true
callbackIfFinished()
}
DockerRunner.startContainer(
options,
volumes,
attachStreamHandler,
(error, containerId) => {
if (error != null) {
return callback(error)
}
DockerRunner.waitForContainer(name, timeout, (error, exitCode) => {
if (error != null) {
return callback(error)
}
if (exitCode === 137) {
// exit status from kill -9
const err = new Error('terminated')
err.terminated = true
return callback(err)
}
if (exitCode === 1) {
// exit status from chktex
const err = new Error('exited')
err.code = exitCode
return callback(err)
}
containerReturned = true
if (options != null && options.HostConfig != null) {
options.HostConfig.SecurityOpt = null
}
logger.log({ exitCode, options }, 'docker container has exited')
callbackIfFinished()
})
}
)
},
_getContainerOptions(
command,
image,
volumes,
timeout,
environment,
compileGroup
) {
const timeoutInSeconds = timeout / 1000
const dockerVolumes = {}
for (const hostVol in volumes) {
const dockerVol = volumes[hostVol]
dockerVolumes[dockerVol] = {}
if (volumes[hostVol].slice(-3).indexOf(':r') === -1) {
volumes[hostVol] = `${dockerVol}:rw`
}
}
// merge settings and environment parameter
const env = {}
for (const src of [Settings.clsi.docker.env, environment || {}]) {
for (const key in src) {
const value = src[key]
env[key] = value
}
}
// set the path based on the image year
const match = image.match(/:([0-9]+)\.[0-9]+/)
const year = match ? match[1] : '2014'
env.PATH = `/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/texlive/${year}/bin/x86_64-linux/`
const options = {
Cmd: command,
Image: image,
Volumes: dockerVolumes,
WorkingDir: '/compile',
NetworkDisabled: true,
Memory: 1024 * 1024 * 1024 * 1024, // 1 Gb
User: Settings.clsi.docker.user,
Env: Object.entries(env).map(([key, value]) => `${key}=${value}`),
HostConfig: {
Binds: Object.entries(volumes).map(
([hostVol, dockerVol]) => `${hostVol}:${dockerVol}`
),
LogConfig: { Type: 'none', Config: {} },
Ulimits: [
{
Name: 'cpu',
Soft: timeoutInSeconds + 5,
Hard: timeoutInSeconds + 10,
},
],
CapDrop: 'ALL',
SecurityOpt: ['no-new-privileges'],
},
}
if (Settings.path != null && Settings.path.synctexBinHostPath != null) {
options.HostConfig.Binds.push(
`${Settings.path.synctexBinHostPath}:/opt/synctex:ro`
)
}
if (Settings.clsi.docker.seccomp_profile != null) {
options.HostConfig.SecurityOpt.push(
`seccomp=${Settings.clsi.docker.seccomp_profile}`
)
}
if (Settings.clsi.docker.apparmor_profile != null) {
options.HostConfig.SecurityOpt.push(
`apparmor=${Settings.clsi.docker.apparmor_profile}`
)
}
if (Settings.clsi.docker.runtime) {
options.HostConfig.Runtime = Settings.clsi.docker.runtime
}
if (Settings.clsi.docker.Readonly) {
options.HostConfig.ReadonlyRootfs = true
options.HostConfig.Tmpfs = { '/tmp': 'rw,noexec,nosuid,size=65536k' }
options.Volumes['/home/tex'] = {}
}
// Allow per-compile group overriding of individual settings
if (
Settings.clsi.docker.compileGroupConfig &&
Settings.clsi.docker.compileGroupConfig[compileGroup]
) {
const override = Settings.clsi.docker.compileGroupConfig[compileGroup]
for (const key in override) {
_.set(options, key, override[key])
}
}
return options
},
_fingerprintContainer(containerOptions) {
// Yay, Hashing!
const json = JSON.stringify(containerOptions)
return crypto.createHash('md5').update(json).digest('hex')
},
startContainer(options, volumes, attachStreamHandler, callback) {
LockManager.runWithLock(
options.name,
releaseLock =>
// Check that volumes exist before starting the container.
// When a container is started with volume pointing to a
// non-existent directory then docker creates the directory but
// with root ownership.
DockerRunner._checkVolumes(options, volumes, err => {
if (err != null) {
return releaseLock(err)
}
DockerRunner._startContainer(
options,
volumes,
attachStreamHandler,
releaseLock
)
}),
callback
)
},
// Check that volumes exist and are directories
_checkVolumes(options, volumes, callback) {
if (usingSiblingContainers()) {
// Server Pro, with sibling-containers active, skip checks
return callback(null)
}
const checkVolume = (path, cb) =>
fs.stat(path, (err, stats) => {
if (err != null) {
return cb(err)
}
if (!stats.isDirectory()) {
return cb(new Error('not a directory'))
}
cb()
})
const jobs = []
for (const vol in volumes) {
jobs.push(cb => checkVolume(vol, cb))
}
async.series(jobs, callback)
},
_startContainer(options, volumes, attachStreamHandler, callback) {
callback = _.once(callback)
const { name } = options
logger.log({ container_name: name }, 'starting container')
const container = dockerode.getContainer(name)
function createAndStartContainer() {
dockerode.createContainer(options, (error, container) => {
if (error != null) {
return callback(error)
}
startExistingContainer()
})
}
function startExistingContainer() {
DockerRunner.attachToContainer(
options.name,
attachStreamHandler,
error => {
if (error != null) {
return callback(error)
}
container.start(error => {
if (error != null && error.statusCode !== 304) {
callback(error)
} else {
// already running
callback()
}
})
}
)
}
container.inspect((error, stats) => {
if (error != null && error.statusCode === 404) {
createAndStartContainer()
} else if (error != null) {
logger.err(
{ container_name: name, error },
'unable to inspect container to start'
)
callback(error)
} else {
startExistingContainer()
}
})
},
attachToContainer(containerId, attachStreamHandler, attachStartCallback) {
const container = dockerode.getContainer(containerId)
container.attach({ stdout: 1, stderr: 1, stream: 1 }, (error, stream) => {
if (error != null) {
logger.error(
{ err: error, containerId },
'error attaching to container'
)
return attachStartCallback(error)
} else {
attachStartCallback()
}
logger.log({ containerId }, 'attached to container')
const MAX_OUTPUT = 1024 * 1024 // limit output to 1MB
function createStringOutputStream(name) {
return {
data: '',
overflowed: false,
write(data) {
if (this.overflowed) {
return
}
if (this.data.length < MAX_OUTPUT) {
this.data += data
} else {
logger.error(
{
containerId,
length: this.data.length,
maxLen: MAX_OUTPUT,
},
`${name} exceeds max size`
)
this.data += `(...truncated at ${MAX_OUTPUT} chars...)`
this.overflowed = true
}
},
// kill container if too much output
// docker.containers.kill(containerId, () ->)
}
}
const stdout = createStringOutputStream('stdout')
const stderr = createStringOutputStream('stderr')
container.modem.demuxStream(stream, stdout, stderr)
stream.on('error', err =>
logger.error(
{ err, containerId },
'error reading from container stream'
)
)
stream.on('end', () =>
attachStreamHandler(null, { stdout: stdout.data, stderr: stderr.data })
)
})
},
waitForContainer(containerId, timeout, _callback) {
const callback = _.once(_callback)
const container = dockerode.getContainer(containerId)
let timedOut = false
const timeoutId = setTimeout(() => {
timedOut = true
logger.log({ containerId }, 'timeout reached, killing container')
container.kill(err => {
logger.warn({ err, containerId }, 'failed to kill container')
})
}, timeout)
logger.log({ containerId }, 'waiting for docker container')
container.wait((error, res) => {
if (error != null) {
clearTimeout(timeoutId)
logger.error({ err: error, containerId }, 'error waiting for container')
return callback(error)
}
if (timedOut) {
logger.log({ containerId }, 'docker container timed out')
error = new Error('container timed out')
error.timedout = true
callback(error)
} else {
clearTimeout(timeoutId)
logger.log(
{ containerId, exitCode: res.StatusCode },
'docker container returned'
)
callback(null, res.StatusCode)
}
})
},
destroyContainer(containerName, containerId, shouldForce, callback) {
// We want the containerName for the lock and, ideally, the
// containerId to delete. There is a bug in the docker.io module
// where if you delete by name and there is an error, it throws an
// async exception, but if you delete by id it just does a normal
// error callback. We fall back to deleting by name if no id is
// supplied.
LockManager.runWithLock(
containerName,
releaseLock =>
DockerRunner._destroyContainer(
containerId || containerName,
shouldForce,
releaseLock
),
callback
)
},
_destroyContainer(containerId, shouldForce, callback) {
logger.log({ containerId }, 'destroying docker container')
const container = dockerode.getContainer(containerId)
container.remove({ force: shouldForce === true, v: true }, error => {
if (error != null && error.statusCode === 404) {
logger.warn(
{ err: error, containerId },
'container not found, continuing'
)
error = null
}
if (error != null) {
logger.error({ err: error, containerId }, 'error destroying container')
} else {
logger.log({ containerId }, 'destroyed container')
}
callback(error)
})
},
// handle expiry of docker containers
MAX_CONTAINER_AGE: Settings.clsi.docker.maxContainerAge || ONE_HOUR_IN_MS,
examineOldContainer(container, callback) {
const name = container.Name || (container.Names && container.Names[0])
const created = container.Created * 1000 // creation time is returned in seconds
const now = Date.now()
const age = now - created
const maxAge = DockerRunner.MAX_CONTAINER_AGE
const ttl = maxAge - age
logger.log(
{ containerName: name, created, now, age, maxAge, ttl },
'checking whether to destroy container'
)
return { name, id: container.Id, ttl }
},
destroyOldContainers(callback) {
dockerode.listContainers({ all: true }, (error, containers) => {
if (error != null) {
return callback(error)
}
const jobs = []
for (const container of containers) {
const { name, id, ttl } = DockerRunner.examineOldContainer(container)
if (name.slice(0, 9) === '/project-' && ttl <= 0) {
// strip the / prefix
// the LockManager uses the plain container name
const plainName = name.slice(1)
jobs.push(cb =>
DockerRunner.destroyContainer(plainName, id, false, () => cb())
)
}
}
// Ignore errors because some containers get stuck but
// will be destroyed next time
async.series(jobs, callback)
})
},
startContainerMonitor() {
logger.log(
{ maxAge: DockerRunner.MAX_CONTAINER_AGE },
'starting container expiry'
)
// guarantee only one monitor is running
DockerRunner.stopContainerMonitor()
// randomise the start time
const randomDelay = Math.floor(Math.random() * 5 * 60 * 1000)
containerMonitorTimeout = setTimeout(() => {
containerMonitorInterval = setInterval(
() =>
DockerRunner.destroyOldContainers(err => {
if (err) {
logger.error({ err }, 'failed to destroy old containers')
}
}),
ONE_HOUR_IN_MS
)
}, randomDelay)
},
stopContainerMonitor() {
if (containerMonitorTimeout) {
clearTimeout(containerMonitorTimeout)
containerMonitorTimeout = undefined
}
if (containerMonitorInterval) {
clearInterval(containerMonitorInterval)
containerMonitorInterval = undefined
}
},
}
DockerRunner.startContainerMonitor()
module.exports = DockerRunner

57
services/clsi/app/js/DraftModeManager.js Normal file
View file

@ -0,0 +1,57 @@
/* eslint-disable
camelcase,
handle-callback-err,
no-useless-escape,
*/
// TODO: This file was created by bulk-decaffeinate.
// Fix any style issues and re-enable lint.
/*
* decaffeinate suggestions:
* DS102: Remove unnecessary code created because of implicit returns
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
let DraftModeManager
const fs = require('fs')
const logger = require('logger-sharelatex')
module.exports = DraftModeManager = {
injectDraftMode(filename, callback) {
if (callback == null) {
callback = function (error) {}
}
return fs.readFile(filename, 'utf8', function (error, content) {
if (error != null) {
return callback(error)
}
// avoid adding draft mode more than once
if (
(content != null
? content.indexOf('\\documentclass[draft')
: undefined) >= 0
) {
return callback()
}
const modified_content = DraftModeManager._injectDraftOption(content)
logger.log(
{
content: content.slice(0, 1024), // \documentclass is normally v near the top
modified_content: modified_content.slice(0, 1024),
filename,
},
'injected draft class'
)
return fs.writeFile(filename, modified_content, callback)
})
},
_injectDraftOption(content) {
return (
content
// With existing options (must be first, otherwise both are applied)
.replace(/\\documentclass\[/g, '\\documentclass[draft,')
// Without existing options
.replace(/\\documentclass\{/g, '\\documentclass[draft]{')
)
},
}

41
services/clsi/app/js/Errors.js Normal file
View file

@ -0,0 +1,41 @@
/* eslint-disable
no-proto,
no-unused-vars,
*/
// TODO: This file was created by bulk-decaffeinate.
// Fix any style issues and re-enable lint.
const OError = require('@overleaf/o-error')
let Errors
var NotFoundError = function (message) {
const error = new Error(message)
error.name = 'NotFoundError'
error.__proto__ = NotFoundError.prototype
return error
}
NotFoundError.prototype.__proto__ = Error.prototype
var FilesOutOfSyncError = function (message) {
const error = new Error(message)
error.name = 'FilesOutOfSyncError'
error.__proto__ = FilesOutOfSyncError.prototype
return error
}
FilesOutOfSyncError.prototype.__proto__ = Error.prototype
var AlreadyCompilingError = function (message) {
const error = new Error(message)
error.name = 'AlreadyCompilingError'
error.__proto__ = AlreadyCompilingError.prototype
return error
}
AlreadyCompilingError.prototype.__proto__ = Error.prototype
class TimedOutError extends OError {}
module.exports = Errors = {
TimedOutError,
NotFoundError,
FilesOutOfSyncError,
AlreadyCompilingError,
}

237
services/clsi/app/js/LatexRunner.js Normal file
View file

@ -0,0 +1,237 @@
/* eslint-disable
camelcase,
handle-callback-err,
no-return-assign,
no-unused-vars,
*/
// TODO: This file was created by bulk-decaffeinate.
// Fix any style issues and re-enable lint.
/*
* decaffeinate suggestions:
* DS102: Remove unnecessary code created because of implicit returns
* DS103: Rewrite code to no longer use __guard__
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
let LatexRunner
const Path = require('path')
const Settings = require('@overleaf/settings')
const logger = require('logger-sharelatex')
const Metrics = require('./Metrics')
const CommandRunner = require('./CommandRunner')
const fs = require('fs')
const ProcessTable = {} // table of currently running jobs (pids or docker container names)
const TIME_V_METRICS = Object.entries({
'cpu-percent': /Percent of CPU this job got: (\d+)/m,
'cpu-time': /User time.*: (\d+.\d+)/m,
'sys-time': /System time.*: (\d+.\d+)/m,
})
module.exports = LatexRunner = {
runLatex(project_id, options, callback) {
let command
if (callback == null) {
callback = function (error) {}
}
let {
directory,
mainFile,
compiler,
timeout,
image,
environment,
flags,
compileGroup,
} = options
if (!compiler) {
compiler = 'pdflatex'
}
if (!timeout) {
timeout = 60000
} // milliseconds
logger.log(
{
directory,
compiler,
timeout,
mainFile,
environment,
flags,
compileGroup,
},
'starting compile'
)
// We want to run latexmk on the tex file which we will automatically
// generate from the Rtex/Rmd/md file.
mainFile = mainFile.replace(/\.(Rtex|md|Rmd)$/, '.tex')
if (compiler === 'pdflatex') {
command = LatexRunner._pdflatexCommand(mainFile, flags)
} else if (compiler === 'latex') {
command = LatexRunner._latexCommand(mainFile, flags)
} else if (compiler === 'xelatex') {
command = LatexRunner._xelatexCommand(mainFile, flags)
} else if (compiler === 'lualatex') {
command = LatexRunner._lualatexCommand(mainFile, flags)
} else {
return callback(new Error(`unknown compiler: ${compiler}`))
}
if (Settings.clsi != null ? Settings.clsi.strace : undefined) {
command = ['strace', '-o', 'strace', '-ff'].concat(command)
}
const id = `${project_id}` // record running project under this id
return (ProcessTable[id] = CommandRunner.run(
project_id,
command,
directory,
image,
timeout,
environment,
compileGroup,
function (error, output) {
delete ProcessTable[id]
if (error != null) {
return callback(error)
}
const runs =
__guard__(
__guard__(output != null ? output.stderr : undefined, x1 =>
x1.match(/^Run number \d+ of .*latex/gm)
),
x => x.length
) || 0
const failed =
__guard__(output != null ? output.stdout : undefined, x2 =>
x2.match(/^Latexmk: Errors/m)
) != null
? 1
: 0
// counters from latexmk output
const stats = {}
stats['latexmk-errors'] = failed
stats['latex-runs'] = runs
stats['latex-runs-with-errors'] = failed ? runs : 0
stats[`latex-runs-${runs}`] = 1
stats[`latex-runs-with-errors-${runs}`] = failed ? 1 : 0
// timing information from /usr/bin/time
const timings = {}
const stderr = (output && output.stderr) || ''
if (stderr.includes('Command being timed:')) {
// Add metrics for runs with `$ time -v ...`
for (const [timing, matcher] of TIME_V_METRICS) {
const match = stderr.match(matcher)
if (match) {
timings[timing] = parseFloat(match[1])
}
}
}
// record output files
LatexRunner.writeLogOutput(project_id, directory, output, () => {
return callback(error, output, stats, timings)
})
}
))
},
writeLogOutput(project_id, directory, output, callback) {
if (!output) {
return callback()
}
// internal method for writing non-empty log files
function _writeFile(file, content, cb) {
if (content && content.length > 0) {
fs.writeFile(file, content, err => {
if (err) {
logger.error({ project_id, file }, 'error writing log file') // don't fail on error
}
cb()
})
} else {
cb()
}
}
// write stdout and stderr, ignoring errors
_writeFile(Path.join(directory, 'output.stdout'), output.stdout, () => {
_writeFile(Path.join(directory, 'output.stderr'), output.stderr, () => {
callback()
})
})
},
killLatex(project_id, callback) {
if (callback == null) {
callback = function (error) {}
}
const id = `${project_id}`
logger.log({ id }, 'killing running compile')
if (ProcessTable[id] == null) {
logger.warn({ id }, 'no such project to kill')
return callback(null)
} else {
return CommandRunner.kill(ProcessTable[id], callback)
}
},
_latexmkBaseCommand(flags) {
let args = [
'latexmk',
'-cd',
'-f',
'-jobname=output',
'-auxdir=$COMPILE_DIR',
'-outdir=$COMPILE_DIR',
'-synctex=1',
'-interaction=batchmode',
]
if (flags) {
args = args.concat(flags)
}
return (
__guard__(
Settings != null ? Settings.clsi : undefined,
x => x.latexmkCommandPrefix
) || []
).concat(args)
},
_pdflatexCommand(mainFile, flags) {
return LatexRunner._latexmkBaseCommand(flags).concat([
'-pdf',
Path.join('$COMPILE_DIR', mainFile),
])
},
_latexCommand(mainFile, flags) {
return LatexRunner._latexmkBaseCommand(flags).concat([
'-pdfdvi',
Path.join('$COMPILE_DIR', mainFile),
])
},
_xelatexCommand(mainFile, flags) {
return LatexRunner._latexmkBaseCommand(flags).concat([
'-xelatex',
Path.join('$COMPILE_DIR', mainFile),
])
},
_lualatexCommand(mainFile, flags) {
return LatexRunner._latexmkBaseCommand(flags).concat([
'-lualatex',
Path.join('$COMPILE_DIR', mainFile),
])
},
}
function __guard__(value, transform) {
return typeof value !== 'undefined' && value !== null
? transform(value)
: undefined
}

103
services/clsi/app/js/LocalCommandRunner.js Normal file
View file

@ -0,0 +1,103 @@
/* eslint-disable
camelcase,
handle-callback-err,
no-return-assign,
no-unused-vars,
*/
// TODO: This file was created by bulk-decaffeinate.
// Fix any style issues and re-enable lint.
/*
* decaffeinate suggestions:
* DS101: Remove unnecessary use of Array.from
* DS102: Remove unnecessary code created because of implicit returns
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
let CommandRunner
const { spawn } = require('child_process')
const _ = require('lodash')
const logger = require('logger-sharelatex')
logger.info('using standard command runner')
module.exports = CommandRunner = {
run(
project_id,
command,
directory,
image,
timeout,
environment,
compileGroup,
callback
) {
let key, value
if (callback == null) {
callback = function (error) {}
} else {
callback = _.once(callback)
}
command = Array.from(command).map(arg =>
arg.toString().replace('$COMPILE_DIR', directory)
)
logger.log({ project_id, command, directory }, 'running command')
logger.warn('timeouts and sandboxing are not enabled with CommandRunner')
// merge environment settings
const env = {}
for (key in process.env) {
value = process.env[key]
env[key] = value
}
for (key in environment) {
value = environment[key]
env[key] = value
}
// run command as detached process so it has its own process group (which can be killed if needed)
const proc = spawn(command[0], command.slice(1), { cwd: directory, env })
let stdout = ''
proc.stdout.setEncoding('utf8').on('data', data => (stdout += data))
proc.on('error', function (err) {
logger.err(
{ err, project_id, command, directory },
'error running command'
)
return callback(err)
})
proc.on('close', function (code, signal) {
let err
logger.info({ code, signal, project_id }, 'command exited')
if (signal === 'SIGTERM') {
// signal from kill method below
err = new Error('terminated')
err.terminated = true
return callback(err)
} else if (code === 1) {
// exit status from chktex
err = new Error('exited')
err.code = code
return callback(err)
} else {
return callback(null, { stdout: stdout })
}
})
return proc.pid
}, // return process id to allow job to be killed if necessary
kill(pid, callback) {
if (callback == null) {
callback = function (error) {}
}
try {
process.kill(-pid) // kill all processes in group
} catch (err) {
return callback(err)
}
return callback()
},
}

72
services/clsi/app/js/LockManager.js Normal file
View file

@ -0,0 +1,72 @@
/* eslint-disable
handle-callback-err,
no-unused-vars,
*/
// TODO: This file was created by bulk-decaffeinate.
// Fix any style issues and re-enable lint.
/*
* decaffeinate suggestions:
* DS101: Remove unnecessary use of Array.from
* DS102: Remove unnecessary code created because of implicit returns
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
let LockManager
const Settings = require('@overleaf/settings')
const logger = require('logger-sharelatex')
const Lockfile = require('lockfile') // from https://github.com/npm/lockfile
const Errors = require('./Errors')
const fs = require('fs')
const Path = require('path')
module.exports = LockManager = {
LOCK_TEST_INTERVAL: 1000, // 50ms between each test of the lock
MAX_LOCK_WAIT_TIME: 15000, // 10s maximum time to spend trying to get the lock
LOCK_STALE: 5 * 60 * 1000, // 5 mins time until lock auto expires
runWithLock(path, runner, callback) {
if (callback == null) {
callback = function (error) {}
}
const lockOpts = {
wait: this.MAX_LOCK_WAIT_TIME,
pollPeriod: this.LOCK_TEST_INTERVAL,
stale: this.LOCK_STALE,
}
return Lockfile.lock(path, lockOpts, function (error) {
if ((error != null ? error.code : undefined) === 'EEXIST') {
return callback(new Errors.AlreadyCompilingError('compile in progress'))
} else if (error != null) {
return fs.lstat(path, (statLockErr, statLock) =>
fs.lstat(Path.dirname(path), (statDirErr, statDir) =>
fs.readdir(Path.dirname(path), function (readdirErr, readdirDir) {
logger.err(
{
error,
path,
statLock,
statLockErr,
statDir,
statDirErr,
readdirErr,
readdirDir,
},
'unable to get lock'
)
return callback(error)
})
)
)
} else {
return runner((error1, ...args) =>
Lockfile.unlock(path, function (error2) {
error = error1 || error2
if (error != null) {
return callback(error)
}
return callback(null, ...Array.from(args))
})
)
}
})
},
}

3
services/clsi/app/js/Metrics.js Normal file
View file

@ -0,0 +1,3 @@
// TODO: This file was created by bulk-decaffeinate.
// Sanity-check the conversion and remove this comment.
module.exports = require('@overleaf/metrics')

563
services/clsi/app/js/OutputCacheManager.js Normal file
View file

@ -0,0 +1,563 @@
/* eslint-disable
handle-callback-err,
*/
// TODO: This file was created by bulk-decaffeinate.
// Fix any style issues and re-enable lint.
/*
* decaffeinate suggestions:
* DS101: Remove unnecessary use of Array.from
* DS102: Remove unnecessary code created because of implicit returns
* DS103: Rewrite code to no longer use __guard__
* DS104: Avoid inline assignments
* DS204: Change includes calls to have a more natural evaluation order
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
let OutputCacheManager
const async = require('async')
const fs = require('fs')
const fse = require('fs-extra')
const Path = require('path')
const logger = require('logger-sharelatex')
const _ = require('lodash')
const Settings = require('@overleaf/settings')
const crypto = require('crypto')
const Metrics = require('./Metrics')
const OutputFileOptimiser = require('./OutputFileOptimiser')
const ContentCacheManager = require('./ContentCacheManager')
const { TimedOutError } = require('./Errors')
module.exports = OutputCacheManager = {
CONTENT_SUBDIR: 'content',
CACHE_SUBDIR: 'generated-files',
ARCHIVE_SUBDIR: 'archived-logs',
// build id is HEXDATE-HEXRANDOM from Date.now()and RandomBytes
// for backwards compatibility, make the randombytes part optional
BUILD_REGEX: /^[0-9a-f]+(-[0-9a-f]+)?$/,
CONTENT_REGEX: /^[0-9a-f]+(-[0-9a-f]+)?$/,
CACHE_LIMIT: 2, // maximum number of cache directories
CACHE_AGE: 60 * 60 * 1000, // up to one hour old
path(buildId, file) {
// used by static server, given build id return '.cache/clsi/buildId'
if (buildId.match(OutputCacheManager.BUILD_REGEX)) {
return Path.join(OutputCacheManager.CACHE_SUBDIR, buildId, file)
} else {
// for invalid build id, return top level
return file
}
},
generateBuildId(callback) {
// generate a secure build id from Date.now() and 8 random bytes in hex
if (callback == null) {
callback = function (error, buildId) {}
}
return crypto.randomBytes(8, function (err, buf) {
if (err != null) {
return callback(err)
}
const random = buf.toString('hex')
const date = Date.now().toString(16)
return callback(err, `${date}-${random}`)
})
},
saveOutputFiles(
{ request, stats, timings },
outputFiles,
compileDir,
outputDir,
callback
) {
if (callback == null) {
callback = function (error) {}
}
return OutputCacheManager.generateBuildId(function (err, buildId) {
if (err != null) {
return callback(err)
}
return OutputCacheManager.saveOutputFilesInBuildDir(
outputFiles,
compileDir,
outputDir,
buildId,
function (err, result) {
if (err != null) {
return callback(err)
}
OutputCacheManager.collectOutputPdfSize(
result,
outputDir,
stats,
(err, result) => {
if (err) return callback(err, result)
if (!Settings.enablePdfCaching || !request.enablePdfCaching) {
return callback(null, result)
}
OutputCacheManager.saveStreamsInContentDir(
{ stats, timings },
result,
compileDir,
outputDir,
callback
)
}
)
}
)
})
},
saveOutputFilesInBuildDir(
outputFiles,
compileDir,
outputDir,
buildId,
callback
) {
// make a compileDir/CACHE_SUBDIR/build_id directory and
// copy all the output files into it
if (callback == null) {
callback = function (error) {}
}
const cacheRoot = Path.join(outputDir, OutputCacheManager.CACHE_SUBDIR)
// Put the files into a new cache subdirectory
const cacheDir = Path.join(
outputDir,
OutputCacheManager.CACHE_SUBDIR,
buildId
)
// Is it a per-user compile? check if compile directory is PROJECTID-USERID
const perUser = Path.basename(compileDir).match(
/^[0-9a-f]{24}-[0-9a-f]{24}$/
)
// Archive logs in background
if (
(Settings.clsi != null ? Settings.clsi.archive_logs : undefined) ||
(Settings.clsi != null ? Settings.clsi.strace : undefined)
) {
OutputCacheManager.archiveLogs(
outputFiles,
compileDir,
outputDir,
buildId,
function (err) {
if (err != null) {
return logger.warn({ err }, 'erroring archiving log files')
}
}
)
}
// make the new cache directory
return fse.ensureDir(cacheDir, function (err) {
if (err != null) {
logger.error(
{ err, directory: cacheDir },
'error creating cache directory'
)
return callback(err, outputFiles)
} else {
// copy all the output files into the new cache directory
const results = []
return async.mapSeries(
outputFiles,
function (file, cb) {
// don't send dot files as output, express doesn't serve them
if (OutputCacheManager._fileIsHidden(file.path)) {
logger.debug(
{ compileDir, path: file.path },
'ignoring dotfile in output'
)
return cb()
}
// copy other files into cache directory if valid
const newFile = _.clone(file)
const [src, dst] = Array.from([
Path.join(compileDir, file.path),
Path.join(cacheDir, file.path),
])
return OutputCacheManager._checkFileIsSafe(
src,
function (err, isSafe) {
if (err != null) {
return cb(err)
}
if (!isSafe) {
return cb()
}
return OutputCacheManager._checkIfShouldCopy(
src,
function (err, shouldCopy) {
if (err != null) {
return cb(err)
}
if (!shouldCopy) {
return cb()
}
return OutputCacheManager._copyFile(
src,
dst,
function (err) {
if (err != null) {
return cb(err)
}
newFile.build = buildId // attach a build id if we cached the file
results.push(newFile)
return cb()
}
)
}
)
}
)
},
function (err) {
if (err != null) {
// pass back the original files if we encountered *any* error
callback(err, outputFiles)
// clean up the directory we just created
return fse.remove(cacheDir, function (err) {
if (err != null) {
return logger.error(
{ err, dir: cacheDir },
'error removing cache dir after failure'
)
}
})
} else {
// pass back the list of new files in the cache
callback(err, results)
// let file expiry run in the background, expire all previous files if per-user
return OutputCacheManager.expireOutputFiles(cacheRoot, {
keep: buildId,
limit: perUser ? 1 : null,
})
}
}
)
}
})
},
collectOutputPdfSize(outputFiles, outputDir, stats, callback) {
const outputFile = outputFiles.find(x => x.path === 'output.pdf')
if (!outputFile) return callback(null, outputFiles)
const outputFilePath = Path.join(
outputDir,
OutputCacheManager.path(outputFile.build, outputFile.path)
)
fs.stat(outputFilePath, (err, stat) => {
if (err) return callback(err, outputFiles)
outputFile.size = stat.size
stats['pdf-size'] = outputFile.size
callback(null, outputFiles)
})
},
saveStreamsInContentDir(
{ stats, timings },
outputFiles,
compileDir,
outputDir,
callback
) {
const cacheRoot = Path.join(outputDir, OutputCacheManager.CONTENT_SUBDIR)
// check if content dir exists
OutputCacheManager.ensureContentDir(cacheRoot, function (err, contentDir) {
if (err) return callback(err, outputFiles)
const outputFile = outputFiles.find(x => x.path === 'output.pdf')
if (outputFile) {
// possibly we should copy the file from the build dir here
const outputFilePath = Path.join(
outputDir,
OutputCacheManager.path(outputFile.build, outputFile.path)
)
const pdfSize = outputFile.size
const timer = new Metrics.Timer('compute-pdf-ranges')
ContentCacheManager.update(
contentDir,
outputFilePath,
pdfSize,
timings.compile,
function (err, result) {
if (err && err instanceof TimedOutError) {
logger.warn(
{ err, outputDir, stats, timings },
'pdf caching timed out'
)
stats['pdf-caching-timed-out'] = 1
return callback(null, outputFiles)
}
if (err) return callback(err, outputFiles)
const [contentRanges, newContentRanges, reclaimedSpace] = result
if (Settings.enablePdfCachingDark) {
// In dark mode we are doing the computation only and do not emit
// any ranges to the frontend.
} else {
outputFile.contentId = Path.basename(contentDir)
outputFile.ranges = contentRanges
}
timings['compute-pdf-caching'] = timer.done()
stats['pdf-caching-n-ranges'] = contentRanges.length
stats['pdf-caching-total-ranges-size'] = contentRanges.reduce(
(sum, next) => sum + (next.end - next.start),
0
)
stats['pdf-caching-n-new-ranges'] = newContentRanges.length
stats['pdf-caching-new-ranges-size'] = newContentRanges.reduce(
(sum, next) => sum + (next.end - next.start),
0
)
stats['pdf-caching-reclaimed-space'] = reclaimedSpace
callback(null, outputFiles)
}
)
} else {
callback(null, outputFiles)
}
})
},
ensureContentDir(contentRoot, callback) {
fse.ensureDir(contentRoot, function (err) {
if (err != null) {
return callback(err)
}
fs.readdir(contentRoot, function (err, results) {
const dirs = results.sort()
const contentId = dirs.find(dir =>
OutputCacheManager.BUILD_REGEX.test(dir)
)
if (contentId) {
callback(null, Path.join(contentRoot, contentId))
} else {
// make a content directory
OutputCacheManager.generateBuildId(function (err, contentId) {
if (err) {
return callback(err)
}
const contentDir = Path.join(contentRoot, contentId)
fse.ensureDir(contentDir, function (err) {
if (err) {
return callback(err)
}
return callback(null, contentDir)
})
})
}
})
})
},
archiveLogs(outputFiles, compileDir, outputDir, buildId, callback) {
if (callback == null) {
callback = function (error) {}
}
const archiveDir = Path.join(
outputDir,
OutputCacheManager.ARCHIVE_SUBDIR,
buildId
)
logger.log({ dir: archiveDir }, 'archiving log files for project')
return fse.ensureDir(archiveDir, function (err) {
if (err != null) {
return callback(err)
}
return async.mapSeries(
outputFiles,
function (file, cb) {
const [src, dst] = Array.from([
Path.join(compileDir, file.path),
Path.join(archiveDir, file.path),
])
return OutputCacheManager._checkFileIsSafe(
src,
function (err, isSafe) {
if (err != null) {
return cb(err)
}
if (!isSafe) {
return cb()
}
return OutputCacheManager._checkIfShouldArchive(
src,
function (err, shouldArchive) {
if (err != null) {
return cb(err)
}
if (!shouldArchive) {
return cb()
}
return OutputCacheManager._copyFile(src, dst, cb)
}
)
}
)
},
callback
)
})
},
expireOutputFiles(cacheRoot, options, callback) {
// look in compileDir for build dirs and delete if > N or age of mod time > T
if (callback == null) {
callback = function (error) {}
}
return fs.readdir(cacheRoot, function (err, results) {
if (err != null) {
if (err.code === 'ENOENT') {
return callback(null)
} // cache directory is empty
logger.error({ err, project_id: cacheRoot }, 'error clearing cache')
return callback(err)
}
const dirs = results.sort().reverse()
const currentTime = Date.now()
const isExpired = function (dir, index) {
if ((options != null ? options.keep : undefined) === dir) {
return false
}
// remove any directories over the requested (non-null) limit
if (
(options != null ? options.limit : undefined) != null &&
index > options.limit
) {
return true
}
// remove any directories over the hard limit
if (index > OutputCacheManager.CACHE_LIMIT) {
return true
}
// we can get the build time from the first part of the directory name DDDD-RRRR
// DDDD is date and RRRR is random bytes
const dirTime = parseInt(
__guard__(dir.split('-'), x => x[0]),
16
)
const age = currentTime - dirTime
return age > OutputCacheManager.CACHE_AGE
}
const toRemove = _.filter(dirs, isExpired)
const removeDir = (dir, cb) =>
fse.remove(Path.join(cacheRoot, dir), function (err, result) {
logger.log({ cache: cacheRoot, dir }, 'removed expired cache dir')
if (err != null) {
logger.error({ err, dir }, 'cache remove error')
}
return cb(err, result)
})
return async.eachSeries(
toRemove,
(dir, cb) => removeDir(dir, cb),
callback
)
})
},
_fileIsHidden(path) {
return (path != null ? path.match(/^\.|\/\./) : undefined) != null
},
_checkFileIsSafe(src, callback) {
// check if we have a valid file to copy into the cache
if (callback == null) {
callback = function (error, isSafe) {}
}
return fs.stat(src, function (err, stats) {
if ((err != null ? err.code : undefined) === 'ENOENT') {
logger.warn(
{ err, file: src },
'file has disappeared before copying to build cache'
)
return callback(err, false)
} else if (err != null) {
// some other problem reading the file
logger.error({ err, file: src }, 'stat error for file in cache')
return callback(err, false)
} else if (!stats.isFile()) {
// other filetype - reject it
logger.warn(
{ src, stat: stats },
'nonfile output - refusing to copy to cache'
)
return callback(null, false)
} else {
// it's a plain file, ok to copy
return callback(null, true)
}
})
},
_copyFile(src, dst, callback) {
// copy output file into the cache
return fse.copy(src, dst, function (err) {
if ((err != null ? err.code : undefined) === 'ENOENT') {
logger.warn(
{ err, file: src },
'file has disappeared when copying to build cache'
)
return callback(err, false)
} else if (err != null) {
logger.error({ err, src, dst }, 'copy error for file in cache')
return callback(err)
} else {
if (
Settings.clsi != null ? Settings.clsi.optimiseInDocker : undefined
) {
// don't run any optimisations on the pdf when they are done
// in the docker container
return callback()
} else {
// call the optimiser for the file too
return OutputFileOptimiser.optimiseFile(src, dst, callback)
}
}
})
},
_checkIfShouldCopy(src, callback) {
if (callback == null) {
callback = function (err, shouldCopy) {}
}
return callback(null, !Path.basename(src).match(/^strace/))
},
_checkIfShouldArchive(src, callback) {
let needle
if (callback == null) {
callback = function (err, shouldCopy) {}
}
if (Path.basename(src).match(/^strace/)) {
return callback(null, true)
}
if (
(Settings.clsi != null ? Settings.clsi.archive_logs : undefined) &&
((needle = Path.basename(src)),
['output.log', 'output.blg'].includes(needle))
) {
return callback(null, true)
}
return callback(null, false)
},
}
function __guard__(value, transform) {
return typeof value !== 'undefined' && value !== null
? transform(value)
: undefined
}

78
services/clsi/app/js/OutputFileFinder.js Normal file
View file

@ -0,0 +1,78 @@
let OutputFileFinder
const Path = require('path')
const _ = require('lodash')
const { spawn } = require('child_process')
const logger = require('logger-sharelatex')
module.exports = OutputFileFinder = {
findOutputFiles(resources, directory, callback) {
const incomingResources = new Set(resources.map(resource => resource.path))
OutputFileFinder._getAllFiles(directory, function (error, allFiles) {
if (allFiles == null) {
allFiles = []
}
if (error) {
logger.err({ err: error }, 'error finding all output files')
return callback(error)
}
const outputFiles = []
for (const file of allFiles) {
if (!incomingResources.has(file)) {
outputFiles.push({
path: file,
type: Path.extname(file).replace(/^\./, '') || undefined,
})
}
}
callback(null, outputFiles, allFiles)
})
},
_getAllFiles(directory, callback) {
callback = _.once(callback)
// don't include clsi-specific files/directories in the output list
const EXCLUDE_DIRS = [
'-name',
'.cache',
'-o',
'-name',
'.archive',
'-o',
'-name',
'.project-*',
]
const args = [
directory,
'(',
...EXCLUDE_DIRS,
')',
'-prune',
'-o',
'-type',
'f',
'-print',
]
logger.log({ args }, 'running find command')
const proc = spawn('find', args)
let stdout = ''
proc.stdout.setEncoding('utf8').on('data', chunk => (stdout += chunk))
proc.on('error', callback)
proc.on('close', function (code) {
if (code !== 0) {
logger.warn(
{ directory, code },
"find returned error, directory likely doesn't exist"
)
return callback(null, [])
}
let fileList = stdout.trim().split('\n')
fileList = fileList.map(function (file) {
// Strip leading directory
return Path.relative(directory, file)
})
callback(null, fileList)
})
},
}

103
services/clsi/app/js/OutputFileOptimiser.js Normal file
View file

@ -0,0 +1,103 @@
/* eslint-disable
handle-callback-err,
no-return-assign,
no-undef,
no-unused-vars,
node/no-deprecated-api,
*/
// TODO: This file was created by bulk-decaffeinate.
// Fix any style issues and re-enable lint.
/*
* decaffeinate suggestions:
* DS102: Remove unnecessary code created because of implicit returns
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
let OutputFileOptimiser
const fs = require('fs')
const Path = require('path')
const { spawn } = require('child_process')
const logger = require('logger-sharelatex')
const Metrics = require('./Metrics')
const _ = require('lodash')
module.exports = OutputFileOptimiser = {
optimiseFile(src, dst, callback) {
// check output file (src) and see if we can optimise it, storing
// the result in the build directory (dst)
if (callback == null) {
callback = function (error) {}
}
if (src.match(/\/output\.pdf$/)) {
return OutputFileOptimiser.checkIfPDFIsOptimised(
src,
function (err, isOptimised) {
if (err != null || isOptimised) {
return callback(null)
}
return OutputFileOptimiser.optimisePDF(src, dst, callback)
}
)
} else {
return callback(null)
}
},
checkIfPDFIsOptimised(file, callback) {
const SIZE = 16 * 1024 // check the header of the pdf
const result = Buffer.alloc(SIZE) // fills with zeroes by default
return fs.open(file, 'r', function (err, fd) {
if (err != null) {
return callback(err)
}
return fs.read(fd, result, 0, SIZE, 0, (errRead, bytesRead, buffer) =>
fs.close(fd, function (errClose) {
if (errRead != null) {
return callback(errRead)
}
if (typeof errReadClose !== 'undefined' && errReadClose !== null) {
return callback(errClose)
}
const isOptimised =
buffer.toString('ascii').indexOf('/Linearized 1') >= 0
return callback(null, isOptimised)
})
)
})
},
optimisePDF(src, dst, callback) {
if (callback == null) {
callback = function (error) {}
}
const tmpOutput = dst + '.opt'
const args = ['--linearize', '--newline-before-endstream', src, tmpOutput]
logger.log({ args }, 'running qpdf command')
const timer = new Metrics.Timer('qpdf')
const proc = spawn('qpdf', args)
let stdout = ''
proc.stdout.setEncoding('utf8').on('data', chunk => (stdout += chunk))
callback = _.once(callback) // avoid double call back for error and close event
proc.on('error', function (err) {
logger.warn({ err, args }, 'qpdf failed')
return callback(null)
}) // ignore the error
return proc.on('close', function (code) {
timer.done()
if (code !== 0) {
logger.warn({ code, args }, 'qpdf returned error')
return callback(null) // ignore the error
}
return fs.rename(tmpOutput, dst, function (err) {
if (err != null) {
logger.warn(
{ tmpOutput, dst },
'failed to rename output of qpdf command'
)
}
return callback(null)
})
})
}, // ignore the error
}

207
services/clsi/app/js/ProjectPersistenceManager.js Normal file
View file

@ -0,0 +1,207 @@
/* eslint-disable
camelcase,
handle-callback-err,
*/
// TODO: This file was created by bulk-decaffeinate.
// Fix any style issues and re-enable lint.
/*
* decaffeinate suggestions:
* DS101: Remove unnecessary use of Array.from
* DS102: Remove unnecessary code created because of implicit returns
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
let ProjectPersistenceManager
const Metrics = require('./Metrics')
const UrlCache = require('./UrlCache')
const CompileManager = require('./CompileManager')
const db = require('./db')
const dbQueue = require('./DbQueue')
const async = require('async')
const logger = require('logger-sharelatex')
const oneDay = 24 * 60 * 60 * 1000
const Settings = require('@overleaf/settings')
const diskusage = require('diskusage')
const { callbackify } = require('util')
async function refreshExpiryTimeout() {
const paths = [
Settings.path.compilesDir,
Settings.path.outputDir,
Settings.path.clsiCacheDir,
]
for (const path of paths) {
try {
const stats = await diskusage.check(path)
const lowDisk = stats.available / stats.total < 0.1
const lowerExpiry = ProjectPersistenceManager.EXPIRY_TIMEOUT * 0.9
if (lowDisk && Settings.project_cache_length_ms / 2 < lowerExpiry) {
logger.warn(
{
stats,
newExpiryTimeoutInDays: (lowerExpiry / oneDay).toFixed(2),
},
'disk running low on space, modifying EXPIRY_TIMEOUT'
)
ProjectPersistenceManager.EXPIRY_TIMEOUT = lowerExpiry
break
}
} catch (err) {
logger.err({ err, path }, 'error getting disk usage')
}
}
}
module.exports = ProjectPersistenceManager = {
EXPIRY_TIMEOUT: Settings.project_cache_length_ms || oneDay * 2.5,
promises: {
refreshExpiryTimeout,
},
refreshExpiryTimeout: callbackify(refreshExpiryTimeout),
markProjectAsJustAccessed(project_id, callback) {
if (callback == null) {
callback = function (error) {}
}
const timer = new Metrics.Timer('db-bump-last-accessed')
const job = cb =>
db.Project.findOrCreate({ where: { project_id } })
.spread((project, created) =>
project
.update({ lastAccessed: new Date() })
.then(() => cb())
.error(cb)
)
.error(cb)
dbQueue.queue.push(job, error => {
timer.done()
callback(error)
})
},
clearExpiredProjects(callback) {
if (callback == null) {
callback = function (error) {}
}
return ProjectPersistenceManager._findExpiredProjectIds(function (
error,
project_ids
) {
if (error != null) {
return callback(error)
}
logger.log({ project_ids }, 'clearing expired projects')
const jobs = Array.from(project_ids || []).map(project_id =>
(
project_id => callback =>
ProjectPersistenceManager.clearProjectFromCache(
project_id,
function (err) {
if (err != null) {
logger.error({ err, project_id }, 'error clearing project')
}
return callback()
}
)
)(project_id)
)
return async.series(jobs, function (error) {
if (error != null) {
return callback(error)
}
return CompileManager.clearExpiredProjects(
ProjectPersistenceManager.EXPIRY_TIMEOUT,
error => callback()
)
})
})
}, // ignore any errors from deleting directories
clearProject(project_id, user_id, callback) {
if (callback == null) {
callback = function (error) {}
}
logger.log({ project_id, user_id }, 'clearing project for user')
return CompileManager.clearProject(project_id, user_id, function (error) {
if (error != null) {
return callback(error)
}
return ProjectPersistenceManager.clearProjectFromCache(
project_id,
function (error) {
if (error != null) {
return callback(error)
}
return callback()
}
)
})
},
clearProjectFromCache(project_id, callback) {
if (callback == null) {
callback = function (error) {}
}
logger.log({ project_id }, 'clearing project from cache')
return UrlCache.clearProject(project_id, function (error) {
if (error != null) {
logger.err({ error, project_id }, 'error clearing project from cache')
return callback(error)
}
return ProjectPersistenceManager._clearProjectFromDatabase(
project_id,
function (error) {
if (error != null) {
logger.err(
{ error, project_id },
'error clearing project from database'
)
}
return callback(error)
}
)
})
},
_clearProjectFromDatabase(project_id, callback) {
if (callback == null) {
callback = function (error) {}
}
logger.log({ project_id }, 'clearing project from database')
const job = cb =>
db.Project.destroy({ where: { project_id } })
.then(() => cb())
.error(cb)
return dbQueue.queue.push(job, callback)
},
_findExpiredProjectIds(callback) {
if (callback == null) {
callback = function (error, project_ids) {}
}
const job = function (cb) {
const keepProjectsFrom = new Date(
Date.now() - ProjectPersistenceManager.EXPIRY_TIMEOUT
)
const q = {}
q[db.op.lt] = keepProjectsFrom
return db.Project.findAll({ where: { lastAccessed: q } })
.then(projects =>
cb(
null,
projects.map(project => project.project_id)
)
)
.error(cb)
}
return dbQueue.queue.push(job, callback)
},
}
logger.log(
{ EXPIRY_TIMEOUT: ProjectPersistenceManager.EXPIRY_TIMEOUT },
'project assets kept timeout'
)

239
services/clsi/app/js/RequestParser.js Normal file
View file

@ -0,0 +1,239 @@
/* eslint-disable
handle-callback-err,
no-control-regex,
no-throw-literal,
no-unused-vars,
no-useless-escape,
valid-typeof,
*/
// TODO: This file was created by bulk-decaffeinate.
// Fix any style issues and re-enable lint.
/*
* decaffeinate suggestions:
* DS101: Remove unnecessary use of Array.from
* DS102: Remove unnecessary code created because of implicit returns
* DS205: Consider reworking code to avoid use of IIFEs
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
let RequestParser
const settings = require('@overleaf/settings')
module.exports = RequestParser = {
VALID_COMPILERS: ['pdflatex', 'latex', 'xelatex', 'lualatex'],
MAX_TIMEOUT: 600,
parse(body, callback) {
let resource
if (callback == null) {
callback = function (error, data) {}
}
const response = {}
if (body.compile == null) {
return callback('top level object should have a compile attribute')
}
const { compile } = body
if (!compile.options) {
compile.options = {}
}
try {
response.compiler = this._parseAttribute(
'compiler',
compile.options.compiler,
{
validValues: this.VALID_COMPILERS,
default: 'pdflatex',
type: 'string',
}
)
response.enablePdfCaching = this._parseAttribute(
'enablePdfCaching',
compile.options.enablePdfCaching,
{
default: false,
type: 'boolean',
}
)
response.timeout = this._parseAttribute(
'timeout',
compile.options.timeout,
{
default: RequestParser.MAX_TIMEOUT,
type: 'number',
}
)
response.imageName = this._parseAttribute(
'imageName',
compile.options.imageName,
{
type: 'string',
validValues:
settings.clsi &&
settings.clsi.docker &&
settings.clsi.docker.allowedImages,
}
)
response.draft = this._parseAttribute('draft', compile.options.draft, {
default: false,
type: 'boolean',
})
response.check = this._parseAttribute('check', compile.options.check, {
type: 'string',
})
response.flags = this._parseAttribute('flags', compile.options.flags, {
default: [],
type: 'object',
})
if (settings.allowedCompileGroups) {
response.compileGroup = this._parseAttribute(
'compileGroup',
compile.options.compileGroup,
{
validValues: settings.allowedCompileGroups,
default: '',
type: 'string',
}
)
}
// The syncType specifies whether the request contains all
// resources (full) or only those resources to be updated
// in-place (incremental).
response.syncType = this._parseAttribute(
'syncType',
compile.options.syncType,
{
validValues: ['full', 'incremental'],
type: 'string',
}
)
// The syncState is an identifier passed in with the request
// which has the property that it changes when any resource is
// added, deleted, moved or renamed.
//
// on syncType full the syncState identifier is passed in and
// stored
//
// on syncType incremental the syncState identifier must match
// the stored value
response.syncState = this._parseAttribute(
'syncState',
compile.options.syncState,
{ type: 'string' }
)
if (response.timeout > RequestParser.MAX_TIMEOUT) {
response.timeout = RequestParser.MAX_TIMEOUT
}
response.timeout = response.timeout * 1000 // milliseconds
response.resources = (() => {
const result = []
for (resource of Array.from(compile.resources || [])) {
result.push(this._parseResource(resource))
}
return result
})()
const rootResourcePath = this._parseAttribute(
'rootResourcePath',
compile.rootResourcePath,
{
default: 'main.tex',
type: 'string',
}
)
const originalRootResourcePath = rootResourcePath
const sanitizedRootResourcePath =
RequestParser._sanitizePath(rootResourcePath)
response.rootResourcePath = RequestParser._checkPath(
sanitizedRootResourcePath
)
for (resource of Array.from(response.resources)) {
if (resource.path === originalRootResourcePath) {
resource.path = sanitizedRootResourcePath
}
}
} catch (error1) {
const error = error1
return callback(error)
}
return callback(null, response)
},
_parseResource(resource) {
let modified
if (resource.path == null || typeof resource.path !== 'string') {
throw 'all resources should have a path attribute'
}
if (resource.modified != null) {
modified = new Date(resource.modified)
if (isNaN(modified.getTime())) {
throw `resource modified date could not be understood: ${resource.modified}`
}
}
if (resource.url == null && resource.content == null) {
throw 'all resources should have either a url or content attribute'
}
if (resource.content != null && typeof resource.content !== 'string') {
throw 'content attribute should be a string'
}
if (resource.url != null && typeof resource.url !== 'string') {
throw 'url attribute should be a string'
}
return {
path: resource.path,
modified,
url: resource.url,
content: resource.content,
}
},
_parseAttribute(name, attribute, options) {
if (attribute != null) {
if (options.validValues != null) {
if (options.validValues.indexOf(attribute) === -1) {
throw `${name} attribute should be one of: ${options.validValues.join(
', '
)}`
}
}
if (options.type != null) {
if (typeof attribute !== options.type) {
throw `${name} attribute should be a ${options.type}`
}
}
} else {
if (options.default != null) {
return options.default
}
}
return attribute
},
_sanitizePath(path) {
// See http://php.net/manual/en/function.escapeshellcmd.php
return path.replace(
/[\#\&\;\`\|\*\?\~\<\>\^\(\)\[\]\{\}\$\\\x0A\xFF\x00]/g,
''
)
},
_checkPath(path) {
// check that the request does not use a relative path
for (const dir of Array.from(path.split('/'))) {
if (dir === '..') {
throw 'relative path in root resource'
}
}
return path
},
}

116
services/clsi/app/js/ResourceStateManager.js Normal file
View file

@ -0,0 +1,116 @@
const Path = require('path')
const fs = require('fs')
const logger = require('logger-sharelatex')
const Errors = require('./Errors')
const SafeReader = require('./SafeReader')
module.exports = {
// The sync state is an identifier which must match for an
// incremental update to be allowed.
//
// The initial value is passed in and stored on a full
// compile, along with the list of resources..
//
// Subsequent incremental compiles must come with the same value - if
// not they will be rejected with a 409 Conflict response. The
// previous list of resources is returned.
//
// An incremental compile can only update existing files with new
// content. The sync state identifier must change if any docs or
// files are moved, added, deleted or renamed.
SYNC_STATE_FILE: '.project-sync-state',
SYNC_STATE_MAX_SIZE: 128 * 1024,
saveProjectState(state, resources, basePath, callback) {
const stateFile = Path.join(basePath, this.SYNC_STATE_FILE)
if (state == null) {
// remove the file if no state passed in
logger.log({ state, basePath }, 'clearing sync state')
fs.unlink(stateFile, function (err) {
if (err && err.code !== 'ENOENT') {
return callback(err)
} else {
return callback()
}
})
} else {
logger.log({ state, basePath }, 'writing sync state')
const resourceList = resources.map(resource => resource.path)
fs.writeFile(
stateFile,
[...resourceList, `stateHash:${state}`].join('\n'),
callback
)
}
},
checkProjectStateMatches(state, basePath, callback) {
const stateFile = Path.join(basePath, this.SYNC_STATE_FILE)
const size = this.SYNC_STATE_MAX_SIZE
SafeReader.readFile(
stateFile,
size,
'utf8',
function (err, result, bytesRead) {
if (err) {
return callback(err)
}
if (bytesRead === size) {
logger.error(
{ file: stateFile, size, bytesRead },
'project state file truncated'
)
}
const array = result ? result.toString().split('\n') : []
const adjustedLength = Math.max(array.length, 1)
const resourceList = array.slice(0, adjustedLength - 1)
const oldState = array[adjustedLength - 1]
const newState = `stateHash:${state}`
logger.log(
{ state, oldState, basePath, stateMatches: newState === oldState },
'checking sync state'
)
if (newState !== oldState) {
return callback(
new Errors.FilesOutOfSyncError(
'invalid state for incremental update'
)
)
} else {
const resources = resourceList.map(path => ({ path }))
callback(null, resources)
}
}
)
},
checkResourceFiles(resources, allFiles, basePath, callback) {
// check the paths are all relative to current directory
const containsRelativePath = resource => {
const dirs = resource.path.split('/')
return dirs.indexOf('..') !== -1
}
if (resources.some(containsRelativePath)) {
return callback(new Error('relative path in resource file list'))
}
// check if any of the input files are not present in list of files
const seenFiles = new Set(allFiles)
const missingFiles = resources
.map(resource => resource.path)
.filter(path => !seenFiles.has(path))
if (missingFiles.length > 0) {
logger.err(
{ missingFiles, basePath, allFiles, resources },
'missing input files for project'
)
return callback(
new Errors.FilesOutOfSyncError(
'resource files missing in incremental update'
)
)
} else {
callback()
}
},
}

354
services/clsi/app/js/ResourceWriter.js Normal file
View file

@ -0,0 +1,354 @@
/* eslint-disable
camelcase,
handle-callback-err,
no-return-assign,
no-unused-vars,
no-useless-escape,
*/
// TODO: This file was created by bulk-decaffeinate.
// Fix any style issues and re-enable lint.
/*
* decaffeinate suggestions:
* DS101: Remove unnecessary use of Array.from
* DS102: Remove unnecessary code created because of implicit returns
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
let ResourceWriter
const UrlCache = require('./UrlCache')
const Path = require('path')
const fs = require('fs')
const async = require('async')
const OutputFileFinder = require('./OutputFileFinder')
const ResourceStateManager = require('./ResourceStateManager')
const Metrics = require('./Metrics')
const logger = require('logger-sharelatex')
const settings = require('@overleaf/settings')
const parallelFileDownloads = settings.parallelFileDownloads || 1
module.exports = ResourceWriter = {
syncResourcesToDisk(request, basePath, callback) {
if (callback == null) {
callback = function (error, resourceList) {}
}
if (request.syncType === 'incremental') {
logger.log(
{ project_id: request.project_id, user_id: request.user_id },
'incremental sync'
)
return ResourceStateManager.checkProjectStateMatches(
request.syncState,
basePath,
function (error, resourceList) {
if (error != null) {
return callback(error)
}
return ResourceWriter._removeExtraneousFiles(
resourceList,
basePath,
function (error, outputFiles, allFiles) {
if (error != null) {
return callback(error)
}
return ResourceStateManager.checkResourceFiles(
resourceList,
allFiles,
basePath,
function (error) {
if (error != null) {
return callback(error)
}
return ResourceWriter.saveIncrementalResourcesToDisk(
request.project_id,
request.resources,
basePath,
function (error) {
if (error != null) {
return callback(error)
}
return callback(null, resourceList)
}
)
}
)
}
)
}
)
} else {
logger.log(
{ project_id: request.project_id, user_id: request.user_id },
'full sync'
)
return this.saveAllResourcesToDisk(
request.project_id,
request.resources,
basePath,
function (error) {
if (error != null) {
return callback(error)
}
return ResourceStateManager.saveProjectState(
request.syncState,
request.resources,
basePath,
function (error) {
if (error != null) {
return callback(error)
}
return callback(null, request.resources)
}
)
}
)
}
},
saveIncrementalResourcesToDisk(project_id, resources, basePath, callback) {
if (callback == null) {
callback = function (error) {}
}
return this._createDirectory(basePath, error => {
if (error != null) {
return callback(error)
}
const jobs = Array.from(resources).map(resource =>
(resource => {
return callback =>
this._writeResourceToDisk(project_id, resource, basePath, callback)
})(resource)
)
return async.parallelLimit(jobs, parallelFileDownloads, callback)
})
},
saveAllResourcesToDisk(project_id, resources, basePath, callback) {
if (callback == null) {
callback = function (error) {}
}
return this._createDirectory(basePath, error => {
if (error != null) {
return callback(error)
}
return this._removeExtraneousFiles(resources, basePath, error => {
if (error != null) {
return callback(error)
}
const jobs = Array.from(resources).map(resource =>
(resource => {
return callback =>
this._writeResourceToDisk(
project_id,
resource,
basePath,
callback
)
})(resource)
)
return async.parallelLimit(jobs, parallelFileDownloads, callback)
})
})
},
_createDirectory(basePath, callback) {
if (callback == null) {
callback = function (error) {}
}
return fs.mkdir(basePath, function (err) {
if (err != null) {
if (err.code === 'EEXIST') {
return callback()
} else {
logger.log({ err, dir: basePath }, 'error creating directory')
return callback(err)
}
} else {
return callback()
}
})
},
_removeExtraneousFiles(resources, basePath, _callback) {
if (_callback == null) {
_callback = function (error, outputFiles, allFiles) {}
}
const timer = new Metrics.Timer('unlink-output-files')
const callback = function (error, ...result) {
timer.done()
return _callback(error, ...Array.from(result))
}
return OutputFileFinder.findOutputFiles(
resources,
basePath,
function (error, outputFiles, allFiles) {
if (error != null) {
return callback(error)
}
const jobs = []
for (const file of Array.from(outputFiles || [])) {
;(function (file) {
const { path } = file
let should_delete = true
if (
path.match(/^output\./) ||
path.match(/\.aux$/) ||
path.match(/^cache\//)
) {
// knitr cache
should_delete = false
}
if (path.match(/^output-.*/)) {
// Tikz cached figures (default case)
should_delete = false
}
if (path.match(/\.(pdf|dpth|md5)$/)) {
// Tikz cached figures (by extension)
should_delete = false
}
if (
path.match(/\.(pygtex|pygstyle)$/) ||
path.match(/(^|\/)_minted-[^\/]+\//)
) {
// minted files/directory
should_delete = false
}
if (
path.match(/\.md\.tex$/) ||
path.match(/(^|\/)_markdown_[^\/]+\//)
) {
// markdown files/directory
should_delete = false
}
if (path.match(/-eps-converted-to\.pdf$/)) {
// Epstopdf generated files
should_delete = false
}
if (
path === 'output.pdf' ||
path === 'output.dvi' ||
path === 'output.log' ||
path === 'output.xdv' ||
path === 'output.stdout' ||
path === 'output.stderr'
) {
should_delete = true
}
if (path === 'output.tex') {
// created by TikzManager if present in output files
should_delete = true
}
if (should_delete) {
return jobs.push(callback =>
ResourceWriter._deleteFileIfNotDirectory(
Path.join(basePath, path),
callback
)
)
}
})(file)
}
return async.series(jobs, function (error) {
if (error != null) {
return callback(error)
}
return callback(null, outputFiles, allFiles)
})
}
)
},
_deleteFileIfNotDirectory(path, callback) {
if (callback == null) {
callback = function (error) {}
}
return fs.stat(path, function (error, stat) {
if (error != null && error.code === 'ENOENT') {
return callback()
} else if (error != null) {
logger.err(
{ err: error, path },
'error stating file in deleteFileIfNotDirectory'
)
return callback(error)
} else if (stat.isFile()) {
return fs.unlink(path, function (error) {
if (error != null) {
logger.err(
{ err: error, path },
'error removing file in deleteFileIfNotDirectory'
)
return callback(error)
} else {
return callback()
}
})
} else {
return callback()
}
})
},
_writeResourceToDisk(project_id, resource, basePath, callback) {
if (callback == null) {
callback = function (error) {}
}
return ResourceWriter.checkPath(
basePath,
resource.path,
function (error, path) {
if (error != null) {
return callback(error)
}
return fs.mkdir(
Path.dirname(path),
{ recursive: true },
function (error) {
if (error != null) {
return callback(error)
}
// TODO: Don't overwrite file if it hasn't been modified
if (resource.url != null) {
return UrlCache.downloadUrlToFile(
project_id,
resource.url,
path,
resource.modified,
function (err) {
if (err != null) {
logger.err(
{
err,
project_id,
path,
resource_url: resource.url,
modified: resource.modified,
},
'error downloading file for resources'
)
Metrics.inc('download-failed')
}
return callback()
}
) // try and continue compiling even if http resource can not be downloaded at this time
} else {
fs.writeFile(path, resource.content, callback)
}
}
)
}
)
},
checkPath(basePath, resourcePath, callback) {
const path = Path.normalize(Path.join(basePath, resourcePath))
if (path.slice(0, basePath.length + 1) !== basePath + '/') {
return callback(new Error('resource path is outside root directory'))
} else {
return callback(null, path)
}
},
}

63
services/clsi/app/js/SafeReader.js Normal file
View file

@ -0,0 +1,63 @@
/* eslint-disable
handle-callback-err,
no-unused-vars,
node/no-deprecated-api,
*/
// TODO: This file was created by bulk-decaffeinate.
// Fix any style issues and re-enable lint.
/*
* decaffeinate suggestions:
* DS101: Remove unnecessary use of Array.from
* DS102: Remove unnecessary code created because of implicit returns
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
let SafeReader
const fs = require('fs')
const logger = require('logger-sharelatex')
module.exports = SafeReader = {
// safely read up to size bytes from a file and return result as a
// string
readFile(file, size, encoding, callback) {
if (callback == null) {
callback = function (error, result) {}
}
return fs.open(file, 'r', function (err, fd) {
if (err != null && err.code === 'ENOENT') {
return callback()
}
if (err != null) {
return callback(err)
}
// safely return always closing the file
const callbackWithClose = (err, ...result) =>
fs.close(fd, function (err1) {
if (err != null) {
return callback(err)
}
if (err1 != null) {
return callback(err1)
}
return callback(null, ...Array.from(result))
})
const buff = Buffer.alloc(size) // fills with zeroes by default
return fs.read(
fd,
buff,
0,
buff.length,
0,
function (err, bytesRead, buffer) {
if (err != null) {
return callbackWithClose(err)
}
const result = buffer.toString(encoding, 0, bytesRead)
return callbackWithClose(null, result, bytesRead)
}
)
})
},
}

94
services/clsi/app/js/StaticServerForbidSymlinks.js Normal file
View file

@ -0,0 +1,94 @@
/* eslint-disable
camelcase,
no-cond-assign,
no-unused-vars,
node/no-deprecated-api,
*/
// TODO: This file was created by bulk-decaffeinate.
// Fix any style issues and re-enable lint.
/*
* decaffeinate suggestions:
* DS101: Remove unnecessary use of Array.from
* DS102: Remove unnecessary code created because of implicit returns
* DS103: Rewrite code to no longer use __guard__
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
let ForbidSymlinks
const Path = require('path')
const fs = require('fs')
const Settings = require('@overleaf/settings')
const logger = require('logger-sharelatex')
const url = require('url')
module.exports = ForbidSymlinks = function (staticFn, root, options) {
const expressStatic = staticFn(root, options)
const basePath = Path.resolve(root)
return function (req, res, next) {
let file, project_id, result
const path = __guard__(url.parse(req.url), x => x.pathname)
// check that the path is of the form /project_id_or_name/path/to/file.log
if ((result = path.match(/^\/?([a-zA-Z0-9_-]+)\/(.*)/))) {
project_id = result[1]
file = result[2]
} else {
logger.warn({ path }, 'unrecognized file request')
return res.sendStatus(404)
}
// check that the file does not use a relative path
for (const dir of Array.from(file.split('/'))) {
if (dir === '..') {
logger.warn({ path }, 'attempt to use a relative path')
return res.sendStatus(404)
}
}
// check that the requested path is normalized
const requestedFsPath = `${basePath}/${project_id}/${file}`
if (requestedFsPath !== Path.normalize(requestedFsPath)) {
logger.error(
{ path: requestedFsPath },
'requestedFsPath is not normalized'
)
return res.sendStatus(404)
}
// check that the requested path is not a symlink
return fs.realpath(requestedFsPath, function (err, realFsPath) {
if (err != null) {
if (err.code === 'ENOENT') {
return res.sendStatus(404)
} else {
logger.error(
{
err,
requestedFsPath,
realFsPath,
path: req.params[0],
project_id: req.params.project_id,
},
'error checking file access'
)
return res.sendStatus(500)
}
} else if (requestedFsPath !== realFsPath) {
logger.warn(
{
requestedFsPath,
realFsPath,
path: req.params[0],
project_id: req.params.project_id,
},
'trying to access a different file (symlink), aborting'
)
return res.sendStatus(404)
} else {
return expressStatic(req, res, next)
}
})
}
}
function __guard__(value, transform) {
return typeof value !== 'undefined' && value !== null
? transform(value)
: undefined
}

101
services/clsi/app/js/TikzManager.js Normal file
View file

@ -0,0 +1,101 @@
/* eslint-disable
handle-callback-err,
no-unused-vars,
*/
// TODO: This file was created by bulk-decaffeinate.
// Fix any style issues and re-enable lint.
/*
* decaffeinate suggestions:
* DS101: Remove unnecessary use of Array.from
* DS102: Remove unnecessary code created because of implicit returns
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
let TikzManager
const fs = require('fs')
const Path = require('path')
const ResourceWriter = require('./ResourceWriter')
const SafeReader = require('./SafeReader')
const logger = require('logger-sharelatex')
// for \tikzexternalize or pstool to work the main file needs to match the
// jobname. Since we set the -jobname to output, we have to create a
// copy of the main file as 'output.tex'.
module.exports = TikzManager = {
checkMainFile(compileDir, mainFile, resources, callback) {
// if there's already an output.tex file, we don't want to touch it
if (callback == null) {
callback = function (error, needsMainFile) {}
}
for (const resource of Array.from(resources)) {
if (resource.path === 'output.tex') {
logger.log({ compileDir, mainFile }, 'output.tex already in resources')
return callback(null, false)
}
}
// if there's no output.tex, see if we are using tikz/pgf or pstool in the main file
return ResourceWriter.checkPath(
compileDir,
mainFile,
function (error, path) {
if (error != null) {
return callback(error)
}
return SafeReader.readFile(
path,
65536,
'utf8',
function (error, content) {
if (error != null) {
return callback(error)
}
const usesTikzExternalize =
(content != null
? content.indexOf('\\tikzexternalize')
: undefined) >= 0
const usesPsTool =
(content != null ? content.indexOf('{pstool}') : undefined) >= 0
logger.log(
{ compileDir, mainFile, usesTikzExternalize, usesPsTool },
'checked for packages needing main file as output.tex'
)
const needsMainFile = usesTikzExternalize || usesPsTool
return callback(null, needsMainFile)
}
)
}
)
},
injectOutputFile(compileDir, mainFile, callback) {
if (callback == null) {
callback = function (error) {}
}
return ResourceWriter.checkPath(
compileDir,
mainFile,
function (error, path) {
if (error != null) {
return callback(error)
}
return fs.readFile(path, 'utf8', function (error, content) {
if (error != null) {
return callback(error)
}
logger.log(
{ compileDir, mainFile },
'copied file to output.tex as project uses packages which require it'
)
// use wx flag to ensure that output file does not already exist
return fs.writeFile(
Path.join(compileDir, 'output.tex'),
content,
{ flag: 'wx' },
callback
)
})
}
)
},
}

281
services/clsi/app/js/UrlCache.js Normal file
View file

@ -0,0 +1,281 @@
/* eslint-disable
camelcase,
handle-callback-err,
no-return-assign,
*/
// TODO: This file was created by bulk-decaffeinate.
// Fix any style issues and re-enable lint.
/*
* decaffeinate suggestions:
* DS101: Remove unnecessary use of Array.from
* DS102: Remove unnecessary code created because of implicit returns
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
let UrlCache
const db = require('./db')
const dbQueue = require('./DbQueue')
const UrlFetcher = require('./UrlFetcher')
const Settings = require('@overleaf/settings')
const crypto = require('crypto')
const fs = require('fs')
const logger = require('logger-sharelatex')
const async = require('async')
const Metrics = require('./Metrics')
module.exports = UrlCache = {
downloadUrlToFile(project_id, url, destPath, lastModified, callback) {
if (callback == null) {
callback = function (error) {}
}
return UrlCache._ensureUrlIsInCache(
project_id,
url,
lastModified,
(error, pathToCachedUrl) => {
if (error != null) {
return callback(error)
}
return fs.copyFile(pathToCachedUrl, destPath, function (error) {
if (error != null) {
logger.error(
{ err: error, from: pathToCachedUrl, to: destPath },
'error copying file from cache'
)
return UrlCache._clearUrlDetails(project_id, url, () =>
callback(error)
)
} else {
return callback(error)
}
})
}
)
},
clearProject(project_id, callback) {
if (callback == null) {
callback = function (error) {}
}
return UrlCache._findAllUrlsInProject(project_id, function (error, urls) {
logger.log(
{ project_id, url_count: urls.length },
'clearing project URLs'
)
if (error != null) {
return callback(error)
}
const jobs = Array.from(urls || []).map(url =>
(
url => callback =>
UrlCache._clearUrlFromCache(project_id, url, function (error) {
if (error != null) {
logger.error(
{ err: error, project_id, url },
'error clearing project URL'
)
}
return callback()
})
)(url)
)
return async.series(jobs, callback)
})
},
_ensureUrlIsInCache(project_id, url, lastModified, callback) {
if (callback == null) {
callback = function (error, pathOnDisk) {}
}
if (lastModified != null) {
// MYSQL only stores dates to an accuracy of a second but the incoming lastModified might have milliseconds.
// So round down to seconds
lastModified = new Date(Math.floor(lastModified.getTime() / 1000) * 1000)
}
return UrlCache._doesUrlNeedDownloading(
project_id,
url,
lastModified,
(error, needsDownloading) => {
if (error != null) {
return callback(error)
}
if (needsDownloading) {
logger.log({ url, lastModified }, 'downloading URL')
return UrlFetcher.pipeUrlToFileWithRetry(
url,
UrlCache._cacheFilePathForUrl(project_id, url),
error => {
if (error != null) {
return callback(error)
}
return UrlCache._updateOrCreateUrlDetails(
project_id,
url,
lastModified,
error => {
if (error != null) {
return callback(error)
}
return callback(
null,
UrlCache._cacheFilePathForUrl(project_id, url)
)
}
)
}
)
} else {
logger.log({ url, lastModified }, 'URL is up to date in cache')
return callback(null, UrlCache._cacheFilePathForUrl(project_id, url))
}
}
)
},
_doesUrlNeedDownloading(project_id, url, lastModified, callback) {
if (callback == null) {
callback = function (error, needsDownloading) {}
}
if (lastModified == null) {
return callback(null, true)
}
return UrlCache._findUrlDetails(
project_id,
url,
function (error, urlDetails) {
if (error != null) {
return callback(error)
}
if (
urlDetails == null ||
urlDetails.lastModified == null ||
urlDetails.lastModified.getTime() < lastModified.getTime()
) {
return callback(null, true)
} else {
return callback(null, false)
}
}
)
},
_cacheFileNameForUrl(project_id, url) {
return project_id + ':' + crypto.createHash('md5').update(url).digest('hex')
},
_cacheFilePathForUrl(project_id, url) {
return `${Settings.path.clsiCacheDir}/${UrlCache._cacheFileNameForUrl(
project_id,
url
)}`
},
_clearUrlFromCache(project_id, url, callback) {
if (callback == null) {
callback = function (error) {}
}
return UrlCache._clearUrlDetails(project_id, url, function (error) {
if (error != null) {
return callback(error)
}
return UrlCache._deleteUrlCacheFromDisk(
project_id,
url,
function (error) {
if (error != null) {
return callback(error)
}
return callback(null)
}
)
})
},
_deleteUrlCacheFromDisk(project_id, url, callback) {
if (callback == null) {
callback = function (error) {}
}
return fs.unlink(
UrlCache._cacheFilePathForUrl(project_id, url),
function (error) {
if (error != null && error.code !== 'ENOENT') {
// no error if the file isn't present
return callback(error)
} else {
return callback()
}
}
)
},
_findUrlDetails(project_id, url, callback) {
if (callback == null) {
callback = function (error, urlDetails) {}
}
const timer = new Metrics.Timer('db-find-url-details')
const job = cb =>
db.UrlCache.findOne({ where: { url, project_id } })
.then(urlDetails => cb(null, urlDetails))
.error(cb)
dbQueue.queue.push(job, (error, urlDetails) => {
timer.done()
callback(error, urlDetails)
})
},
_updateOrCreateUrlDetails(project_id, url, lastModified, callback) {
if (callback == null) {
callback = function (error) {}
}
const timer = new Metrics.Timer('db-update-or-create-url-details')
const job = cb =>
db.UrlCache.findOrCreate({ where: { url, project_id } })
.spread((urlDetails, created) =>
urlDetails
.update({ lastModified })
.then(() => cb())
.error(cb)
)
.error(cb)
dbQueue.queue.push(job, error => {
timer.done()
callback(error)
})
},
_clearUrlDetails(project_id, url, callback) {
if (callback == null) {
callback = function (error) {}
}
const timer = new Metrics.Timer('db-clear-url-details')
const job = cb =>
db.UrlCache.destroy({ where: { url, project_id } })
.then(() => cb(null))
.error(cb)
dbQueue.queue.push(job, error => {
timer.done()
callback(error)
})
},
_findAllUrlsInProject(project_id, callback) {
if (callback == null) {
callback = function (error, urls) {}
}
const timer = new Metrics.Timer('db-find-urls-in-project')
const job = cb =>
db.UrlCache.findAll({ where: { project_id } })
.then(urlEntries =>
cb(
null,
urlEntries.map(entry => entry.url)
)
)
.error(cb)
dbQueue.queue.push(job, (err, urls) => {
timer.done()
callback(err, urls)
})
},
}

131
services/clsi/app/js/UrlFetcher.js Normal file
View file

@ -0,0 +1,131 @@
/* eslint-disable
handle-callback-err,
no-return-assign,
no-unused-vars,
node/no-deprecated-api,
*/
// TODO: This file was created by bulk-decaffeinate.
// Fix any style issues and re-enable lint.
/*
* decaffeinate suggestions:
* DS102: Remove unnecessary code created because of implicit returns
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
let UrlFetcher
const request = require('request').defaults({ jar: false })
const fs = require('fs')
const logger = require('logger-sharelatex')
const settings = require('@overleaf/settings')
const URL = require('url')
const async = require('async')
const oneMinute = 60 * 1000
module.exports = UrlFetcher = {
pipeUrlToFileWithRetry(url, filePath, callback) {
const doDownload = function (cb) {
UrlFetcher.pipeUrlToFile(url, filePath, cb)
}
async.retry(3, doDownload, callback)
},
pipeUrlToFile(url, filePath, _callback) {
if (_callback == null) {
_callback = function (error) {}
}
const callbackOnce = function (error) {
if (timeoutHandler != null) {
clearTimeout(timeoutHandler)
}
_callback(error)
return (_callback = function () {})
}
const u = URL.parse(url)
if (
settings.filestoreDomainOveride &&
u.host !== settings.apis.clsiPerf.host
) {
url = `${settings.filestoreDomainOveride}${u.path}`
}
var timeoutHandler = setTimeout(
function () {
timeoutHandler = null
logger.error({ url, filePath }, 'Timed out downloading file to cache')
return callbackOnce(
new Error(`Timed out downloading file to cache ${url}`)
)
},
// FIXME: maybe need to close fileStream here
3 * oneMinute
)
logger.log({ url, filePath }, 'started downloading url to cache')
const urlStream = request.get({ url, timeout: oneMinute })
urlStream.pause() // stop data flowing until we are ready
// attach handlers before setting up pipes
urlStream.on('error', function (error) {
logger.error({ err: error, url, filePath }, 'error downloading url')
return callbackOnce(
error || new Error(`Something went wrong downloading the URL ${url}`)
)
})
urlStream.on('end', () =>
logger.log({ url, filePath }, 'finished downloading file into cache')
)
return urlStream.on('response', function (res) {
if (res.statusCode >= 200 && res.statusCode < 300) {
const fileStream = fs.createWriteStream(filePath)
// attach handlers before setting up pipes
fileStream.on('error', function (error) {
logger.error(
{ err: error, url, filePath },
'error writing file into cache'
)
return fs.unlink(filePath, function (err) {
if (err != null) {
logger.err({ err, filePath }, 'error deleting file from cache')
}
return callbackOnce(error)
})
})
fileStream.on('finish', function () {
logger.log({ url, filePath }, 'finished writing file into cache')
return callbackOnce()
})
fileStream.on('pipe', () =>
logger.log({ url, filePath }, 'piping into filestream')
)
urlStream.pipe(fileStream)
return urlStream.resume() // now we are ready to handle the data
} else {
logger.error(
{ statusCode: res.statusCode, url, filePath },
'unexpected status code downloading url to cache'
)
// https://nodejs.org/api/http.html#http_class_http_clientrequest
// If you add a 'response' event handler, then you must consume
// the data from the response object, either by calling
// response.read() whenever there is a 'readable' event, or by
// adding a 'data' handler, or by calling the .resume()
// method. Until the data is consumed, the 'end' event will not
// fire. Also, until the data is read it will consume memory
// that can eventually lead to a 'process out of memory' error.
urlStream.resume() // discard the data
return callbackOnce(
new Error(
`URL returned non-success status code: ${res.statusCode} ${url}`
)
)
}
})
},
}

67
services/clsi/app/js/db.js Normal file
View file

@ -0,0 +1,67 @@
/* eslint-disable
no-console,
*/
// TODO: This file was created by bulk-decaffeinate.
// Fix any style issues and re-enable lint.
/*
* decaffeinate suggestions:
* DS102: Remove unnecessary code created because of implicit returns
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
const Sequelize = require('sequelize')
const Settings = require('@overleaf/settings')
const _ = require('lodash')
const logger = require('logger-sharelatex')
const options = _.extend({ logging: false }, Settings.mysql.clsi)
logger.log({ dbPath: Settings.mysql.clsi.storage }, 'connecting to db')
const sequelize = new Sequelize(
Settings.mysql.clsi.database,
Settings.mysql.clsi.username,
Settings.mysql.clsi.password,
options
)
if (Settings.mysql.clsi.dialect === 'sqlite') {
logger.log('running PRAGMA journal_mode=WAL;')
sequelize.query('PRAGMA journal_mode=WAL;')
sequelize.query('PRAGMA synchronous=OFF;')
sequelize.query('PRAGMA read_uncommitted = true;')
}
module.exports = {
UrlCache: sequelize.define(
'UrlCache',
{
url: Sequelize.STRING,
project_id: Sequelize.STRING,
lastModified: Sequelize.DATE,
},
{
indexes: [{ fields: ['url', 'project_id'] }, { fields: ['project_id'] }],
}
),
Project: sequelize.define(
'Project',
{
project_id: { type: Sequelize.STRING, primaryKey: true },
lastAccessed: Sequelize.DATE,
},
{
indexes: [{ fields: ['lastAccessed'] }],
}
),
op: Sequelize.Op,
sync() {
logger.log({ dbPath: Settings.mysql.clsi.storage }, 'syncing db schema')
return sequelize
.sync()
.then(() => logger.log('db sync complete'))
.catch(err => console.log(err, 'error syncing'))
},
}

43
services/clsi/app/lib/pdfjs/FSPdfManager.js Normal file
View file

@ -0,0 +1,43 @@
const { PDFDocument } = require('pdfjs-dist/lib/core/document')
const { LocalPdfManager } = require('pdfjs-dist/lib/core/pdf_manager')
const { MissingDataException } = require('pdfjs-dist/lib/core/core_utils')
const { FSStream } = require('./FSStream')
class FSPdfManager extends LocalPdfManager {
constructor(docId, { fh, size, checkDeadline }) {
const nonEmptyDummyBuffer = Buffer.alloc(1, 0)
super(docId, nonEmptyDummyBuffer)
this.stream = new FSStream(fh, 0, size, null, null, checkDeadline)
this.pdfDocument = new PDFDocument(this, this.stream)
}
async ensure(obj, prop, args) {
try {
const value = obj[prop]
if (typeof value === 'function') {
return value.apply(obj, args)
}
return value
} catch (ex) {
if (!(ex instanceof MissingDataException)) {
throw ex
}
await this.requestRange(ex.begin, ex.end)
return this.ensure(obj, prop, args)
}
}
requestRange(begin, end) {
return this.stream.requestRange(begin, end)
}
requestLoadedStream() {}
onLoadedStream() {}
terminate(reason) {}
}
module.exports = {
FSPdfManager,
}

148
services/clsi/app/lib/pdfjs/FSStream.js Normal file
View file

@ -0,0 +1,148 @@
const { Stream } = require('pdfjs-dist/lib/core/stream')
const { MissingDataException } = require('pdfjs-dist/lib/core/core_utils')
const BUF_SIZE = 1024 // read from the file in 1024 byte pages
class FSStream extends Stream {
constructor(fh, start, length, dict, cachedBytes, checkDeadline) {
const nonEmptyDummyBuffer = Buffer.alloc(1, 0)
super(nonEmptyDummyBuffer, start, length, dict)
delete this.bytes
this.fh = fh
this.checkDeadline = checkDeadline
this.cachedBytes = cachedBytes || []
}
get length() {
return this.end - this.start
}
get isEmpty() {
return this.length === 0
}
// Manage cached reads from the file
requestRange(begin, end) {
this.checkDeadline(`request range ${begin} - ${end}`)
// expand small ranges to read a larger amount
if (end - begin < BUF_SIZE) {
end = begin + BUF_SIZE
}
end = Math.min(end, this.length)
// keep a cache of previous reads with {begin,end,buffer} values
const result = {
begin: begin,
end: end,
buffer: Buffer.alloc(end - begin, 0),
}
this.cachedBytes.push(result)
return this.fh.read(result.buffer, 0, end - begin, begin)
}
_ensureGetPos(pos) {
const found = this.cachedBytes.find(x => {
return x.begin <= pos && pos < x.end
})
if (!found) {
throw new MissingDataException(pos, pos + 1)
}
return found
}
_ensureGetRange(begin, end) {
end = Math.min(end, this.length) // BG: handle overflow case
const found = this.cachedBytes.find(x => {
return x.begin <= begin && end <= x.end
})
if (!found) {
throw new MissingDataException(begin, end)
}
return found
}
_readByte(found, pos) {
return found.buffer[pos - found.begin]
}
_readBytes(found, pos, end) {
return found.buffer.subarray(pos - found.begin, end - found.begin)
}
// handle accesses to the bytes
ensureByte(pos) {
this._ensureGetPos(pos) // may throw a MissingDataException
}
getByte() {
const pos = this.pos
if (this.pos >= this.end) {
return -1
}
const found = this._ensureGetPos(pos)
return this._readByte(found, this.pos++)
}
// BG: for a range, end is not included (see Buffer.subarray for example)
ensureBytes(length, forceClamped = false) {
const pos = this.pos
this._ensureGetRange(pos, pos + length)
}
getBytes(length, forceClamped = false) {
const pos = this.pos
const strEnd = this.end
const found = this._ensureGetRange(pos, pos + length)
if (!length) {
const subarray = this._readBytes(found, pos, strEnd)
// `this.bytes` is always a `Uint8Array` here.
return forceClamped ? new Uint8ClampedArray(subarray) : subarray
}
let end = pos + length
if (end > strEnd) {
end = strEnd
}
this.pos = end
const subarray = this._readBytes(found, pos, end)
// `this.bytes` is always a `Uint8Array` here.
return forceClamped ? new Uint8ClampedArray(subarray) : subarray
}
getByteRange() {
// BG: this isn't needed as far as I can tell
throw new Error('not implemented')
}
reset() {
this.pos = this.start
}
moveStart() {
this.start = this.pos
}
makeSubStream(start, length, dict = null) {
this.checkDeadline(`make sub stream start=${start}/length=${length}`)
// BG: had to add this check for null length, it is being called with only
// the start value at one point in the xref decoding. The intent is clear
// enough
// - a null length means "to the end of the file" -- not sure how it is
// working in the existing pdfjs code without this.
if (!length) {
length = this.end - start
}
return new FSStream(
this.fh,
start,
length,
dict,
this.cachedBytes,
this.checkDeadline
)
}
}
module.exports = { FSStream }

27
services/clsi/app/lib/pdfjs/parseXrefTable.js Normal file
View file

@ -0,0 +1,27 @@
const fs = require('fs')
const { FSPdfManager } = require('./FSPdfManager')
async function parseXrefTable(path, size, checkDeadline) {
if (size === 0) {
return []
}
const file = await fs.promises.open(path)
try {
const manager = new FSPdfManager(0, { fh: file, size, checkDeadline })
await manager.ensureDoc('checkHeader')
checkDeadline('pdfjs: after checkHeader')
await manager.ensureDoc('parseStartXRef')
checkDeadline('pdfjs: after parseStartXRef')
await manager.ensureDoc('parse')
checkDeadline('pdfjs: after parse')
return manager.pdfDocument.catalog.xref.entries
} finally {
file.close()
}
}
module.exports = {
parseXrefTable,
}

0
services/clsi/bin/.gitignore vendored Normal file
View file

4
services/clsi/bin/acceptance_test Normal file
View file

@ -0,0 +1,4 @@
#!/bin/bash
set -e;
MOCHA="node_modules/.bin/mocha --recursive --reporter spec --timeout 15000"
$MOCHA "$@"

BIN
services/clsi/bin/synctex Executable file
View file

Binary file not shown.

9
services/clsi/buildscript.txt Normal file
View file

@ -0,0 +1,9 @@
clsi
--data-dirs=cache,compiles,db,output
--dependencies=
--docker-repos=gcr.io/overleaf-ops
--env-add=ENABLE_PDF_CACHING="true"
--env-pass-through=TEXLIVE_IMAGE
--node-version=12.22.3
--public-repo=True
--script-version=3.11.0

170
services/clsi/config/settings.defaults.js Normal file
View file

@ -0,0 +1,170 @@
const Path = require('path')
module.exports = {
// Options are passed to Sequelize.
// See http://sequelizejs.com/documentation#usage-options for details
mysql: {
clsi: {
database: 'clsi',
username: 'clsi',
dialect: 'sqlite',
storage:
process.env.SQLITE_PATH || Path.resolve(__dirname, '../db/db.sqlite'),
pool: {
max: 1,
min: 1,
},
retry: {
max: 10,
},
},
},
compileSizeLimit: process.env.COMPILE_SIZE_LIMIT || '7mb',
processLifespanLimitMs:
parseInt(process.env.PROCESS_LIFE_SPAN_LIMIT_MS) || 60 * 60 * 24 * 1000 * 2,
catchErrors: process.env.CATCH_ERRORS === 'true',
path: {
compilesDir: Path.resolve(__dirname, '../compiles'),
outputDir: Path.resolve(__dirname, '../output'),
clsiCacheDir: Path.resolve(__dirname, '../cache'),
synctexBaseDir(projectId) {
return Path.join(this.compilesDir, projectId)
},
},
internal: {
clsi: {
port: 3013,
host: process.env.LISTEN_ADDRESS || 'localhost',
},
load_balancer_agent: {
report_load: true,
load_port: 3048,
local_port: 3049,
},
},
apis: {
clsi: {
url: `http://${process.env.CLSI_HOST || 'localhost'}:3013`,
},
clsiPerf: {
host: `${process.env.CLSI_PERF_HOST || 'localhost'}:${
process.env.CLSI_PERF_PORT || '3043'
}`,
},
},
smokeTest: process.env.SMOKE_TEST || false,
project_cache_length_ms: 1000 * 60 * 60 * 24,
parallelFileDownloads: process.env.FILESTORE_PARALLEL_FILE_DOWNLOADS || 1,
parallelSqlQueryLimit: process.env.FILESTORE_PARALLEL_SQL_QUERY_LIMIT || 1,
filestoreDomainOveride: process.env.FILESTORE_DOMAIN_OVERRIDE,
texliveImageNameOveride: process.env.TEX_LIVE_IMAGE_NAME_OVERRIDE,
texliveOpenoutAny: process.env.TEXLIVE_OPENOUT_ANY,
sentry: {
dsn: process.env.SENTRY_DSN,
},
enablePdfCaching: process.env.ENABLE_PDF_CACHING === 'true',
enablePdfCachingDark: process.env.ENABLE_PDF_CACHING_DARK === 'true',
pdfCachingMinChunkSize:
parseInt(process.env.PDF_CACHING_MIN_CHUNK_SIZE, 10) || 1024,
pdfCachingMaxProcessingTime:
parseInt(process.env.PDF_CACHING_MAX_PROCESSING_TIME, 10) || 10 * 1000,
}
if (process.env.ALLOWED_COMPILE_GROUPS) {
try {
module.exports.allowedCompileGroups =
process.env.ALLOWED_COMPILE_GROUPS.split(' ')
} catch (error) {
console.error(error, 'could not apply allowed compile group setting')
process.exit(1)
}
}
if (process.env.DOCKER_RUNNER) {
let seccompProfilePath
module.exports.clsi = {
dockerRunner: process.env.DOCKER_RUNNER === 'true',
docker: {
runtime: process.env.DOCKER_RUNTIME,
image:
process.env.TEXLIVE_IMAGE || 'quay.io/sharelatex/texlive-full:2017.1',
env: {
HOME: '/tmp',
},
socketPath: '/var/run/docker.sock',
user: process.env.TEXLIVE_IMAGE_USER || 'tex',
},
optimiseInDocker: true,
expireProjectAfterIdleMs: 24 * 60 * 60 * 1000,
checkProjectsIntervalMs: 10 * 60 * 1000,
}
try {
// Override individual docker settings using path-based keys, e.g.:
// compileGroupDockerConfigs = {
// priority: { 'HostConfig.CpuShares': 100 }
// beta: { 'dotted.path.here', 'value'}
// }
const compileGroupConfig = JSON.parse(
process.env.COMPILE_GROUP_DOCKER_CONFIGS || '{}'
)
// Automatically clean up wordcount and synctex containers
const defaultCompileGroupConfig = {
wordcount: { 'HostConfig.AutoRemove': true },
synctex: { 'HostConfig.AutoRemove': true },
}
module.exports.clsi.docker.compileGroupConfig = Object.assign(
defaultCompileGroupConfig,
compileGroupConfig
)
} catch (error) {
console.error(error, 'could not apply compile group docker configs')
process.exit(1)
}
try {
seccompProfilePath = Path.resolve(__dirname, '../seccomp/clsi-profile.json')
module.exports.clsi.docker.seccomp_profile = JSON.stringify(
JSON.parse(require('fs').readFileSync(seccompProfilePath))
)
} catch (error) {
console.error(
error,
`could not load seccomp profile from ${seccompProfilePath}`
)
process.exit(1)
}
if (process.env.APPARMOR_PROFILE) {
try {
module.exports.clsi.docker.apparmor_profile = process.env.APPARMOR_PROFILE
} catch (error) {
console.error(error, 'could not apply apparmor profile setting')
process.exit(1)
}
}
if (process.env.ALLOWED_IMAGES) {
try {
module.exports.clsi.docker.allowedImages =
process.env.ALLOWED_IMAGES.split(' ')
} catch (error) {
console.error(error, 'could not apply allowed images setting')
process.exit(1)
}
}
module.exports.path.synctexBaseDir = () => '/compile'
module.exports.path.sandboxedCompilesHostDir = process.env.COMPILES_HOST_DIR
module.exports.path.synctexBinHostPath = process.env.SYNCTEX_BIN_HOST_PATH
}

2
services/clsi/db/.gitignore vendored Normal file
View file

@ -0,0 +1,2 @@
*
!.gitignore

5
services/clsi/debug Executable file
View file

@ -0,0 +1,5 @@
#!/bin/bash
echo "hello world"
sleep 3
echo "awake"
/opt/synctex pdf /compile/output.pdf 1 100 200

34
services/clsi/docker-compose-config.yml Normal file
View file

@ -0,0 +1,34 @@
version: "2.3"
services:
dev:
environment:
ALLOWED_IMAGES: "quay.io/sharelatex/texlive-full:2017.1"
TEXLIVE_IMAGE: quay.io/sharelatex/texlive-full:2017.1
TEXLIVE_IMAGE_USER: "tex"
SHARELATEX_CONFIG: /app/config/settings.defaults.js
DOCKER_RUNNER: "true"
COMPILES_HOST_DIR: $PWD/compiles
SYNCTEX_BIN_HOST_PATH: $PWD/bin/synctex
volumes:
- /var/run/docker.sock:/var/run/docker.sock
- ./compiles:/app/compiles
- ./cache:/app/cache
- ./bin/synctex:/app/bin/synctex
ci:
environment:
ALLOWED_IMAGES: ${TEXLIVE_IMAGE}
TEXLIVE_IMAGE: quay.io/sharelatex/texlive-full:2017.1
TEXLIVE_IMAGE_USER: "tex"
SHARELATEX_CONFIG: /app/config/settings.defaults.js
DOCKER_RUNNER: "true"
COMPILES_HOST_DIR: $PWD/compiles
SYNCTEX_BIN_HOST_PATH: $PWD/bin/synctex
SQLITE_PATH: /app/compiles/db.sqlite
volumes:
- /var/run/docker.sock:/var/run/docker.sock:rw
- ./compiles:/app/compiles
- ./cache:/app/cache
- ./bin/synctex:/app/bin/synctex

42
services/clsi/docker-compose.ci.yml Normal file
View file

@ -0,0 +1,42 @@
# This file was auto-generated, do not edit it directly.
# Instead run bin/update_build_scripts from
# https://github.com/sharelatex/sharelatex-dev-environment
version: "2.3"
services:
test_unit:
image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
command: npm run test:unit:_run
environment:
NODE_ENV: test
NODE_OPTIONS: "--unhandled-rejections=strict"
test_acceptance:
build: .
image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
extends:
file: docker-compose-config.yml
service: ci
environment:
ELASTIC_SEARCH_DSN: es:9200
REDIS_HOST: redis
QUEUES_REDIS_HOST: redis
MONGO_HOST: mongo
POSTGRES_HOST: postgres
MOCHA_GREP: ${MOCHA_GREP}
NODE_ENV: test
NODE_OPTIONS: "--unhandled-rejections=strict"
TEXLIVE_IMAGE:
ENABLE_PDF_CACHING: "true"
command: npm run test:acceptance:_run
tar:
build: .
image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
volumes:
- ./:/tmp/build/
command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs .
user: root

43
services/clsi/docker-compose.yml Normal file
View file

@ -0,0 +1,43 @@
# This file was auto-generated, do not edit it directly.
# Instead run bin/update_build_scripts from
# https://github.com/sharelatex/sharelatex-dev-environment
version: "2.3"
services:
test_unit:
build:
context: .
target: base
volumes:
- .:/app
working_dir: /app
environment:
MOCHA_GREP: ${MOCHA_GREP}
NODE_ENV: test
NODE_OPTIONS: "--unhandled-rejections=strict"
command: npm run --silent test:unit
test_acceptance:
build:
context: .
target: base
volumes:
- .:/app
working_dir: /app
extends:
file: docker-compose-config.yml
service: dev
environment:
ELASTIC_SEARCH_DSN: es:9200
REDIS_HOST: redis
QUEUES_REDIS_HOST: redis
MONGO_HOST: mongo
POSTGRES_HOST: postgres
MOCHA_GREP: ${MOCHA_GREP}
LOG_LEVEL: ERROR
NODE_ENV: test
NODE_OPTIONS: "--unhandled-rejections=strict"
ENABLE_PDF_CACHING: "true"
command: npm run --silent test:acceptance

19
services/clsi/entrypoint.sh Executable file
View file

@ -0,0 +1,19 @@
#!/bin/sh
docker --version >&2
# add the node user to the docker group on the host
DOCKER_GROUP=$(stat -c '%g' /var/run/docker.sock)
groupadd --non-unique --gid ${DOCKER_GROUP} dockeronhost
usermod -aG dockeronhost node
# compatibility: initial volume setup
mkdir -p /app/cache && chown node:node /app/cache
mkdir -p /app/compiles && chown node:node /app/compiles
mkdir -p /app/db && chown node:node /app/db
mkdir -p /app/output && chown node:node /app/output
# make synctex available for remount in compiles
cp /app/bin/synctex /app/bin/synctex-mount/synctex
exec runuser -u node -- "$@"

24
services/clsi/install_deps.sh Executable file
View file

@ -0,0 +1,24 @@
#!/bin/bash
set -ex
apt-get update
apt-get install -y \
apt-transport-https \
ca-certificates \
curl \
gnupg \
lsb-release
curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
echo \
"deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian \
$(lsb_release -cs) stable" \
> /etc/apt/sources.list.d/docker.list
apt-get update
apt-get install -y \
docker-ce-cli \
poppler-utils \
ghostscript \
rm -rf /var/lib/apt/lists/*

41
services/clsi/kube.yaml Normal file
View file

@ -0,0 +1,41 @@
apiVersion: v1
kind: Service
metadata:
name: clsi
namespace: default
spec:
type: LoadBalancer
ports:
- port: 80
protocol: TCP
targetPort: 80
selector:
run: clsi
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: clsi
namespace: default
spec:
replicas: 2
template:
metadata:
labels:
run: clsi
spec:
containers:
- name: clsi
image: gcr.io/henry-terraform-admin/clsi
imagePullPolicy: Always
readinessProbe:
httpGet:
path: status
port: 80
periodSeconds: 5
initialDelaySeconds: 0
failureThreshold: 3
successThreshold: 1

17
services/clsi/nodemon.json Normal file
View file

@ -0,0 +1,17 @@
{
"ignore": [
".git",
"node_modules/"
],
"verbose": true,
"legacyWatch": true,
"execMap": {
"js": "npm run start"
},
"watch": [
"app/js/",
"app.js",
"config/"
],
"ext": "js"
}

6775
services/clsi/package-lock.json generated Normal file
View file

File diff suppressed because it is too large Load diff

67
services/clsi/package.json Normal file
View file

@ -0,0 +1,67 @@
{
"name": "node-clsi",
"description": "A Node.js implementation of the CLSI LaTeX web-API",
"version": "0.1.4",
"repository": {
"type": "git",
"url": "https://github.com/sharelatex/clsi-sharelatex.git"
},
"scripts": {
"start": "node $NODE_APP_OPTIONS app.js",
"test:acceptance:_run": "mocha --recursive --reporter spec --timeout 15000 --exit $@ test/acceptance/js",
"test:acceptance": "npm run test:acceptance:_run -- --grep=$MOCHA_GREP",
"test:unit:_run": "mocha --recursive --reporter spec $@ test/unit/js",
"test:unit": "npm run test:unit:_run -- --grep=$MOCHA_GREP",
"nodemon": "nodemon --config nodemon.json",
"lint": "eslint --max-warnings 0 --format unix .",
"format": "prettier --list-different $PWD/'**/*.js'",
"format:fix": "prettier --write $PWD/'**/*.js'",
"lint:fix": "eslint --fix ."
},
"author": "James Allen <james@sharelatex.com>",
"dependencies": {
"@overleaf/metrics": "^3.5.1",
"@overleaf/o-error": "^3.3.1",
"@overleaf/settings": "^2.1.1",
"async": "3.2.0",
"body-parser": "^1.19.0",
"bunyan": "^1.8.15",
"diskusage": "^1.1.3",
"dockerode": "^3.1.0",
"express": "^4.17.1",
"fs-extra": "^10.0.0",
"heapdump": "^0.3.15",
"lockfile": "^1.0.4",
"lodash": "^4.17.21",
"logger-sharelatex": "^2.2.0",
"mysql": "^2.18.1",
"p-limit": "^3.1.0",
"pdfjs-dist": "^2.7.570",
"request": "^2.88.2",
"send": "^0.17.1",
"sequelize": "^5.21.5",
"sqlite3": "^4.1.1",
"v8-profiler-node8": "^6.1.1",
"wrench": "~1.5.9"
},
"devDependencies": {
"chai": "^4.2.0",
"chai-as-promised": "^7.1.1",
"eslint": "^7.21.0",
"eslint-config-prettier": "^8.1.0",
"eslint-config-standard": "^16.0.2",
"eslint-plugin-chai-expect": "^2.2.0",
"eslint-plugin-chai-friendly": "^0.6.0",
"eslint-plugin-import": "^2.22.1",
"eslint-plugin-mocha": "^8.0.0",
"eslint-plugin-node": "^11.1.0",
"eslint-plugin-prettier": "^3.1.2",
"eslint-plugin-promise": "^4.2.1",
"mocha": "^8.3.2",
"nodemon": "^2.0.7",
"prettier": "^2.2.1",
"sandboxed-module": "^2.0.3",
"sinon": "~9.0.1",
"timekeeper": "2.2.0"
}
}

3
services/clsi/patch-texlive-dockerfile Normal file
View file

@ -0,0 +1,3 @@
FROM quay.io/sharelatex/texlive-full:2017.1
# RUN usermod -u 1001 tex

12
services/clsi/scripts/demo-pdfjs-Xref.js Normal file
View file

@ -0,0 +1,12 @@
const fs = require('fs')
const { parseXrefTable } = require('../app/lib/pdfjs/parseXrefTable')
const pdfPath = process.argv[2]
async function main() {
const size = (await fs.promises.stat(pdfPath)).size
const xRefEntries = await parseXrefTable(pdfPath, size)
console.log('Xref entries', xRefEntries)
}
main().catch(console.error)

836
services/clsi/seccomp/clsi-profile.json Normal file
View file

@ -0,0 +1,836 @@
{
"defaultAction": "SCMP_ACT_ERRNO",
"architectures": [
"SCMP_ARCH_X86_64",
"SCMP_ARCH_X86",
"SCMP_ARCH_X32"
],
"syscalls": [
{
"name": "access",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "arch_prctl",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "brk",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "chdir",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "chmod",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "clock_getres",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "clock_gettime",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "clock_nanosleep",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "clone",
"action": "SCMP_ACT_ALLOW",
"args": [
{
"index": 0,
"value": 2080505856,
"valueTwo": 0,
"op": "SCMP_CMP_MASKED_EQ"
}
]
},
{
"name": "close",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "copy_file_range",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "creat",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "dup",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "dup2",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "dup3",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "execve",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "execveat",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "exit",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "exit_group",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "faccessat",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "fadvise64",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "fadvise64_64",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "fallocate",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "fchdir",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "fchmod",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "fchmodat",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "fcntl",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "fcntl64",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "fdatasync",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "fork",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "fstat",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "fstat64",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "fstatat64",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "fstatfs",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "fstatfs64",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "fsync",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "ftruncate",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "ftruncate64",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "futex",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "futimesat",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "getcpu",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "getcwd",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "getdents",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "getdents64",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "getegid",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "getegid32",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "geteuid",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "geteuid32",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "getgid",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "getgid32",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "getgroups",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "getgroups32",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "getpgid",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "getpgrp",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "getpid",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "getppid",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "getpriority",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "getresgid",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "getresgid32",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "getresuid",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "getresuid32",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "getrlimit",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "get_robust_list",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "getrusage",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "getsid",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "gettid",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "getuid",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "getuid32",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "ioctl",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "kill",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "_llseek",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "lseek",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "lstat",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "lstat64",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "madvise",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "mkdir",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "mkdirat",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "mmap",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "mmap2",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "mprotect",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "mremap",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "munmap",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "newfstatat",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "open",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "openat",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "pause",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "pipe",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "pipe2",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "prctl",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "pread64",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "preadv",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "prlimit64",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "pwrite64",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "pwritev",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "read",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "readlink",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "readlinkat",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "readv",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "rename",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "renameat",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "renameat2",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "restart_syscall",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "rmdir",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "rt_sigaction",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "rt_sigpending",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "rt_sigprocmask",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "rt_sigqueueinfo",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "rt_sigreturn",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "rt_sigsuspend",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "rt_sigtimedwait",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "rt_tgsigqueueinfo",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "sched_getaffinity",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "sched_getparam",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "sched_get_priority_max",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "sched_get_priority_min",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "sched_getscheduler",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "sched_rr_get_interval",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "sched_yield",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "sendfile",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "sendfile64",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "setgroups",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "setgroups32",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "set_robust_list",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "set_tid_address",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "sigaltstack",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "stat",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "stat64",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "statfs",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "statfs64",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "sync",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "sync_file_range",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "syncfs",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "sysinfo",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "tgkill",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "timer_create",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "timer_delete",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "timer_getoverrun",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "timer_gettime",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "timer_settime",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "times",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "tkill",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "truncate",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "truncate64",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "umask",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "uname",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "unlink",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "unlinkat",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "utime",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "utimensat",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "utimes",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "vfork",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "vhangup",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "wait4",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "waitid",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "write",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "writev",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "pread",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "setgid",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "setuid",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "capget",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "capset",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "fchown",
"action": "SCMP_ACT_ALLOW",
"args": []
},
{
"name": "gettimeofday",
"action": "SCMP_ACT_ALLOW",
"args": []
}, {
"name": "epoll_pwait",
"action": "SCMP_ACT_ALLOW",
"args": []
}
]
}

66
services/clsi/src/synctex.c Normal file
View file

@ -0,0 +1,66 @@
#include <stdlib.h>
#include <stdio.h>
#include <errno.h>
#include <string.h>
#include "synctex/synctex_parser.h"
void print_usage() {
fprintf (stderr, "Usage: synctex code <synctex_file> <filename> <line> <column>\n");
fprintf (stderr, " synctex pdf <synctex_file> <page> <h> <v>\n");
}
int main(int argc, char *argv[], char *envp[]) {
synctex_scanner_t scanner;
if (argc < 6 || (strcmp(argv[1], "code") != 0 && strcmp(argv[1], "pdf") != 0)) {
print_usage();
return EXIT_FAILURE;
}
const char* direction = argv[1];
const char* synctex_file = argv[2];
scanner = synctex_scanner_new_with_output_file(synctex_file, NULL, 1);
if(!(scanner = synctex_scanner_parse(scanner))) {
fprintf (stderr, "Could not parse output file\n");
return EXIT_FAILURE;
}
if (strcmp(direction, "code") == 0) {
const char* name = argv[3];
int line = atoi(argv[4]);
int column = atoi(argv[5]);
if(synctex_display_query(scanner, name, line, column) > 0) {
synctex_node_t node;
while((node = synctex_next_result(scanner))) {
int page = synctex_node_page(node);
float h = synctex_node_box_visible_h(node);
float v = synctex_node_box_visible_v(node);
float width = synctex_node_box_visible_width(node);
float height = synctex_node_box_visible_height(node);
printf ("NODE\t%d\t%.2f\t%.2f\t%.2f\t%.2f\n", page, h, v, width, height);
}
}
} else if (strcmp(direction, "pdf") == 0) {
int page = atoi(argv[3]);
float h = atof(argv[4]);
float v = atof(argv[5]);
if(synctex_edit_query(scanner, page, h, v) > 0) {
synctex_node_t node;
while((node = synctex_next_result(scanner))) {
int tag = synctex_node_tag(node);
const char* name = synctex_scanner_get_name(scanner, tag);
int line = synctex_node_line(node);
int column = synctex_node_column(node);
printf ("NODE\t%s\t%d\t%d\n", name, line, column);
}
}
}
return 0;
}

4249
services/clsi/src/synctex/synctex_parser.c Normal file
View file

File diff suppressed because it is too large Load diff

346
services/clsi/src/synctex/synctex_parser.h Normal file
View file

@ -0,0 +1,346 @@
/*
Copyright (c) 2008, 2009, 2010 , 2011 jerome DOT laurens AT u-bourgogne DOT fr
This file is part of the SyncTeX package.
Latest Revision: Tue Jun 14 08:23:30 UTC 2011
Version: 1.16
See synctex_parser_readme.txt for more details
License:
--------
Permission is hereby granted, free of charge, to any person
obtaining a copy of this software and associated documentation
files (the "Software"), to deal in the Software without
restriction, including without limitation the rights to use,
copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the
Software is furnished to do so, subject to the following
conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
OTHER DEALINGS IN THE SOFTWARE
Except as contained in this notice, the name of the copyright holder
shall not be used in advertising or otherwise to promote the sale,
use or other dealings in this Software without prior written
authorization from the copyright holder.
Acknowledgments:
----------------
The author received useful remarks from the pdfTeX developers, especially Hahn The Thanh,
and significant help from XeTeX developer Jonathan Kew
Nota Bene:
----------
If you include or use a significant part of the synctex package into a software,
I would appreciate to be listed as contributor and see "SyncTeX" highlighted.
Version 1
Thu Jun 19 09:39:21 UTC 2008
*/
#ifndef __SYNCTEX_PARSER__
# define __SYNCTEX_PARSER__
#ifdef __cplusplus
extern "C" {
#endif
/* synctex_node_t is the type for all synctex nodes.
* The synctex file is parsed into a tree of nodes, either sheet, boxes, math nodes... */
typedef struct _synctex_node * synctex_node_t;
/* The main synctex object is a scanner
* Its implementation is considered private.
* The basic workflow is
* - create a "synctex scanner" with the contents of a file
* - perform actions on that scanner like display or edit queries
* - free the scanner when the work is done
*/
typedef struct __synctex_scanner_t _synctex_scanner_t;
typedef _synctex_scanner_t * synctex_scanner_t;
/* This is the designated method to create a new synctex scanner object.
* output is the pdf/dvi/xdv file associated to the synctex file.
* If necessary, it can be the tex file that originated the synctex file
* but this might cause problems if the \jobname has a custom value.
* Despite this method can accept a relative path in practice,
* you should only pass a full path name.
* The path should be encoded by the underlying file system,
* assuming that it is based on 8 bits characters, including UTF8,
* not 16 bits nor 32 bits.
* The last file extension is removed and replaced by the proper extension.
* Then the private method _synctex_scanner_new_with_contents_of_file is called.
* NULL is returned in case of an error or non existent file.
* Once you have a scanner, use the synctex_display_query and synctex_edit_query below.
* The new "build_directory" argument is available since version 1.5.
* It is the directory where all the auxiliary stuff is created.
* Sometimes, the synctex output file and the pdf, dvi or xdv files are not created in the same directory.
* This is the case in MikTeX (I will include this into TeX Live).
* This directory path can be nil, it will be ignored then.
* It can be either absolute or relative to the directory of the output pdf (dvi or xdv) file.
* If no synctex file is found in the same directory as the output file, then we try to find one in the build directory.
* Please note that this new "build_directory" is provided as a convenient argument but should not be used.
* In fact, this is implempented as a work around of a bug in MikTeX where the synctex file does not follow the pdf file.
* The new "parse" argument is available since version 1.5. In general, use 1.
* Use 0 only if you do not want to parse the content but just check the existence.
*/
synctex_scanner_t synctex_scanner_new_with_output_file(const char * output, const char * build_directory, int parse);
/* This is the designated method to delete a synctex scanner object.
* Frees all the memory, you must call it when you are finished with the scanner.
*/
void synctex_scanner_free(synctex_scanner_t scanner);
/* Send this message to force the scanner to parse the contents of the synctex output file.
* Nothing is performed if the file was already parsed.
* In each query below, this message is sent, but if you need to access information more directly,
* you must be sure that the parsing did occur.
* Usage:
* if((my_scanner = synctex_scanner_parse(my_scanner))) {
* continue with my_scanner...
* } else {
* there was a problem
* }
*/
synctex_scanner_t synctex_scanner_parse(synctex_scanner_t scanner);
/* The main entry points.
* Given the file name, a line and a column number, synctex_display_query returns the number of nodes
* satisfying the contrain. Use code like
*
* if(synctex_display_query(scanner,name,line,column)>0) {
* synctex_node_t node;
* while((node = synctex_next_result(scanner))) {
* // do something with node
* ...
* }
* }
*
* For example, one can
* - highlight each resulting node in the output, using synctex_node_h and synctex_node_v
* - highlight all the rectangles enclosing those nodes, using synctex_box_... functions
* - highlight just the character using that information
*
* Given the page and the position in the page, synctex_edit_query returns the number of nodes
* satisfying the contrain. Use code like
*
* if(synctex_edit_query(scanner,page,h,v)>0) {
* synctex_node_t node;
* while(node = synctex_next_result(scanner)) {
* // do something with node
* ...
* }
* }
*
* For example, one can
* - highlight each resulting line in the input,
* - highlight just the character using that information
*
* page is 1 based
* h and v are coordinates in 72 dpi unit, relative to the top left corner of the page.
* If you make a new query, the result of the previous one is discarded.
* If one of this function returns a non positive integer,
* it means that an error occurred.
*
* Both methods are conservative, in the sense that matching is weak.
* If the exact column number is not found, there will be an answer with the whole line.
*
* Sumatra-PDF, Skim, iTeXMac2 and Texworks are examples of open source software that use this library.
* You can browse their code for a concrete implementation.
*/
int synctex_display_query(synctex_scanner_t scanner,const char * name,int line,int column);
int synctex_edit_query(synctex_scanner_t scanner,int page,float h,float v);
synctex_node_t synctex_next_result(synctex_scanner_t scanner);
/* Display all the information contained in the scanner object.
* If the records are too numerous, only the first ones are displayed.
* This is mainly for informatinal purpose to help developers.
*/
void synctex_scanner_display(synctex_scanner_t scanner);
/* The x and y offset of the origin in TeX coordinates. The magnification
These are used by pdf viewers that want to display the real box size.
For example, getting the horizontal coordinates of a node would require
synctex_node_box_h(node)*synctex_scanner_magnification(scanner)+synctex_scanner_x_offset(scanner)
Getting its TeX width would simply require
synctex_node_box_width(node)*synctex_scanner_magnification(scanner)
but direct methods are available for that below.
*/
int synctex_scanner_x_offset(synctex_scanner_t scanner);
int synctex_scanner_y_offset(synctex_scanner_t scanner);
float synctex_scanner_magnification(synctex_scanner_t scanner);
/* Managing the input file names.
* Given a tag, synctex_scanner_get_name will return the corresponding file name.
* Conversely, given a file name, synctex_scanner_get_tag will retur, the corresponding tag.
* The file name must be the very same as understood by TeX.
* For example, if you \input myDir/foo.tex, the file name is myDir/foo.tex.
* No automatic path expansion is performed.
* Finally, synctex_scanner_input is the first input node of the scanner.
* To browse all the input node, use a loop like
*
* if((input_node = synctex_scanner_input(scanner))){
* do {
* blah
* } while((input_node=synctex_node_sibling(input_node)));
* }
*
* The output is the name that was used to create the scanner.
* The synctex is the real name of the synctex file,
* it was obtained from output by setting the proper file extension.
*/
const char * synctex_scanner_get_name(synctex_scanner_t scanner,int tag);
int synctex_scanner_get_tag(synctex_scanner_t scanner,const char * name);
synctex_node_t synctex_scanner_input(synctex_scanner_t scanner);
const char * synctex_scanner_get_output(synctex_scanner_t scanner);
const char * synctex_scanner_get_synctex(synctex_scanner_t scanner);
/* Browsing the nodes
* parent, child and sibling are standard names for tree nodes.
* The parent is one level higher, the child is one level deeper,
* and the sibling is at the same level.
* The sheet of a node is the first ancestor, it is of type sheet.
* A node and its sibling have the same parent.
* A node is the parent of its child.
* A node is either the child of its parent,
* or belongs to the sibling chain of its parent's child.
* The next node is either the child, the sibling or the parent's sibling,
* unless the parent is a sheet.
* This allows to navigate through all the nodes of a given sheet node:
*
* synctex_node_t node = sheet;
* while((node = synctex_node_next(node))) {
* // do something with node
* }
*
* With synctex_sheet_content, you can retrieve the sheet node given the page.
* The page is 1 based, according to TeX standards.
* Conversely synctex_node_sheet allows to retrieve the sheet containing a given node.
*/
synctex_node_t synctex_node_parent(synctex_node_t node);
synctex_node_t synctex_node_sheet(synctex_node_t node);
synctex_node_t synctex_node_child(synctex_node_t node);
synctex_node_t synctex_node_sibling(synctex_node_t node);
synctex_node_t synctex_node_next(synctex_node_t node);
synctex_node_t synctex_sheet_content(synctex_scanner_t scanner,int page);
/* These are the types of the synctex nodes */
typedef enum {
synctex_node_type_error = 0,
synctex_node_type_input,
synctex_node_type_sheet,
synctex_node_type_vbox,
synctex_node_type_void_vbox,
synctex_node_type_hbox,
synctex_node_type_void_hbox,
synctex_node_type_kern,
synctex_node_type_glue,
synctex_node_type_math,
synctex_node_type_boundary,
synctex_node_number_of_types
} synctex_node_type_t;
/* synctex_node_type gives the type of a given node,
* synctex_node_isa gives the same information as a human readable text. */
synctex_node_type_t synctex_node_type(synctex_node_t node);
const char * synctex_node_isa(synctex_node_t node);
/* This is primarily used for debugging purpose.
* The second one logs information for the node and recursively displays information for its next node */
void synctex_node_log(synctex_node_t node);
void synctex_node_display(synctex_node_t node);
/* Given a node, access to its tag, line and column.
* The line and column numbers are 1 based.
* The latter is not yet fully supported in TeX, the default implementation returns 0 which means the whole line.
* When the tag is known, the scanner of the node will give the corresponding file name.
* When the tag is known, the scanner of the node will give the name.
*/
int synctex_node_tag(synctex_node_t node);
int synctex_node_line(synctex_node_t node);
int synctex_node_column(synctex_node_t node);
/* This is the page where the node appears.
* This is a 1 based index as given by TeX.
*/
int synctex_node_page(synctex_node_t node);
/* For quite all nodes, horizontal, vertical coordinates, and width.
* These are expressed in TeX small points coordinates, with origin at the top left corner.
*/
int synctex_node_h(synctex_node_t node);
int synctex_node_v(synctex_node_t node);
int synctex_node_width(synctex_node_t node);
/* For all nodes, dimensions of the enclosing box.
* These are expressed in TeX small points coordinates, with origin at the top left corner.
* A box is enclosing itself.
*/
int synctex_node_box_h(synctex_node_t node);
int synctex_node_box_v(synctex_node_t node);
int synctex_node_box_width(synctex_node_t node);
int synctex_node_box_height(synctex_node_t node);
int synctex_node_box_depth(synctex_node_t node);
/* For quite all nodes, horizontal, vertical coordinates, and width.
* The visible dimensions are bigger than real ones to compensate 0 width boxes
* that do contain nodes.
* These are expressed in page coordinates, with origin at the top left corner.
* A box is enclosing itself.
*/
float synctex_node_visible_h(synctex_node_t node);
float synctex_node_visible_v(synctex_node_t node);
float synctex_node_visible_width(synctex_node_t node);
/* For all nodes, visible dimensions of the enclosing box.
* A box is enclosing itself.
* The visible dimensions are bigger than real ones to compensate 0 width boxes
* that do contain nodes.
*/
float synctex_node_box_visible_h(synctex_node_t node);
float synctex_node_box_visible_v(synctex_node_t node);
float synctex_node_box_visible_width(synctex_node_t node);
float synctex_node_box_visible_height(synctex_node_t node);
float synctex_node_box_visible_depth(synctex_node_t node);
/* The main synctex updater object.
* This object is used to append information to the synctex file.
* Its implementation is considered private.
* It is used by the synctex command line tool to take into account modifications
* that could occur while postprocessing files by dvipdf like filters.
*/
typedef struct __synctex_updater_t _synctex_updater_t;
typedef _synctex_updater_t * synctex_updater_t;
/* Designated initializer.
* Once you are done with your whole job,
* free the updater */
synctex_updater_t synctex_updater_new_with_output_file(const char * output, const char * directory);
/* Use the next functions to append records to the synctex file,
* no consistency tests made on the arguments */
void synctex_updater_append_magnification(synctex_updater_t updater, char * magnification);
void synctex_updater_append_x_offset(synctex_updater_t updater, char * x_offset);
void synctex_updater_append_y_offset(synctex_updater_t updater, char * y_offset);
/* You MUST free the updater, once everything is properly appended */
void synctex_updater_free(synctex_updater_t updater);
#ifdef __cplusplus
}
#endif
#endif

45
services/clsi/src/synctex/synctex_parser_local.h Normal file
View file

@ -0,0 +1,45 @@
/*
Copyright (c) 2008, 2009, 2010 , 2011 jerome DOT laurens AT u-bourgogne DOT fr
This file is part of the SyncTeX package.
Latest Revision: Tue Jun 14 08:23:30 UTC 2011
Version: 1.16
See synctex_parser_readme.txt for more details
License:
--------
Permission is hereby granted, free of charge, to any person
obtaining a copy of this software and associated documentation
files (the "Software"), to deal in the Software without
restriction, including without limitation the rights to use,
copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the
Software is furnished to do so, subject to the following
conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
OTHER DEALINGS IN THE SOFTWARE
Except as contained in this notice, the name of the copyright holder
shall not be used in advertising or otherwise to promote the sale,
use or other dealings in this Software without prior written
authorization from the copyright holder.
*/
/* This local header file is for TEXLIVE, use your own header to fit your system */
# include <w2c/c-auto.h> /* for inline && HAVE_xxx */
/* No inlining for synctex tool in texlive. */
# define SYNCTEX_INLINE

141
services/clsi/src/synctex/synctex_parser_readme.txt Normal file
View file

@ -0,0 +1,141 @@
This file is part of the SyncTeX package.
The Synchronization TeXnology named SyncTeX is a new feature
of recent TeX engines designed by Jerome Laurens.
It allows to synchronize between input and output, which means to
navigate from the source document to the typeset material and vice versa.
More informations on http://itexmac2.sourceforge.net/SyncTeX.html
This package is mainly for developers, it mainly contains the following files:
synctex_parser_readme.txt
synctex_parser_version.txt
synctex_parser_utils.c
synctex_parser_utils.h
synctex_parser_local.h
synctex_parser.h
synctex_parser.c
The file you are reading contains more informations about the SyncTeX parser history.
In order to support SyncTeX in a viewer, it is sufficient to include
in the source the files synctex_parser.h and synctex_parser.c.
The synctex parser usage is described in synctex_parser.h header file.
The other files are used by tex engines or by the synctex command line utility:
ChangeLog
README.txt
am
man1
man5
synctex-common.h
synctex-convert.sh
synctex-e-mem.ch0
synctex-e-mem.ch1
synctex-e-rec.ch0
synctex-e-rec.ch1
synctex-etex.h
synctex-mem.ch0
synctex-mem.ch1
synctex-mem.ch2
synctex-pdf-rec.ch2
synctex-pdftex.h
synctex-rec.ch0
synctex-rec.ch1
synctex-rec.ch2
synctex-tex.h
synctex-xe-mem.ch2
synctex-xe-rec.ch2
synctex-xe-rec.ch3
synctex-xetex.h
synctex.c
synctex.defines
synctex.h
synctex_main.c
tests
Version:
--------
This is version 1, which refers to the synctex output file format.
The files are identified by a build number.
In order to help developers to automatically manage the version and build numbers
and download the parser only when necessary, the synctex_parser.version
is an ASCII text file just containing the current version and build numbers.
History:
--------
1.1: Thu Jul 17 09:28:13 UTC 2008
- First official version available in TeXLive 2008 DVD.
Unfortunately, the backwards synchronization is not working properly mainly for ConTeXt users, see below.
1.2: Tue Sep 2 10:28:32 UTC 2008
- Correction for ConTeXt support in the edit query.
The previous method was assuming that TeX boxes do not overlap,
which is reasonable for LaTeX but not for ConTeXt.
This assumption is no longer considered.
1.3: Fri Sep 5 09:39:57 UTC 2008
- Local variable "read" renamed to "already_read" to avoid conflicts.
- "inline" compiler directive renamed to "SYNCTEX_INLINE" for code support and maintenance
- _synctex_error cannot be inlined due to variable arguments (thanks Christiaan Hofman)
- Correction in the display query, extra boundary nodes are used for a more precise forwards synchronization
1.4: Fri Sep 12 08:12:34 UTC 2008
- For an unknown reason, the previous version was not the real 1.3 (as used in iTeXMac2 build 747).
As a consequence, a crash was observed.
- Some typos are fixed.
1.6: Mon Nov 3 20:20:02 UTC 2008
- The bug that prevented synchronization with compressed files on windows has been fixed.
- New interface to allow system specific customization.
- Note that some APIs have changed.
1.8: Mer 8 jul 2009 11:32:38 UTC
Note that version 1.7 was delivered privately.
- bug fix: synctex was causing a memory leak in pdftex and xetex, thus some processing speed degradation
- bug fix: the synctex command line tool was broken when updating a .synctex file
- enhancement: better accuracy of the synchronization process
- enhancement: the pdf output file and the associated .synctex file no longer need to live in the same directory.
The new -d option of the synctex command line tool manages this situation.
This is handy when using something like tex -output-directory=DIR ...
1.9: Wed Nov 4 11:52:35 UTC 2009
- Various typo fixed
- OutputDebugString replaced by OutputDebugStringA to deliberately disable unicode preprocessing
- New conditional created because OutputDebugStringA is only available since Windows 2K professional
1.10: Sun Jan 10 10:12:32 UTC 2010
- Bug fix in synctex_parser.c to solve a synchronization problem with amsmath's gather environment.
Concerns the synctex tool.
1.11: Sun Jan 17 09:12:31 UTC 2010
- Bug fix in synctex_parser.c, function synctex_node_box_visible_v: 'x' replaced by 'y'.
Only 3rd party tools are concerned.
1.12: Mon Jul 19 21:52:10 UTC 2010
- Bug fix in synctex_parser.c, function __synctex_open: the io_mode was modified even in case of a non zero return,
causing a void .synctex.gz file to be created even if it was not expected. Reported by Marek Kasik concerning a bug on evince.
1.13: Fri Mar 11 07:39:12 UTC 2011
- Bug fix in synctex_parser.c, better synchronization as suggested by Jan Sundermeyer (near line 3388).
- Stronger code design in synctex_parser_utils.c, function _synctex_get_name (really neutral behavior).
Only 3rd party tools are concerned.
1.14: Fri Apr 15 19:10:57 UTC 2011
- taking output_directory into account
- Replaced FOPEN_WBIN_MODE by FOPEN_W_MODE when opening the text version of the .synctex file.
- Merging with LuaTeX's version of synctex.c
1.15: Fri Jun 10 14:10:17 UTC 2011
This concerns the synctex command line tool and 3rd party developers.
TeX and friends are not concerned by these changes.
- Bug fixed in _synctex_get_io_mode_name, sometimes the wrong mode was returned
- Support for LuaTeX convention of './' file prefixing
1.16: Tue Jun 14 08:23:30 UTC 2011
This concerns the synctex command line tool and 3rd party developers.
TeX and friends are not concerned by these changes.
- Better forward search (thanks Jose Alliste)
- Support for LuaTeX convention of './' file prefixing now for everyone, not only for Windows
Acknowledgments:
----------------
The author received useful remarks from the pdfTeX developers, especially Hahn The Thanh,
and significant help from XeTeX developer Jonathan Kew
Nota Bene:
----------
If you include or use a significant part of the synctex package into a software,
I would appreciate to be listed as contributor and see "SyncTeX" highlighted.
Copyright (c) 2008-2011 jerome DOT laurens AT u-bourgogne DOT fr

479
services/clsi/src/synctex/synctex_parser_utils.c Normal file
View file

@ -0,0 +1,479 @@
/*
Copyright (c) 2008, 2009, 2010 , 2011 jerome DOT laurens AT u-bourgogne DOT fr
This file is part of the SyncTeX package.
Latest Revision: Tue Jun 14 08:23:30 UTC 2011
Version: 1.16
See synctex_parser_readme.txt for more details
License:
--------
Permission is hereby granted, free of charge, to any person
obtaining a copy of this software and associated documentation
files (the "Software"), to deal in the Software without
restriction, including without limitation the rights to use,
copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the
Software is furnished to do so, subject to the following
conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
OTHER DEALINGS IN THE SOFTWARE
Except as contained in this notice, the name of the copyright holder
shall not be used in advertising or otherwise to promote the sale,
use or other dealings in this Software without prior written
authorization from the copyright holder.
*/
/* In this file, we find all the functions that may depend on the operating system. */
#include <synctex_parser_utils.h>
#include <stdlib.h>
#include <string.h>
#include <stdarg.h>
#include <stdio.h>
#include <limits.h>
#include <ctype.h>
#include <string.h>
#include <sys/stat.h>
#if defined(_WIN32) || defined(__WIN32__) || defined(__TOS_WIN__) || defined(__WINDOWS__)
#define SYNCTEX_WINDOWS 1
#endif
#ifdef _WIN32_WINNT_WINXP
#define SYNCTEX_RECENT_WINDOWS 1
#endif
#ifdef SYNCTEX_WINDOWS
#include <windows.h>
#endif
void *_synctex_malloc(size_t size) {
void * ptr = malloc(size);
if(ptr) {
/* There used to be a switch to use bzero because it is more secure. JL */
memset(ptr,0, size);
}
return (void *)ptr;
}
int _synctex_error(const char * reason,...) {
va_list arg;
int result;
va_start (arg, reason);
# ifdef SYNCTEX_RECENT_WINDOWS
{/* This code is contributed by William Blum.
As it does not work on some older computers,
the _WIN32 conditional here is replaced with a SYNCTEX_RECENT_WINDOWS one.
According to http://msdn.microsoft.com/en-us/library/aa363362(VS.85).aspx
Minimum supported client Windows 2000 Professional
Minimum supported server Windows 2000 Server
People running Windows 2K standard edition will not have OutputDebugStringA.
JL.*/
char *buff;
size_t len;
OutputDebugStringA("SyncTeX ERROR: ");
len = _vscprintf(reason, arg) + 1;
buff = (char*)malloc( len * sizeof(char) );
result = vsprintf(buff, reason, arg) +strlen("SyncTeX ERROR: ");
OutputDebugStringA(buff);
OutputDebugStringA("\n");
free(buff);
}
# else
result = fprintf(stderr,"SyncTeX ERROR: ");
result += vfprintf(stderr, reason, arg);
result += fprintf(stderr,"\n");
# endif
va_end (arg);
return result;
}
/* strip the last extension of the given string, this string is modified! */
void _synctex_strip_last_path_extension(char * string) {
if(NULL != string){
char * last_component = NULL;
char * last_extension = NULL;
char * next = NULL;
/* first we find the last path component */
if(NULL == (last_component = strstr(string,"/"))){
last_component = string;
} else {
++last_component;
while((next = strstr(last_component,"/"))){
last_component = next+1;
}
}
# ifdef SYNCTEX_WINDOWS
/* On Windows, the '\' is also a path separator. */
while((next = strstr(last_component,"\\"))){
last_component = next+1;
}
# endif
/* then we find the last path extension */
if((last_extension = strstr(last_component,"."))){
++last_extension;
while((next = strstr(last_extension,"."))){
last_extension = next+1;
}
--last_extension;/* back to the "." */
if(last_extension>last_component){/* filter out paths like ....my/dir/.hidden"*/
last_extension[0] = '\0';
}
}
}
}
const char * synctex_ignore_leading_dot_slash(const char * name)
{
while(SYNCTEX_IS_DOT(*name) && SYNCTEX_IS_PATH_SEPARATOR(name[1])) {
name += 2;
while (SYNCTEX_IS_PATH_SEPARATOR(*name)) {
++name;
}
}
return name;
}
/* Compare two file names, windows is sometimes case insensitive... */
synctex_bool_t _synctex_is_equivalent_file_name(const char *lhs, const char *rhs) {
/* Remove the leading regex '(\./+)*' in both rhs and lhs */
lhs = synctex_ignore_leading_dot_slash(lhs);
rhs = synctex_ignore_leading_dot_slash(rhs);
# if SYNCTEX_WINDOWS
/* On Windows, filename should be compared case insensitive.
* The characters '/' and '\' are both valid path separators.
* There will be a very serious problem concerning UTF8 because
* not all the characters must be toupper...
* I would like to have URL's instead of filenames. */
next_character:
if(SYNCTEX_IS_PATH_SEPARATOR(*lhs)) {/* lhs points to a path separator */
if(!SYNCTEX_IS_PATH_SEPARATOR(*rhs)) {/* but not rhs */
return synctex_NO;
}
} else if(SYNCTEX_IS_PATH_SEPARATOR(*rhs)) {/* rhs points to a path separator but not lhs */
return synctex_NO;
} else if(toupper(*lhs) != toupper(*rhs)){/* uppercase do not match */
return synctex_NO;
} else if (!*lhs) {/* lhs is at the end of the string */
return *rhs ? synctex_NO : synctex_YES;
} else if(!*rhs) {/* rhs is at the end of the string but not lhs */
return synctex_NO;
}
++lhs;
++rhs;
goto next_character;
# else
return 0 == strcmp(lhs,rhs)?synctex_YES:synctex_NO;
# endif
}
synctex_bool_t _synctex_path_is_absolute(const char * name) {
if(!strlen(name)) {
return synctex_NO;
}
# if SYNCTEX_WINDOWS
if(strlen(name)>2) {
return (name[1]==':' && SYNCTEX_IS_PATH_SEPARATOR(name[2]))?synctex_YES:synctex_NO;
}
return synctex_NO;
# else
return SYNCTEX_IS_PATH_SEPARATOR(name[0])?synctex_YES:synctex_NO;
# endif
}
/* We do not take care of UTF-8 */
const char * _synctex_last_path_component(const char * name) {
const char * c = name+strlen(name);
if(c>name) {
if(!SYNCTEX_IS_PATH_SEPARATOR(*c)) {
do {
--c;
if(SYNCTEX_IS_PATH_SEPARATOR(*c)) {
return c+1;
}
} while(c>name);
}
return c;/* the last path component is the void string*/
}
return c;
}
int _synctex_copy_with_quoting_last_path_component(const char * src, char ** dest_ref, size_t size) {
const char * lpc;
if(src && dest_ref) {
# define dest (*dest_ref)
dest = NULL; /* Default behavior: no change and sucess. */
lpc = _synctex_last_path_component(src);
if(strlen(lpc)) {
if(strchr(lpc,' ') && lpc[0]!='"' && lpc[strlen(lpc)-1]!='"') {
/* We are in the situation where adding the quotes is allowed. */
/* Time to add the quotes. */
/* Consistency test: we must have dest+size>dest+strlen(dest)+2
* or equivalently: strlen(dest)+2<size (see below) */
if(strlen(src)<size) {
if((dest = (char *)malloc(size+2))) {
char * dpc = dest + (lpc-src); /* dpc is the last path component of dest. */
if(dest != strncpy(dest,src,size)) {
_synctex_error("! _synctex_copy_with_quoting_last_path_component: Copy problem");
free(dest);
dest = NULL;/* Don't forget to reinitialize. */
return -2;
}
memmove(dpc+1,dpc,strlen(dpc)+1); /* Also move the null terminating character. */
dpc[0]='"';
dpc[strlen(dpc)+1]='\0';/* Consistency test */
dpc[strlen(dpc)]='"';
return 0; /* Success. */
}
return -1; /* Memory allocation error. */
}
_synctex_error("! _synctex_copy_with_quoting_last_path_component: Internal inconsistency");
return -3;
}
return 0; /* Success. */
}
return 0; /* No last path component. */
# undef dest
}
return 1; /* Bad parameter, this value is subject to changes. */
}
/* The client is responsible of the management of the returned string, if any. */
char * _synctex_merge_strings(const char * first,...);
char * _synctex_merge_strings(const char * first,...) {
va_list arg;
size_t size = 0;
const char * temp;
/* First retrieve the size necessary to store the merged string */
va_start (arg, first);
temp = first;
do {
size_t len = strlen(temp);
if(UINT_MAX-len<size) {
_synctex_error("! _synctex_merge_strings: Capacity exceeded.");
return NULL;
}
size+=len;
} while( (temp = va_arg(arg, const char *)) != NULL);
va_end(arg);
if(size>0) {
char * result = NULL;
++size;
/* Create the memory storage */
if(NULL!=(result = (char *)malloc(size))) {
char * dest = result;
va_start (arg, first);
temp = first;
do {
if((size = strlen(temp))>0) {
/* There is something to merge */
if(dest != strncpy(dest,temp,size)) {
_synctex_error("! _synctex_merge_strings: Copy problem");
free(result);
result = NULL;
return NULL;
}
dest += size;
}
} while( (temp = va_arg(arg, const char *)) != NULL);
va_end(arg);
dest[0]='\0';/* Terminate the merged string */
return result;
}
_synctex_error("! _synctex_merge_strings: Memory problem");
return NULL;
}
return NULL;
}
/* The purpose of _synctex_get_name is to find the name of the synctex file.
* There is a list of possible filenames from which we return the most recent one and try to remove all the others.
* With two runs of pdftex or xetex we are sure the the synctex file is really the most appropriate.
*/
int _synctex_get_name(const char * output, const char * build_directory, char ** synctex_name_ref, synctex_io_mode_t * io_mode_ref)
{
if(output && synctex_name_ref && io_mode_ref) {
/* If output is already absolute, we just have to manage the quotes and the compress mode */
size_t size = 0;
char * synctex_name = NULL;
synctex_io_mode_t io_mode = *io_mode_ref;
const char * base_name = _synctex_last_path_component(output); /* do not free, output is the owner. base name of output*/
/* Do we have a real base name ? */
if(strlen(base_name)>0) {
/* Yes, we do. */
const char * temp = NULL;
char * core_name = NULL; /* base name of output without path extension. */
char * dir_name = NULL; /* dir name of output */
char * quoted_core_name = NULL;
char * basic_name = NULL;
char * gz_name = NULL;
char * quoted_name = NULL;
char * quoted_gz_name = NULL;
char * build_name = NULL;
char * build_gz_name = NULL;
char * build_quoted_name = NULL;
char * build_quoted_gz_name = NULL;
struct stat buf;
time_t the_time = 0;
/* Create core_name: let temp point to the dot before the path extension of base_name;
* We start form the \0 terminating character and scan the string upward until we find a dot.
* The leading dot is not accepted. */
if((temp = strrchr(base_name,'.')) && (size = temp - base_name)>0) {
/* There is a dot and it is not at the leading position */
if(NULL == (core_name = (char *)malloc(size+1))) {
_synctex_error("! _synctex_get_name: Memory problem 1");
return -1;
}
if(core_name != strncpy(core_name,base_name,size)) {
_synctex_error("! _synctex_get_name: Copy problem 1");
free(core_name);
dir_name = NULL;
return -2;
}
core_name[size] = '\0';
} else {
/* There is no path extension,
* Just make a copy of base_name */
core_name = _synctex_merge_strings(base_name);
}
/* core_name is properly set up, owned by "self". */
/* creating dir_name. */
size = strlen(output)-strlen(base_name);
if(size>0) {
/* output contains more than one path component */
if(NULL == (dir_name = (char *)malloc(size+1))) {
_synctex_error("! _synctex_get_name: Memory problem");
free(core_name);
dir_name = NULL;
return -1;
}
if(dir_name != strncpy(dir_name,output,size)) {
_synctex_error("! _synctex_get_name: Copy problem");
free(dir_name);
dir_name = NULL;
free(core_name);
dir_name = NULL;
return -2;
}
dir_name[size] = '\0';
}
/* dir_name is properly set up. It ends with a path separator, if non void. */
/* creating quoted_core_name. */
if(strchr(core_name,' ')) {
quoted_core_name = _synctex_merge_strings("\"",core_name,"\"");
}
/* quoted_core_name is properly set up. */
if(dir_name &&strlen(dir_name)>0) {
basic_name = _synctex_merge_strings(dir_name,core_name,synctex_suffix,NULL);
if(quoted_core_name && strlen(quoted_core_name)>0) {
quoted_name = _synctex_merge_strings(dir_name,quoted_core_name,synctex_suffix,NULL);
}
} else {
basic_name = _synctex_merge_strings(core_name,synctex_suffix,NULL);
if(quoted_core_name && strlen(quoted_core_name)>0) {
quoted_name = _synctex_merge_strings(quoted_core_name,synctex_suffix,NULL);
}
}
if(!_synctex_path_is_absolute(output) && build_directory && (size = strlen(build_directory))) {
temp = build_directory + size - 1;
if(_synctex_path_is_absolute(temp)) {
build_name = _synctex_merge_strings(build_directory,basic_name,NULL);
if(quoted_core_name && strlen(quoted_core_name)>0) {
build_quoted_name = _synctex_merge_strings(build_directory,quoted_name,NULL);
}
} else {
build_name = _synctex_merge_strings(build_directory,"/",basic_name,NULL);
if(quoted_core_name && strlen(quoted_core_name)>0) {
build_quoted_name = _synctex_merge_strings(build_directory,"/",quoted_name,NULL);
}
}
}
if(basic_name) {
gz_name = _synctex_merge_strings(basic_name,synctex_suffix_gz,NULL);
}
if(quoted_name) {
quoted_gz_name = _synctex_merge_strings(quoted_name,synctex_suffix_gz,NULL);
}
if(build_name) {
build_gz_name = _synctex_merge_strings(build_name,synctex_suffix_gz,NULL);
}
if(build_quoted_name) {
build_quoted_gz_name = _synctex_merge_strings(build_quoted_name,synctex_suffix_gz,NULL);
}
/* All the others names are properly set up... */
/* retain the most recently modified file */
# define TEST(FILENAME,COMPRESS_MODE) \
if(FILENAME) {\
if (stat(FILENAME, &buf)) { \
free(FILENAME);\
FILENAME = NULL;\
} else if (buf.st_mtime>the_time) { \
the_time=buf.st_mtime; \
synctex_name = FILENAME; \
if (COMPRESS_MODE) { \
io_mode |= synctex_io_gz_mask; \
} else { \
io_mode &= ~synctex_io_gz_mask; \
} \
} \
}
TEST(basic_name,synctex_DONT_COMPRESS);
TEST(gz_name,synctex_COMPRESS);
TEST(quoted_name,synctex_DONT_COMPRESS);
TEST(quoted_gz_name,synctex_COMPRESS);
TEST(build_name,synctex_DONT_COMPRESS);
TEST(build_gz_name,synctex_COMPRESS);
TEST(build_quoted_name,synctex_DONT_COMPRESS);
TEST(build_quoted_gz_name,synctex_COMPRESS);
# undef TEST
/* Free all the intermediate filenames, except the one that will be used as returned value. */
# define CLEAN_AND_REMOVE(FILENAME) \
if(FILENAME && (FILENAME!=synctex_name)) {\
remove(FILENAME);\
printf("synctex tool info: %s removed\n",FILENAME);\
free(FILENAME);\
FILENAME = NULL;\
}
CLEAN_AND_REMOVE(basic_name);
CLEAN_AND_REMOVE(gz_name);
CLEAN_AND_REMOVE(quoted_name);
CLEAN_AND_REMOVE(quoted_gz_name);
CLEAN_AND_REMOVE(build_name);
CLEAN_AND_REMOVE(build_gz_name);
CLEAN_AND_REMOVE(build_quoted_name);
CLEAN_AND_REMOVE(build_quoted_gz_name);
# undef CLEAN_AND_REMOVE
/* set up the returned values */
* synctex_name_ref = synctex_name;
* io_mode_ref = io_mode;
return 0;
}
return -1;/* bad argument */
}
return -2;
}
const char * _synctex_get_io_mode_name(synctex_io_mode_t io_mode) {
static const char * synctex_io_modes[4] = {"r","rb","a","ab"};
unsigned index = ((io_mode & synctex_io_gz_mask)?1:0) + ((io_mode & synctex_io_append_mask)?2:0);// bug pointed out by Jose Alliste
return synctex_io_modes[index];
}

141
services/clsi/src/synctex/synctex_parser_utils.h Normal file
View file

@ -0,0 +1,141 @@
/*
Copyright (c) 2008, 2009, 2010, 2011 jerome DOT laurens AT u-bourgogne DOT fr
This file is part of the SyncTeX package.
Latest Revision: Tue Jun 14 08:23:30 UTC 2011
Version: 1.16
See synctex_parser_readme.txt for more details
License:
--------
Permission is hereby granted, free of charge, to any person
obtaining a copy of this software and associated documentation
files (the "Software"), to deal in the Software without
restriction, including without limitation the rights to use,
copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the
Software is furnished to do so, subject to the following
conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
OTHER DEALINGS IN THE SOFTWARE
Except as contained in this notice, the name of the copyright holder
shall not be used in advertising or otherwise to promote the sale,
use or other dealings in this Software without prior written
authorization from the copyright holder.
*/
/* The utilities declared here are subject to conditional implementation.
* All the operating system special stuff goes here.
* The problem mainly comes from file name management: path separator, encoding...
*/
# define synctex_bool_t int
# define synctex_YES -1
# define synctex_ADD_QUOTES -1
# define synctex_COMPRESS -1
# define synctex_NO 0
# define synctex_DONT_ADD_QUOTES 0
# define synctex_DONT_COMPRESS 0
#ifndef __SYNCTEX_PARSER_UTILS__
# define __SYNCTEX_PARSER_UTILS__
#include <stdlib.h>
#ifdef __cplusplus
extern "C" {
#endif
# if _WIN32
# define SYNCTEX_IS_PATH_SEPARATOR(c) ('/' == c || '\\' == c)
# else
# define SYNCTEX_IS_PATH_SEPARATOR(c) ('/' == c)
# endif
# if _WIN32
# define SYNCTEX_IS_DOT(c) ('.' == c)
# else
# define SYNCTEX_IS_DOT(c) ('.' == c)
# endif
/* This custom malloc functions initializes to 0 the newly allocated memory.
* There is no bzero function on windows. */
void *_synctex_malloc(size_t size);
/* This is used to log some informational message to the standard error stream.
* On Windows, the stderr stream is not exposed and another method is used.
* The return value is the number of characters printed. */
int _synctex_error(const char * reason,...);
/* strip the last extension of the given string, this string is modified!
* This function depends on the OS because the path separator may differ.
* This should be discussed more precisely. */
void _synctex_strip_last_path_extension(char * string);
/* Compare two file names, windows is sometimes case insensitive...
* The given strings may differ stricto sensu, but represent the same file name.
* It might not be the real way of doing things.
* The return value is an undefined non 0 value when the two file names are equivalent.
* It is 0 otherwise. */
synctex_bool_t _synctex_is_equivalent_file_name(const char *lhs, const char *rhs);
/* Description forthcoming.*/
synctex_bool_t _synctex_path_is_absolute(const char * name);
/* Description forthcoming...*/
const char * _synctex_last_path_component(const char * name);
/* If the core of the last path component of src is not already enclosed with double quotes ('"')
* and contains a space character (' '), then a new buffer is created, the src is copied and quotes are added.
* In all other cases, no destination buffer is created and the src is not copied.
* 0 on success, which means no error, something non 0 means error, mainly due to memory allocation failure, or bad parameter.
* This is used to fix a bug in the first version of pdftex with synctex (1.40.9) for which names with spaces
* were not managed in a standard way.
* On success, the caller owns the buffer pointed to by dest_ref (is any) and
* is responsible of freeing the memory when done.
* The size argument is the size of the src buffer. On return the dest_ref points to a buffer sized size+2.*/
int _synctex_copy_with_quoting_last_path_component(const char * src, char ** dest_ref, size_t size);
/* These are the possible extensions of the synctex file */
extern const char * synctex_suffix;
extern const char * synctex_suffix_gz;
typedef unsigned int synctex_io_mode_t;
typedef enum {
synctex_io_append_mask = 1,
synctex_io_gz_mask = synctex_io_append_mask<<1
} synctex_io_mode_masks_t;
typedef enum {
synctex_compress_mode_none = 0,
synctex_compress_mode_gz = 1
} synctex_compress_mode_t;
int _synctex_get_name(const char * output, const char * build_directory, char ** synctex_name_ref, synctex_io_mode_t * io_mode_ref);
/* returns the correct mode required by fopen and gzopen from the given io_mode */
const char * _synctex_get_io_mode_name(synctex_io_mode_t io_mode);
const char * synctex_ignore_leading_dot_slash(const char * name);
#ifdef __cplusplus
}
#endif
#endif

1
services/clsi/src/synctex/synctex_parser_version.txt Normal file
View file

@ -0,0 +1 @@
1.16

34
services/clsi/synctex.profile Normal file
View file

@ -0,0 +1,34 @@
include /etc/firejail/disable-common.inc
include /etc/firejail/disable-devel.inc
# include /etc/firejail/disable-mgmt.inc ## removed in 0.9.40
# include /etc/firejail/disable-secret.inc ## removed in 0.9.40
read-only /bin
blacklist /boot
blacklist /dev
read-only /etc
blacklist /home # blacklisted for synctex
read-only /lib
read-only /lib64
blacklist /media
blacklist /mnt
blacklist /opt
blacklist /root
read-only /run
blacklist /sbin
blacklist /selinux
blacklist /src
blacklist /sys
read-only /usr
caps.drop all
noroot
nogroups
net none
private-tmp
private-dev
shell none
seccomp
nonewprivs

117
services/clsi/test/acceptance/fixtures/examples/asymptote/main.tex Normal file
View file

@ -0,0 +1,117 @@
\documentclass[12pt]{article}
% Use this form to include EPS (latex) or PDF (pdflatex) files:
\usepackage{asymptote}
% Use this form with latex or pdflatex to include inline LaTeX code by default:
%\usepackage[inline]{asymptote}
% Use this form with latex or pdflatex to create PDF attachments by default:
%\usepackage[attach]{asymptote}
% Enable this line to support the attach option:
%\usepackage[dvips]{attachfile2}
\begin{document}
% Optional subdirectory for asy files (no spaces):
\def\asydir{}
\begin{asydef}
// Global Asymptote definitions can be put here.
import three;
usepackage("bm");
texpreamble("\def\V#1{\bm{#1}}");
// One can globally override the default toolbar settings here:
// settings.toolbar=true;
\end{asydef}
Here is a venn diagram produced with Asymptote, drawn to width 4cm:
\def\A{A}
\def\B{\V{B}}
%\begin{figure}
\begin{center}
\begin{asy}
size(4cm,0);
pen colour1=red;
pen colour2=green;
pair z0=(0,0);
pair z1=(-1,0);
pair z2=(1,0);
real r=1.5;
path c1=circle(z1,r);
path c2=circle(z2,r);
fill(c1,colour1);
fill(c2,colour2);
picture intersection=new picture;
fill(intersection,c1,colour1+colour2);
clip(intersection,c2);
add(intersection);
draw(c1);
draw(c2);
//draw("$\A$",box,z1); // Requires [inline] package option.
//draw(Label("$\B$","$B$"),box,z2); // Requires [inline] package option.
draw("$A$",box,z1);
draw("$\V{B}$",box,z2);
pair z=(0,-2);
real m=3;
margin BigMargin=Margin(0,m*dot(unit(z1-z),unit(z0-z)));
draw(Label("$A\cap B$",0),conj(z)--z0,Arrow,BigMargin);
draw(Label("$A\cup B$",0),z--z0,Arrow,BigMargin);
draw(z--z1,Arrow,Margin(0,m));
draw(z--z2,Arrow,Margin(0,m));
shipout(bbox(0.25cm));
\end{asy}
%\caption{Venn diagram}\label{venn}
\end{center}
%\end{figure}
Each graph is drawn in its own environment. One can specify the width
and height to \LaTeX\ explicitly. This 3D example can be viewed
interactively either with Adobe Reader or Asymptote's fast OpenGL-based
renderer. To support {\tt latexmk}, 3D figures should specify
\verb+inline=true+. It is sometimes desirable to embed 3D files as annotated
attachments; this requires the \verb+attach=true+ option as well as the
\verb+attachfile2+ \LaTeX\ package.
\begin{center}
\begin{asy}[height=4cm,inline=true,attach=false,viewportwidth=\linewidth]
currentprojection=orthographic(5,4,2);
draw(unitcube,blue);
label("$V-E+F=2$",(0,1,0.5),3Y,blue+fontsize(17pt));
\end{asy}
\end{center}
One can also scale the figure to the full line width:
\begin{center}
\begin{asy}[width=\the\linewidth,inline=true]
pair z0=(0,0);
pair z1=(2,0);
pair z2=(5,0);
pair zf=z1+0.75*(z2-z1);
draw(z1--z2);
dot(z1,red+0.15cm);
dot(z2,darkgreen+0.3cm);
label("$m$",z1,1.2N,red);
label("$M$",z2,1.5N,darkgreen);
label("$\hat{\ }$",zf,0.2*S,fontsize(24pt)+blue);
pair s=-0.2*I;
draw("$x$",z0+s--z1+s,N,red,Arrows,Bars,PenMargins);
s=-0.5*I;
draw("$\bar{x}$",z0+s--zf+s,blue,Arrows,Bars,PenMargins);
s=-0.95*I;
draw("$X$",z0+s--z2+s,darkgreen,Arrows,Bars,PenMargins);
\end{asy}
\end{center}
\end{document}

BIN
services/clsi/test/acceptance/fixtures/examples/asymptote/output.pdf Normal file
View file

Binary file not shown.

9
services/clsi/test/acceptance/fixtures/examples/biber_bibliography/bibliography.bib Normal file
View file

@ -0,0 +1,9 @@
@book{DouglasAdams,
title={The Hitchhiker's Guide to the Galaxy},
author={Adams, Douglas},
isbn={9781417642595},
url={http://books.google.com/books?id=W-xMPgAACAAJ},
year={1995},
publisher={San Val}
}

12
services/clsi/test/acceptance/fixtures/examples/biber_bibliography/main.tex Normal file
View file

@ -0,0 +1,12 @@
\documentclass{article}
\usepackage[backend=biber]{biblatex}
\addbibresource{bibliography.bib}
\begin{document}
The meaning of life, the universe and everything is 42 \cite{DouglasAdams}
\printbibliography
\end{document}

48
services/clsi/test/acceptance/fixtures/examples/biber_bibliography/output.bbl Normal file
View file

@ -0,0 +1,48 @@
% $ biblatex auxiliary file $
% $ biblatex version 1.5 $
% $ biber version 0.9.3 $
% Do not modify the above lines!
%
% This is an auxiliary file used by the 'biblatex' package.
% This file may safely be deleted. It will be recreated by
% biber or bibtex as required.
%
\begingroup
\makeatletter
\@ifundefined{ver@biblatex.sty}
{\@latex@error
{Missing 'biblatex' package}
{The bibliography requires the 'biblatex' package.}
\aftergroup\endinput}
{}
\endgroup
\refsection{0}
\entry{DouglasAdams}{book}{}
\name{labelname}{1}{}{%
{{}{Adams}{A\bibinitperiod}{Douglas}{D\bibinitperiod}{}{}{}{}}%
}
\name{author}{1}{}{%
{{}{Adams}{A\bibinitperiod}{Douglas}{D\bibinitperiod}{}{}{}{}}%
}
\list{publisher}{1}{%
{San Val}%
}
\strng{namehash}{AD1}
\strng{fullhash}{AD1}
\field{sortinit}{A}
\field{isbn}{9781417642595}
\field{title}{The Hitchhiker's Guide to the Galaxy}
\field{year}{1995}
\verb{url}
\verb http://books.google.com/books?id=W-xMPgAACAAJ
\endverb
\endentry
\lossort
\endlossort
\endrefsection
\endinput

BIN
services/clsi/test/acceptance/fixtures/examples/biber_bibliography/output.pdf Normal file
View file

Binary file not shown.

84
services/clsi/test/acceptance/fixtures/examples/biber_bibliography/output.run.xml Normal file
View file

@ -0,0 +1,84 @@
<?xml version="1.0" standalone="yes"?>
<!-- logreq request file -->
<!-- logreq version 1.0 / dtd version 1.0 -->
<!-- Do not edit this file! -->
<!DOCTYPE requests [
<!ELEMENT requests (internal | external)*>
<!ELEMENT internal (generic, (provides | requires)*)>
<!ELEMENT external (generic, cmdline?, input?, output?, (provides | requires)*)>
<!ELEMENT cmdline (binary, (option | infile | outfile)*)>
<!ELEMENT input (file)+>
<!ELEMENT output (file)+>
<!ELEMENT provides (file)+>
<!ELEMENT requires (file)+>
<!ELEMENT generic (#PCDATA)>
<!ELEMENT binary (#PCDATA)>
<!ELEMENT option (#PCDATA)>
<!ELEMENT infile (#PCDATA)>
<!ELEMENT outfile (#PCDATA)>
<!ELEMENT file (#PCDATA)>
<!ATTLIST requests
version CDATA #REQUIRED
>
<!ATTLIST internal
package CDATA #REQUIRED
priority (9) #REQUIRED
active (0 | 1) #REQUIRED
>
<!ATTLIST external
package CDATA #REQUIRED
priority (1 | 2 | 3 | 4 | 5 | 6 | 7 | 8) #REQUIRED
active (0 | 1) #REQUIRED
>
<!ATTLIST provides
type (static | dynamic | editable) #REQUIRED
>
<!ATTLIST requires
type (static | dynamic | editable) #REQUIRED
>
<!ATTLIST file
type CDATA #IMPLIED
>
]>
<requests version="1.0">
<internal package="biblatex" priority="9" active="0">
<generic>latex</generic>
<provides type="dynamic">
<file>output.bcf</file>
</provides>
<requires type="dynamic">
<file>output.bbl</file>
</requires>
<requires type="static">
<file>blx-compat.def</file>
<file>biblatex.def</file>
<file>numeric.bbx</file>
<file>standard.bbx</file>
<file>numeric.cbx</file>
<file>biblatex.cfg</file>
<file>english.lbx</file>
</requires>
</internal>
<external package="biblatex" priority="5" active="0">
<generic>biber</generic>
<cmdline>
<binary>biber</binary>
<infile>output</infile>
</cmdline>
<input>
<file>output.bcf</file>
</input>
<output>
<file>output.bbl</file>
</output>
<provides type="dynamic">
<file>output.bbl</file>
</provides>
<requires type="dynamic">
<file>output.bcf</file>
</requires>
<requires type="editable">
<file>bibliography.bib</file>
</requires>
</external>
</requests>

BIN
services/clsi/test/acceptance/fixtures/examples/epstopdf/image-eps-converted-to.pdf Normal file
View file

Binary file not shown.

6673
services/clsi/test/acceptance/fixtures/examples/epstopdf/image.eps Normal file
View file

File diff suppressed because it is too large Load diff

10
services/clsi/test/acceptance/fixtures/examples/epstopdf/main.tex Normal file
View file

@ -0,0 +1,10 @@
\documentclass{article}
\usepackage{graphicx}
\usepackage{epstopdf}
\begin{document}
\includegraphics[width=\textwidth]{image}
\end{document}

BIN
services/clsi/test/acceptance/fixtures/examples/epstopdf/output.pdf Normal file
View file

Binary file not shown.

28
services/clsi/test/acceptance/fixtures/examples/feynmf/main.tex Normal file
View file

@ -0,0 +1,28 @@
\documentclass[a4paper]{article}
\usepackage{feynmf}
\begin{document}
\setlength{\unitlength}{1mm}
\begin{fmffile}{diagram}
\begin{center}
\begin{fmfgraph*}(41,17)
\fmfleftn{i}{2}
\fmfrightn{o}{2}
\fmflabel{$g_2$}{i1}
\fmflabel{$g_1$}{i2}
\fmflabel{$p_2$}{o1}
\fmflabel{$p_1$}{o2}
\fmf{quark}{i1,v1}
\fmf{quark}{i2,v1}
\fmfblob{.35w}{v1}
\fmf{quark}{v1,o1}
\fmf{quark}{v1,o2}
\end{fmfgraph*}
\end{center}
\end{fmffile}
\end{document}

BIN
services/clsi/test/acceptance/fixtures/examples/feynmf/output.pdf Normal file
View file

Binary file not shown.

28
services/clsi/test/acceptance/fixtures/examples/feynmp/main.tex Normal file
View file

@ -0,0 +1,28 @@
\documentclass[a4paper]{article}
\usepackage{feynmp}
\begin{document}
\setlength{\unitlength}{1mm}
\begin{fmffile}{diagram}
\begin{center}
\begin{fmfgraph*}(41,17)
\fmfleftn{i}{2}
\fmfrightn{o}{2}
\fmflabel{$g_2$}{i1}
\fmflabel{$g_1$}{i2}
\fmflabel{$p_2$}{o1}
\fmflabel{$p_1$}{o2}
\fmf{quark}{i1,v1}
\fmf{quark}{i2,v1}
\fmfblob{.35w}{v1}
\fmf{quark}{v1,o1}
\fmf{quark}{v1,o2}
\end{fmfgraph*}
\end{center}
\end{fmffile}
\end{document}

3
services/clsi/test/acceptance/fixtures/examples/feynmp/options.json Normal file
View file

@ -0,0 +1,3 @@
{
"compiler": "latex"
}

BIN
services/clsi/test/acceptance/fixtures/examples/feynmp/output.pdf Normal file
View file

Binary file not shown.

12
services/clsi/test/acceptance/fixtures/examples/fontawesome/main.tex Normal file
View file

@ -0,0 +1,12 @@
\documentclass{article}
\usepackage{fontawesome}
\begin{document}
Cloud \faCloud
Cog \faCog
Database \faDatabase
Leaf \faLeaf
\end{document}

BIN
services/clsi/test/acceptance/fixtures/examples/fontawesome/output.pdf Normal file
View file

Binary file not shown.

16
services/clsi/test/acceptance/fixtures/examples/fontawesome_xelatex/main.tex Normal file
View file

@ -0,0 +1,16 @@
\documentclass{article}
\usepackage{fontspec}
\defaultfontfeatures{Extension = .otf} % this is needed because
% fontawesome package loads by
% font name only
\usepackage{fontawesome}
\begin{document}
Cloud \faCloud
Cog \faCog
Database \faDatabase
Leaf \faLeaf
\end{document}

3
services/clsi/test/acceptance/fixtures/examples/fontawesome_xelatex/options.json Normal file
View file

@ -0,0 +1,3 @@
{
"compiler": "xelatex"
}

BIN
services/clsi/test/acceptance/fixtures/examples/fontawesome_xelatex/output.pdf Normal file
View file

Binary file not shown.

17
services/clsi/test/acceptance/fixtures/examples/glossaries/main.tex Normal file
View file

@ -0,0 +1,17 @@
\documentclass{article}
\usepackage{glossaries}
\makeglossaries
\newglossaryentry{Physics}{
name=Physics,
description={is the study of stuff}
}
\begin{document}
To solve various problems in \Gls{Physics} it can useful to express any arbitrary piecewise-smooth function as a Fourier Series composed of multiple sine and cosine funcions.
\printglossaries
\end{document}

7
services/clsi/test/acceptance/fixtures/examples/glossaries/output.glg Normal file
View file

@ -0,0 +1,7 @@
This is makeindex, version 2.15 [TeX Live 2011] (kpathsea + Thai support).
Scanning style file ./output.ist...........................done (27 attributes redefined, 0 ignored).
Scanning input file output.glo....done (1 entries accepted, 0 rejected).
Sorting entries...done (0 comparisons).
Generating output file output.gls....done (6 lines written, 0 warnings).
Output written in output.gls.
Transcript written in output.glg.

1
services/clsi/test/acceptance/fixtures/examples/glossaries/output.glo Normal file
View file

@ -0,0 +1 @@
\glossaryentry{Physics?\glossaryentryfield{Physics}{\glsnamefont{Physics}}{is the study of stuff}{\relax }|setentrycounter[]{page}\glsnumberformat}{1}

6
services/clsi/test/acceptance/fixtures/examples/glossaries/output.gls Normal file
View file

@ -0,0 +1,6 @@
\glossarysection[\glossarytoctitle]{\glossarytitle}\glossarypreamble
\begin{theglossary}\glossaryheader
\glsgroupheading{P}\relax \glsresetentrylist %
\glossaryentryfield{Physics}{\glsnamefont{Physics}}{is the study of stuff}{\relax }{\glossaryentrynumbers{\relax
\setentrycounter[]{page}\glsnumberformat{1}}}%
\end{theglossary}\glossarypostamble

Some files were not shown because too many files have changed in this diff Show more