github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2024-09-16 02:52:31 -04:00
Code Issues Projects Releases Packages Wiki Activity

Fully implement max file size

Browse source 
- add `repoStore.maxFileSize` key to config
- use maxFileSize in ResourceCache on both header path and blob
  download path
- make failures during commit less fragile
This commit is contained in:
Michael Walker 2018-01-12 16:47:13 +00:00
parent dc93df6bc8
commit 1e4ef0cc5b
28 changed files with 367 additions and 83 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
services/git-bridge/conf/example_config.json
View file

@ -11,6 +11,9 @@
"oauth2ClientSecret": "asdf",
"oauth2Server": "https://localhost"
},
"repoStore": {
"maxFileSize": 52428800
},
"swapStore": {
"type": "s3",
"awsAccessKey": "asdf",

12
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/application/config/Config.java
View file

@ -4,6 +4,7 @@ import com.google.gson.Gson;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import uk.ac.ic.wlgitbridge.application.exception.ConfigFileException;
import uk.ac.ic.wlgitbridge.bridge.repo.RepoStoreConfig;
import uk.ac.ic.wlgitbridge.bridge.swap.job.SwapJobConfig;
import uk.ac.ic.wlgitbridge.bridge.swap.store.SwapStoreConfig;
import uk.ac.ic.wlgitbridge.snapshot.base.JSONSource;
@ -30,6 +31,7 @@ public class Config implements JSONSource {
config.postbackURL,
config.serviceName,
Oauth2.asSanitised(config.oauth2),
config.repoStore,
SwapStoreConfig.sanitisedCopy(config.swapStore),
config.swapJob
);
@ -45,6 +47,8 @@ public class Config implements JSONSource {
@Nullable
private Oauth2 oauth2;
@Nullable
private RepoStoreConfig repoStore;
@Nullable
private SwapStoreConfig swapStore;
@Nullable
private SwapJobConfig swapJob;
@ -69,6 +73,7 @@ public class Config implements JSONSource {
String postbackURL,
String serviceName,
Oauth2 oauth2,
RepoStoreConfig repoStore,
SwapStoreConfig swapStore,
SwapJobConfig swapJob
) {
@ -80,6 +85,7 @@ public class Config implements JSONSource {
this.postbackURL = postbackURL;
this.serviceName = serviceName;
this.oauth2 = oauth2;
this.repoStore = repoStore;
this.swapStore = swapStore;
this.swapJob = swapJob;
}
@ -108,6 +114,8 @@ public class Config implements JSONSource {
postbackURL += "/";
}
oauth2 = new Gson().fromJson(configObject.get("oauth2"), Oauth2.class);
repoStore = new Gson().fromJson(
configObject.get("repoStore"), RepoStoreConfig.class);
swapStore = new Gson().fromJson(
configObject.get("swapStore"),
SwapStoreConfig.class
@ -161,6 +169,10 @@ public class Config implements JSONSource {
return oauth2;
}
public Optional<RepoStoreConfig> getRepoStore() {
return Optional.ofNullable(repoStore);
}
public Optional<SwapStoreConfig> getSwapStore() {
return Optional.ofNullable(swapStore);
}

51
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/Bridge.java
View file

@ -2,6 +2,7 @@ package uk.ac.ic.wlgitbridge.bridge;
import com.google.api.client.auth.oauth2.Credential;
import org.eclipse.jgit.errors.RepositoryNotFoundException;
import uk.ac.ic.wlgitbridge.application.config.Config;
import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
import uk.ac.ic.wlgitbridge.bridge.db.ProjectState;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SqliteDBStore;
@ -9,20 +10,16 @@ import uk.ac.ic.wlgitbridge.bridge.gc.GcJob;
import uk.ac.ic.wlgitbridge.bridge.gc.GcJobImpl;
import uk.ac.ic.wlgitbridge.bridge.lock.LockGuard;
import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock;
import uk.ac.ic.wlgitbridge.bridge.repo.FSGitRepoStore;
import uk.ac.ic.wlgitbridge.bridge.repo.GitProjectRepo;
import uk.ac.ic.wlgitbridge.bridge.repo.ProjectRepo;
import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore;
import uk.ac.ic.wlgitbridge.bridge.repo.*;
import uk.ac.ic.wlgitbridge.bridge.resource.ResourceCache;
import uk.ac.ic.wlgitbridge.bridge.resource.UrlResourceCache;
import uk.ac.ic.wlgitbridge.bridge.snapshot.NetSnapshotApi;
import uk.ac.ic.wlgitbridge.bridge.snapshot.SnapshotApi;
import uk.ac.ic.wlgitbridge.bridge.snapshot.SnapshotApiFacade;
import uk.ac.ic.wlgitbridge.bridge.swap.job.SwapJob;
import uk.ac.ic.wlgitbridge.bridge.swap.job.SwapJobConfig;
import uk.ac.ic.wlgitbridge.bridge.swap.job.SwapJobImpl;
import uk.ac.ic.wlgitbridge.bridge.swap.store.S3SwapStore;
import uk.ac.ic.wlgitbridge.bridge.swap.store.SwapStore;
import uk.ac.ic.wlgitbridge.bridge.snapshot.SnapshotApiFacade;
import uk.ac.ic.wlgitbridge.data.CandidateSnapshot;
import uk.ac.ic.wlgitbridge.data.ProjectLockImpl;
import uk.ac.ic.wlgitbridge.data.filestore.GitDirectoryContents;
@ -118,6 +115,7 @@ import java.util.*;
*
* 6. The Snapshot API, which provides data from the Overleaf app.
*
* @see SnapshotApiFacade - wraps a concrete instance of the Snapshot API.
* @see SnapshotApi - the interface for the Snapshot API.
* @see NetSnapshotApi - the default concrete implementation
*
@ -139,6 +137,8 @@ import java.util.*;
*/
public class Bridge {
private final Config config;
private final ProjectLock lock;
private final RepoStore repoStore;
@ -157,29 +157,31 @@ public class Bridge {
* swap store, and the swap job config.
*
* This should be the method used to create a Bridge.
* @param config The config to use
* @param repoStore The repo store to use
* @param dbStore The db store to use
* @param swapStore The swap store to use
* @param swapJobConfig The swap config to use, or empty for no-op
* @param snapshotApi The snapshot api to use
* @return The constructed Bridge.
*/
public static Bridge make(
Config config,
RepoStore repoStore,
DBStore dbStore,
SwapStore swapStore,
Optional<SwapJobConfig> swapJobConfig,
SnapshotApi snapshotApi
) {
ProjectLock lock = new ProjectLockImpl((int threads) ->
Log.info("Waiting for " + threads + " projects...")
);
return new Bridge(
config,
lock,
repoStore,
dbStore,
swapStore,
SwapJob.fromConfig(
swapJobConfig,
config.getSwapJob(),
lock,
repoStore,
dbStore,
@ -205,6 +207,7 @@ public class Bridge {
* @param resourceCache the {@link ResourceCache} to use
*/
Bridge(
Config config,
ProjectLock lock,
RepoStore repoStore,
DBStore dbStore,
@ -214,6 +217,7 @@ public class Bridge {
SnapshotApiFacade snapshotAPI,
ResourceCache resourceCache
) {
this.config = config;
this.lock = lock;
this.repoStore = repoStore;
this.dbStore = dbStore;
@ -621,6 +625,11 @@ public class Bridge {
makeCommitsFromSnapshots(repo, snapshots);
// TODO: in case crashes around here, add an
// "updating_from_commit" column to the DB as a way to rollback the
// any failed partial updates before re-trying
// Also need to consider the empty state (a new git init'd repo being
// the rollback target)
if (!snapshots.isEmpty()) {
dbStore.setLatestVersionForProject(
projectName,
@ -635,7 +644,7 @@ public class Bridge {
* Performs the actual Git commits on the disk.
*
* Each commit adds files to the db store
* ({@link ResourceCache#get(String, String, String, Map, Map)},
* ({@link ResourceCache#get(String, String, String, Map, Map, Optional)},
* and then removes any files that were deleted.
* @param repo The repository to commit to
* @param snapshots The snapshots to commit
@ -647,10 +656,25 @@ public class Bridge {
Collection<Snapshot> snapshots
) throws IOException, GitUserException {
String name = repo.getProjectName();
Optional<Long> maxSize = config
.getRepoStore()
.flatMap(RepoStoreConfig::getMaxFileSize);
for (Snapshot snapshot : snapshots) {
Map<String, RawFile> fileTable = repo.getFiles();
List<RawFile> files = new LinkedList<>();
RawDirectory directory = repo.getDirectory();
Map<String, RawFile> fileTable = directory.getFileTable();
List<RawFile> files = new ArrayList<>();
files.addAll(snapshot.getSrcs());
for (RawFile file : files) {
long size = file.size();
/* Can't throw in ifPresent... */
if (maxSize.isPresent()) {
long maxSize_ = maxSize.get();
if (size >= maxSize_) {
throw new SizeLimitExceededException(
Optional.of(file.getPath()), size, maxSize_);
}
}
}
Map<String, byte[]> fetchedUrls = new HashMap<>();
for (SnapshotAttachment snapshotAttachment : snapshot.getAtts()) {
files.add(
@ -659,7 +683,8 @@ public class Bridge {
snapshotAttachment.getUrl(),
snapshotAttachment.getPath(),
fileTable,
fetchedUrls
fetchedUrls,
maxSize
)
);
}

44
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStore.java
View file

@ -2,6 +2,8 @@ package uk.ac.ic.wlgitbridge.bridge.repo;
import com.google.api.client.repackaged.com.google.common.base.Preconditions;
import org.apache.commons.io.FileUtils;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.Repository;
import uk.ac.ic.wlgitbridge.util.Project;
import uk.ac.ic.wlgitbridge.util.Tar;
@ -12,6 +14,7 @@ import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Optional;
import java.util.function.Function;
import static uk.ac.ic.wlgitbridge.util.Util.deleteInDirectoryApartFrom;
@ -21,17 +24,35 @@ import static uk.ac.ic.wlgitbridge.util.Util.deleteInDirectoryApartFrom;
*/
public class FSGitRepoStore implements RepoStore {
private static final long DEFAULT_MAX_FILE_SIZE = 50 * 1024 * 1024;
private final String repoStorePath;
private final File rootDirectory;
private final long maxFileSize;
private final Function<File, Long> fsSizer;
public FSGitRepoStore(String repoStorePath) {
this(repoStorePath, d -> d.getTotalSpace() - d.getFreeSpace());
public FSGitRepoStore(
String repoStorePath,
Optional<Long> maxFileSize
) {
this(
repoStorePath,
maxFileSize.orElse(DEFAULT_MAX_FILE_SIZE),
d -> d.getTotalSpace() - d.getFreeSpace()
);
}
public FSGitRepoStore(String repoStorePath, Function<File, Long> fsSizer) {
public FSGitRepoStore(
String repoStorePath,
long maxFileSize,
Function<File, Long> fsSizer
) {
this.repoStorePath = repoStorePath;
rootDirectory = initRootGitDirectory(repoStorePath);
this.maxFileSize = maxFileSize;
this.fsSizer = fsSizer;
}
@ -47,16 +68,25 @@ public class FSGitRepoStore implements RepoStore {
@Override
public ProjectRepo initRepo(String project) throws IOException {
GitProjectRepo ret = new GitProjectRepo(project);
GitProjectRepo ret = GitProjectRepo.fromName(project);
ret.initRepo(this);
return ret;
return new WalkOverrideGitRepo(
ret, Optional.of(maxFileSize), Optional.empty());
}
@Override
public ProjectRepo getExistingRepo(String project) throws IOException {
GitProjectRepo ret = new GitProjectRepo(project);
GitProjectRepo ret = GitProjectRepo.fromName(project);
ret.useExistingRepository(this);
return ret;
return new WalkOverrideGitRepo(
ret, Optional.of(maxFileSize), Optional.empty());
}
@Override
public ProjectRepo useJGitRepo(Repository repo, ObjectId commitId) {
GitProjectRepo ret = GitProjectRepo.fromJGitRepo(repo);
return new WalkOverrideGitRepo(
ret, Optional.of(maxFileSize), Optional.of(commitId));
}
/* TODO: Perhaps we should just delete bad directories on the fly. */

24
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/GitProjectRepo.java
View file

@ -9,7 +9,7 @@ import org.eclipse.jgit.lib.PersonIdent;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.storage.file.FileRepositoryBuilder;
import uk.ac.ic.wlgitbridge.data.filestore.GitDirectoryContents;
import uk.ac.ic.wlgitbridge.data.filestore.RawFile;
import uk.ac.ic.wlgitbridge.data.filestore.RawDirectory;
import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
import uk.ac.ic.wlgitbridge.git.util.RepositoryObjectTreeWalker;
import uk.ac.ic.wlgitbridge.util.Log;
@ -43,10 +43,19 @@ public class GitProjectRepo implements ProjectRepo {
private final String projectName;
private Optional<Repository> repository;
public GitProjectRepo(String projectName) {
public static GitProjectRepo fromJGitRepo(Repository repo) {
return new GitProjectRepo(
repo.getWorkTree().getName(), Optional.of(repo));
}
public static GitProjectRepo fromName(String projectName) {
return new GitProjectRepo(projectName, Optional.empty());
}
GitProjectRepo(String projectName, Optional<Repository> repository) {
Preconditions.checkArgument(Project.isValidProjectName(projectName));
this.projectName = projectName;
repository = Optional.empty();
this.repository = repository;
}
@Override
@ -61,7 +70,10 @@ public class GitProjectRepo implements ProjectRepo {
initRepositoryField(repoStore);
Preconditions.checkState(repository.isPresent());
Repository repo = this.repository.get();
Preconditions.checkState(!repo.getObjectDatabase().exists());
// TODO: assert that this is a fresh repo. At the moment, we can't be
// sure whether the repo to be init'd doesn't exist or is just fresh
// and we crashed / aborted while committing
if (repo.getObjectDatabase().exists()) return;
repo.create();
}
@ -77,12 +89,12 @@ public class GitProjectRepo implements ProjectRepo {
}
@Override
public Map<String, RawFile> getFiles()
public RawDirectory getDirectory()
throws IOException, GitUserException {
Preconditions.checkState(repository.isPresent());
return new RepositoryObjectTreeWalker(
repository.get()
).getDirectoryContents().getFileTable();
).getDirectoryContents(Optional.empty());
}
@Override

7
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/ProjectRepo.java
View file

@ -2,13 +2,12 @@ package uk.ac.ic.wlgitbridge.bridge.repo;
import org.eclipse.jgit.lib.Repository;
import uk.ac.ic.wlgitbridge.data.filestore.GitDirectoryContents;
import uk.ac.ic.wlgitbridge.data.filestore.RawFile;
import uk.ac.ic.wlgitbridge.data.filestore.RawDirectory;
import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
import java.io.File;
import java.io.IOException;
import java.util.Collection;
import java.util.Map;
/**
* Created by winston on 20/08/2016.
@ -25,12 +24,12 @@ public interface ProjectRepo {
RepoStore repoStore
) throws IOException;
Map<String, RawFile> getFiles(
RawDirectory getDirectory(
) throws IOException, GitUserException;
Collection<String> commitAndGetMissing(
GitDirectoryContents gitDirectoryContents
) throws IOException;
) throws IOException, GitUserException;
void runGC() throws IOException;

5
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/RepoStore.java
View file

@ -1,5 +1,8 @@
package uk.ac.ic.wlgitbridge.bridge.repo;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.Repository;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
@ -21,6 +24,8 @@ public interface RepoStore {
ProjectRepo getExistingRepo(String project) throws IOException;
ProjectRepo useJGitRepo(Repository repo, ObjectId commitId);
void purgeNonexistentProjects(
Collection<String> existingProjectNames
);

21
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/RepoStoreConfig.java Normal file
View file

@ -0,0 +1,21 @@
package uk.ac.ic.wlgitbridge.bridge.repo;
import javax.annotation.Nullable;
import java.util.Optional;
/**
* Created by winston on 02/07/2017.
*/
public class RepoStoreConfig {
@Nullable
private final Long maxFileSize;
public RepoStoreConfig(Long maxFileSize) {
this.maxFileSize = maxFileSize;
}
public Optional<Long> getMaxFileSize() {
return Optional.ofNullable(maxFileSize);
}
}

95
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/WalkOverrideGitRepo.java Normal file
View file

@ -0,0 +1,95 @@
package uk.ac.ic.wlgitbridge.bridge.repo;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.Repository;
import uk.ac.ic.wlgitbridge.data.filestore.GitDirectoryContents;
import uk.ac.ic.wlgitbridge.data.filestore.RawDirectory;
import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
import uk.ac.ic.wlgitbridge.git.util.RepositoryObjectTreeWalker;
import java.io.File;
import java.io.IOException;
import java.util.Collection;
import java.util.Optional;
/**
* This class takes a GitProjectRepo and delegates all calls to it.
*
* The purpose is to insert a file size check in {@link #getDirectory()}.
*
* We delegate instead of subclass because we can't override the static
* constructors in {@link GitProjectRepo}.
*/
public class WalkOverrideGitRepo implements ProjectRepo {
private final GitProjectRepo gitRepo;
private final Optional<Long> maxFileSize;
private final Optional<ObjectId> commitId;
public WalkOverrideGitRepo(
GitProjectRepo gitRepo,
Optional<Long> maxFileSize,
Optional<ObjectId> commitId
) {
this.gitRepo = gitRepo;
this.maxFileSize = maxFileSize;
this.commitId = commitId;
}
@Override
public String getProjectName() {
return gitRepo.getProjectName();
}
@Override
public void initRepo(RepoStore repoStore) throws IOException {
gitRepo.initRepo(repoStore);
}
@Override
public void useExistingRepository(RepoStore repoStore) throws IOException {
gitRepo.useExistingRepository(repoStore);
}
@Override
public RawDirectory getDirectory() throws IOException, GitUserException {
Repository repo = gitRepo.getJGitRepository();
RepositoryObjectTreeWalker walker;
if (commitId.isPresent()) {
walker = new RepositoryObjectTreeWalker(repo, commitId.get());
} else {
walker = new RepositoryObjectTreeWalker(repo);
}
return walker.getDirectoryContents(maxFileSize);
}
@Override
public Collection<String> commitAndGetMissing(
GitDirectoryContents gitDirectoryContents
) throws GitUserException, IOException {
return gitRepo.commitAndGetMissing(gitDirectoryContents);
}
@Override
public void runGC() throws IOException {
gitRepo.runGC();
}
@Override
public void deleteIncomingPacks() throws IOException {
gitRepo.deleteIncomingPacks();
}
@Override
public File getProjectDir() {
return gitRepo.getProjectDir();
}
@Override
public Repository getJGitRepository() {
return gitRepo.getJGitRepository();
}
}

7
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/resource/ResourceCache.java
View file

@ -1,9 +1,11 @@
package uk.ac.ic.wlgitbridge.bridge.resource;
import uk.ac.ic.wlgitbridge.data.filestore.RawFile;
import uk.ac.ic.wlgitbridge.git.exception.SizeLimitExceededException;
import java.io.IOException;
import java.util.Map;
import java.util.Optional;
/**
* Created by winston on 20/08/2016.
@ -15,7 +17,8 @@ public interface ResourceCache {
String url,
String newPath,
Map<String, RawFile> fileTable,
Map<String, byte[]> fetchedUrls
) throws IOException;
Map<String, byte[]> fetchedUrls,
Optional<Long> maxFileSize
) throws IOException, SizeLimitExceededException;
}

51
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/resource/UrlResourceCache.java
View file

@ -1,18 +1,19 @@
package uk.ac.ic.wlgitbridge.bridge.resource;
import com.ning.http.client.AsyncCompletionHandler;
import com.ning.http.client.HttpResponseBodyPart;
import com.ning.http.client.Response;
import com.ning.http.client.*;
import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
import uk.ac.ic.wlgitbridge.data.filestore.RawFile;
import uk.ac.ic.wlgitbridge.data.filestore.RepositoryFile;
import uk.ac.ic.wlgitbridge.git.exception.SizeLimitExceededException;
import uk.ac.ic.wlgitbridge.snapshot.base.Request;
import uk.ac.ic.wlgitbridge.snapshot.exception.FailedConnectionException;
import uk.ac.ic.wlgitbridge.util.Log;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.ExecutionException;
/**
@ -32,13 +33,14 @@ public class UrlResourceCache implements ResourceCache {
String url,
String newPath,
Map<String, RawFile> fileTable,
Map<String, byte[]> fetchedUrls
) throws IOException {
Map<String, byte[]> fetchedUrls,
Optional<Long> maxFileSize
) throws IOException, SizeLimitExceededException {
String path = dbStore.getPathForURLInProject(projectName, url);
byte[] contents;
if (path == null) {
path = newPath;
contents = fetch(projectName, url, path);
contents = fetch(projectName, url, path, maxFileSize);
fetchedUrls.put(url, contents);
} else {
Log.info("Found (" + projectName + "): " + url);
@ -54,7 +56,7 @@ public class UrlResourceCache implements ResourceCache {
+ "File url is: "
+ url
);
contents = fetch(projectName, url, path);
contents = fetch(projectName, url, path, maxFileSize);
} else {
contents = rawFile.getContents();
}
@ -66,8 +68,9 @@ public class UrlResourceCache implements ResourceCache {
private byte[] fetch(
String projectName,
final String url,
String path
) throws FailedConnectionException {
String path,
Optional<Long> maxFileSize
) throws FailedConnectionException, SizeLimitExceededException {
byte[] contents;
Log.info("GET -> " + url);
try {
@ -76,6 +79,28 @@ public class UrlResourceCache implements ResourceCache {
ByteArrayOutputStream bytes = new ByteArrayOutputStream();
@Override
public STATE onHeadersReceived(
HttpResponseHeaders headers
) throws SizeLimitExceededException {
List<String> contentLengths
= headers.getHeaders().get("Content-Length");
if (!maxFileSize.isPresent()) {
return STATE.CONTINUE;
}
if (contentLengths.isEmpty()) {
return STATE.CONTINUE;
}
long contentLength = Long.parseLong(contentLengths.get(0));
long maxFileSize_ = maxFileSize.get();
if (contentLength <= maxFileSize_) {
return STATE.CONTINUE;
}
throw new SizeLimitExceededException(
Optional.of(path), contentLength, maxFileSize_
);
}
@Override
public STATE onBodyPartReceived(
HttpResponseBodyPart bodyPart
@ -115,6 +140,10 @@ public class UrlResourceCache implements ResourceCache {
);
throw new FailedConnectionException();
} catch (ExecutionException e) {
Throwable cause = e.getCause();
if (cause instanceof SizeLimitExceededException) {
throw (SizeLimitExceededException) cause;
}
Log.warn(
"ExecutionException when fetching project: " +
projectName +
@ -126,6 +155,10 @@ public class UrlResourceCache implements ResourceCache {
);
throw new FailedConnectionException();
}
if (maxFileSize.isPresent() && contents.length > maxFileSize.get()) {
throw new SizeLimitExceededException(
Optional.of(path), contents.length, maxFileSize.get());
}
dbStore.addURLIndexForProject(projectName, url, path);
return contents;
}

5
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/ServletFile.java
View file

@ -25,6 +25,11 @@ public class ServletFile extends RawFile {
return file.getContents();
}
@Override
public long size() {
return getContents().length;
}
public boolean isChanged() {
return changed;
}

3
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/filestore/RawDirectory.java
View file

@ -1,6 +1,9 @@
package uk.ac.ic.wlgitbridge.data.filestore;
import uk.ac.ic.wlgitbridge.git.exception.SizeLimitExceededException;
import java.util.Map;
import java.util.Optional;
/**
* Created by Winston on 16/11/14.

2
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/filestore/RawFile.java
View file

@ -17,6 +17,8 @@ public abstract class RawFile {
public abstract byte[] getContents();
public abstract long size();
public final void writeToDisk(File directory) throws IOException {
File file = new File(directory, getPath());
file.getParentFile().mkdirs();

5
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/filestore/RepositoryFile.java
View file

@ -23,4 +23,9 @@ public class RepositoryFile extends RawFile {
return contents;
}
@Override
public long size() {
return contents.length;
}
}

8
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/handler/WLReceivePackFactory.java
View file

@ -5,6 +5,7 @@ import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.transport.ReceivePack;
import org.eclipse.jgit.transport.resolver.ReceivePackFactory;
import uk.ac.ic.wlgitbridge.bridge.Bridge;
import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore;
import uk.ac.ic.wlgitbridge.bridge.snapshot.SnapshotApi;
import uk.ac.ic.wlgitbridge.git.handler.hook.WriteLatexPutHook;
import uk.ac.ic.wlgitbridge.git.servlet.WLGitServlet;
@ -28,9 +29,12 @@ import java.util.Optional;
public class WLReceivePackFactory
implements ReceivePackFactory<HttpServletRequest> {
private final RepoStore repoStore;
private final Bridge bridge;
public WLReceivePackFactory(Bridge bridge) {
public WLReceivePackFactory(RepoStore repoStore, Bridge bridge) {
this.repoStore = repoStore;
this.bridge = bridge;
}
@ -69,7 +73,7 @@ public class WLReceivePackFactory
hostname = httpServletRequest.getLocalName();
}
receivePack.setPreReceiveHook(
new WriteLatexPutHook(bridge, hostname, oauth2)
new WriteLatexPutHook(repoStore, bridge, hostname, oauth2)
);
return receivePack;
}

20
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/handler/hook/WriteLatexPutHook.java
View file

@ -7,12 +7,12 @@ import org.eclipse.jgit.transport.ReceiveCommand;
import org.eclipse.jgit.transport.ReceiveCommand.Result;
import org.eclipse.jgit.transport.ReceivePack;
import uk.ac.ic.wlgitbridge.bridge.Bridge;
import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore;
import uk.ac.ic.wlgitbridge.data.filestore.RawDirectory;
import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
import uk.ac.ic.wlgitbridge.git.handler.WLReceivePackFactory;
import uk.ac.ic.wlgitbridge.git.handler.hook.exception.ForcedPushException;
import uk.ac.ic.wlgitbridge.git.handler.hook.exception.WrongBranchException;
import uk.ac.ic.wlgitbridge.git.util.RepositoryObjectTreeWalker;
import uk.ac.ic.wlgitbridge.snapshot.push.exception.InternalErrorException;
import uk.ac.ic.wlgitbridge.snapshot.push.exception.OutOfDateException;
import uk.ac.ic.wlgitbridge.snapshot.push.exception.SnapshotPostException;
@ -33,6 +33,8 @@ import java.util.Optional;
*/
public class WriteLatexPutHook implements PreReceiveHook {
private final RepoStore repoStore;
private final Bridge bridge;
private final String hostname;
private final Optional<Credential> oauth2;
@ -41,15 +43,18 @@ public class WriteLatexPutHook implements PreReceiveHook {
* The constructor to use, which provides the hook with the {@link Bridge},
* the hostname (used to construct a URL to give to Overleaf to postback),
* and the oauth2 (used to authenticate with the Snapshot API).
* @param repoStore
* @param bridge the {@link Bridge}
* @param hostname the hostname used for postback from the Snapshot API
* @param oauth2 used to authenticate with the snapshot API, or null
*/
public WriteLatexPutHook(
RepoStore repoStore,
Bridge bridge,
String hostname,
Optional<Credential> oauth2
) {
this.repoStore = repoStore;
this.bridge = bridge;
this.hostname = hostname;
this.oauth2 = oauth2;
@ -152,18 +157,17 @@ public class WriteLatexPutHook implements PreReceiveHook {
Repository repository,
ReceiveCommand receiveCommand
) throws IOException, GitUserException {
return new RepositoryObjectTreeWalker(
repository,
receiveCommand.getNewId()
).getDirectoryContents();
return repoStore
.useJGitRepo(repository, receiveCommand.getNewId())
.getDirectory();
}
private RawDirectory getOldDirectoryContents(
Repository repository
) throws IOException, GitUserException {
return new RepositoryObjectTreeWalker(
repository
).getDirectoryContents();
return repoStore
.useJGitRepo(repository, repository.resolve("HEAD"))
.getDirectory();
}
}

4
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/servlet/WLGitServlet.java
View file

@ -3,6 +3,7 @@ package uk.ac.ic.wlgitbridge.git.servlet;
import org.eclipse.jetty.servlet.ServletContextHandler;
import org.eclipse.jgit.http.server.GitServlet;
import uk.ac.ic.wlgitbridge.bridge.Bridge;
import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore;
import uk.ac.ic.wlgitbridge.git.handler.WLReceivePackFactory;
import uk.ac.ic.wlgitbridge.git.handler.WLRepositoryResolver;
import uk.ac.ic.wlgitbridge.git.handler.WLUploadPackFactory;
@ -39,10 +40,11 @@ public class WLGitServlet extends GitServlet {
*/
public WLGitServlet(
ServletContextHandler ctxHandler,
RepoStore repoStore,
Bridge bridge
) throws ServletException {
setRepositoryResolver(new WLRepositoryResolver(bridge));
setReceivePackFactory(new WLReceivePackFactory(bridge));
setReceivePackFactory(new WLReceivePackFactory(repoStore, bridge));
setUploadPackFactory(new WLUploadPackFactory());
init(new WLGitServletConfig(ctxHandler));
}

14
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/util/RepositoryObjectTreeWalker.java
View file

@ -47,19 +47,7 @@ public class RepositoryObjectTreeWalker {
this(repository, repository.resolve("HEAD~" + fromHead));
}
public RawDirectory getDirectoryContents(
) throws IOException, SizeLimitExceededException, InvalidGitRepository {
return getDirectoryContents(Optional.empty());
}
public RawDirectory getDirectoryContents(long maxFileSize)
throws InvalidGitRepository,
SizeLimitExceededException,
IOException {
return getDirectoryContents(Optional.of(maxFileSize));
}
private RawDirectory getDirectoryContents(Optional<Long> maxFileSize)
public RawDirectory getDirectoryContents(Optional<Long> maxFileSize)
throws IOException,
SizeLimitExceededException,
InvalidGitRepository {

15
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/GitBridgeServer.java
View file

@ -13,6 +13,7 @@ import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SqliteDBStore;
import uk.ac.ic.wlgitbridge.bridge.repo.FSGitRepoStore;
import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore;
import uk.ac.ic.wlgitbridge.bridge.repo.RepoStoreConfig;
import uk.ac.ic.wlgitbridge.bridge.snapshot.NetSnapshotApi;
import uk.ac.ic.wlgitbridge.bridge.snapshot.SnapshotApi;
import uk.ac.ic.wlgitbridge.bridge.swap.store.SwapStore;
@ -52,7 +53,10 @@ public class GitBridgeServer {
org.eclipse.jetty.util.log.Log.setLog(new NullLogger());
this.port = config.getPort();
this.rootGitDirectoryPath = config.getRootGitDirectory();
RepoStore repoStore = new FSGitRepoStore(rootGitDirectoryPath);
RepoStore repoStore = new FSGitRepoStore(
rootGitDirectoryPath,
config.getRepoStore().flatMap(RepoStoreConfig::getMaxFileSize)
);
DBStore dbStore = new SqliteDBStore(
Paths.get(
repoStore.getRootDirectory().getAbsolutePath()
@ -61,14 +65,14 @@ public class GitBridgeServer {
SwapStore swapStore = SwapStore.fromConfig(config.getSwapStore());
SnapshotApi snapshotApi = new NetSnapshotApi();
bridge = Bridge.make(
config,
repoStore,
dbStore,
swapStore,
config.getSwapJob(),
snapshotApi
);
jettyServer = new Server(port);
configureJettyServer(config, snapshotApi);
configureJettyServer(config, repoStore, snapshotApi);
SnapshotAPIRequest.setBasicAuth(
config.getUsername(),
config.getPassword()
@ -110,11 +114,12 @@ public class GitBridgeServer {
private void configureJettyServer(
Config config,
RepoStore repoStore,
SnapshotApi snapshotApi
) throws ServletException {
HandlerCollection handlers = new HandlerList();
handlers.addHandler(initApiHandler());
handlers.addHandler(initGitHandler(config, snapshotApi));
handlers.addHandler(initGitHandler(config, repoStore, snapshotApi));
jettyServer.setHandler(handlers);
}
@ -136,6 +141,7 @@ public class GitBridgeServer {
private Handler initGitHandler(
Config config,
RepoStore repoStore,
SnapshotApi snapshotApi
) throws ServletException {
final ServletContextHandler servletContextHandler =
@ -153,6 +159,7 @@ public class GitBridgeServer {
new ServletHolder(
new WLGitServlet(
servletContextHandler,
repoStore,
bridge
)
),

5
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/snapshot/getforversion/SnapshotFile.java
View file

@ -44,6 +44,11 @@ public class SnapshotFile extends RawFile implements JSONSource {
return contents;
}
@Override
public long size() {
return contents.length;
}
/* Mock server */
public SnapshotFile(String contents, String path) {

8
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Instance.java
View file

@ -21,9 +21,11 @@ public class Instance {
public static final JsonFactory jsonFactory = new GsonFactory();
public static final Gson prettyGson =
new GsonBuilder(
).setPrettyPrinting().disableHtmlEscaping().create();
public static final Gson prettyGson = new GsonBuilder()
.setPrettyPrinting()
.serializeNulls()
.disableHtmlEscaping()
.create();
public static final Gson gson = new Gson();

5
services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/application/config/ConfigTest.java
View file

@ -95,7 +95,10 @@ public class ConfigTest {
" \"oauth2ClientID\": \"<oauth2ClientID>\",\n" +
" \"oauth2ClientSecret\": \"<oauth2ClientSecret>\",\n" +
" \"oauth2Server\": \"https://www.overleaf.com\"\n" +
" }\n" +
" },\n" +
" \"repoStore\": null,\n" +
" \"swapStore\": null,\n" +
" \"swapJob\": null\n" +
"}";
assertEquals(
"sanitised config did not hide sensitive fields",

13
services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/BridgeTest.java
View file

@ -2,6 +2,7 @@ package uk.ac.ic.wlgitbridge.bridge;
import org.junit.Before;
import org.junit.Test;
import uk.ac.ic.wlgitbridge.application.config.Config;
import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
import uk.ac.ic.wlgitbridge.bridge.db.ProjectState;
import uk.ac.ic.wlgitbridge.bridge.gc.GcJob;
@ -50,6 +51,18 @@ public class BridgeTest {
swapJob = mock(SwapJob.class);
gcJob = mock(GcJob.class);
bridge = new Bridge(
new Config(
0,
"",
"",
"",
"",
"",
"",
null,
null,
null,
null),
lock,
repoStore,
dbStore,

3
services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStoreTest.java
View file

@ -10,6 +10,7 @@ import java.io.*;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.Optional;
import static org.junit.Assert.*;
@ -42,7 +43,7 @@ public class FSGitRepoStoreTest {
File tmp = makeTempRepoDir(tmpFolder, "rootdir");
original = tmpFolder.newFolder("original");
FileUtils.copyDirectory(tmp, original);
repoStore = new FSGitRepoStore(tmp.getAbsolutePath());
repoStore = new FSGitRepoStore(tmp.getAbsolutePath(), Optional.empty());
}
@Test

4
services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/repo/GitProjectRepoTest.java
View file

@ -56,7 +56,7 @@ public class GitProjectRepoTest {
@Before
public void setup() throws IOException {
rootdir = makeTempRepoDir(tmpFolder, "rootdir");
repoStore = new FSGitRepoStore(rootdir.getAbsolutePath());
repoStore = new FSGitRepoStore(rootdir.getAbsolutePath(), Optional.empty());
repo = fromExistingDir("repo");
badGitignore = fromExistingDir("badgitignore");
incoming = fromExistingDir("incoming");
@ -64,7 +64,7 @@ public class GitProjectRepoTest {
}
private GitProjectRepo fromExistingDir(String dir) throws IOException {
GitProjectRepo ret = new GitProjectRepo(dir);
GitProjectRepo ret = GitProjectRepo.fromName(dir);
ret.useExistingRepository(repoStore);
return ret;
}

1
services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJobImplTest.java
View file

@ -45,6 +45,7 @@ public class SwapJobImplTest {
tmpFolder,
"repostore"
).getAbsolutePath(),
100_000,
FileUtils::sizeOfDirectory
);
dbStore = new SqliteDBStore(tmpFolder.newFile());

15
services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/data/model/ResourceFetcherTest.java
View file

@ -9,7 +9,6 @@ import org.mockserver.client.server.MockServerClient;
import org.mockserver.junit.MockServerRule;
import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
import uk.ac.ic.wlgitbridge.bridge.repo.FSGitRepoStore;
import uk.ac.ic.wlgitbridge.bridge.repo.GitProjectRepo;
import uk.ac.ic.wlgitbridge.bridge.repo.ProjectRepo;
import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore;
import uk.ac.ic.wlgitbridge.bridge.resource.ResourceCache;
@ -20,6 +19,7 @@ import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
import static org.junit.Assert.assertEquals;
import static org.mockserver.model.HttpRequest.request;
@ -67,12 +67,13 @@ public class ResourceFetcherTest {
TemporaryFolder repositoryFolder = new TemporaryFolder();
repositoryFolder.create();
String repoStorePath = repositoryFolder.getRoot().getAbsolutePath();
RepoStore repoStore = new FSGitRepoStore(repoStorePath);
ProjectRepo repo = new GitProjectRepo("repo");
repo.initRepo(repoStore);
Map<String, RawFile> fileTable = repo.getFiles();
Map<String, byte[]> fetchedUrls = new HashMap<String, byte[]>();
resources.get(testProjectName, testUrl, newTestPath, fileTable, fetchedUrls);
RepoStore repoStore = new FSGitRepoStore(repoStorePath, Optional.empty());
ProjectRepo repo = repoStore.initRepo("repo");
Map<String, RawFile> fileTable = repo.getDirectory().getFileTable();
Map<String, byte[]> fetchedUrls = new HashMap<>();
resources.get(
testProjectName, testUrl, newTestPath,
fileTable, fetchedUrls, Optional.empty());
// We don't bother caching in this case, at present.
assertEquals(0, fetchedUrls.size());