github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2025-04-05 17:00:55 +00:00
Code Issues Projects Releases Packages Wiki Activity

Add implementations, implement S3SwapStore (with only tars), FSRepoStore, Tar and File utils, add tests

Browse source
This commit is contained in:
Winston Li 2016-08-23 17:51:56 +01:00 committed by Michael Mazour
parent 1850689a63
commit 8c0937511e
32 changed files with 848 additions and 34 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
services/git-bridge/pom.xml
View file

@ -140,5 +140,29 @@
<artifactId>mockito-core</artifactId>
<version>1.10.19</version>
</dependency>
<!-- https://mvnrepository.com/artifact/com.amazonaws/aws-java-sdk -->
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk</artifactId>
<version>1.11.28</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.httpcomponents/httpclient -->
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.5.2</version>
</dependency>
<!-- https://mvnrepository.com/artifact/commons-io/commons-io -->
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.5</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.commons/commons-compress -->
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId>
<version>1.12</version>
</dependency>
</dependencies>
</project>

5
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/Bridge.java
View file

@ -30,6 +30,7 @@ import uk.ac.ic.wlgitbridge.snapshot.push.exception.*;
import uk.ac.ic.wlgitbridge.util.Log;
import java.io.IOException;
import java.time.Duration;
import java.util.*;
/**
@ -104,8 +105,8 @@ public class Bridge {
Log.info("Bye");
}
public void startSwapJob(int intervalMillis) {
swapJob.start(intervalMillis);
public void startSwapJob(Duration interval) {
swapJob.start(interval);
}
/* TODO: Remove these when WLBridged is moved into RepoStore */

10
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/DBStore.java
View file

@ -1,5 +1,6 @@
package uk.ac.ic.wlgitbridge.bridge.db;
import java.sql.Timestamp;
import java.util.List;
/**
@ -19,4 +20,13 @@ public interface DBStore {
String getPathForURLInProject(String projectName, String url);
String getOldestUnswappedProject();
/**
* Sets the last accessed time for the given project name.
* @param projectName the project's name
* @param time the time, or null if the project is to be swapped
*/
void setLastAccessedTime(String projectName, Timestamp time);
}

11
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/SqliteDBStore.java
View file

@ -5,6 +5,7 @@ import uk.ac.ic.wlgitbridge.util.Log;
import java.io.File;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.util.Arrays;
import java.util.List;
@ -84,4 +85,14 @@ public class SqliteDBStore implements DBStore {
}
}
@Override
public String getOldestUnswappedProject() {
throw new UnsupportedOperationException();
}
@Override
public void setLastAccessedTime(String projectName, Timestamp time) {
throw new UnsupportedOperationException();
}
}

61
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/FSRepoStore.java
View file

@ -1,12 +1,20 @@
package uk.ac.ic.wlgitbridge.bridge.repo;
import uk.ac.ic.wlgitbridge.util.Util;
import com.google.api.client.repackaged.com.google.common.base.Preconditions;
import org.apache.commons.io.FileUtils;
import uk.ac.ic.wlgitbridge.util.Project;
import uk.ac.ic.wlgitbridge.util.Tar;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import static uk.ac.ic.wlgitbridge.util.Util.deleteInDirectoryApartFrom;
/**
* Created by winston on 20/08/2016.
*/
@ -30,6 +38,7 @@ public class FSRepoStore implements RepoStore {
return rootDirectory;
}
/* TODO: Perhaps we should just delete bad directories on the fly. */
@Override
public void purgeNonexistentProjects(
Collection<String> existingProjectNames
@ -37,15 +46,63 @@ public class FSRepoStore implements RepoStore {
List<String> excludedFromDeletion =
new ArrayList<>(existingProjectNames);
excludedFromDeletion.add(".wlgb");
Util.deleteInDirectoryApartFrom(
deleteInDirectoryApartFrom(
rootDirectory,
excludedFromDeletion.toArray(new String[] {})
);
}
@Override
public long totalSize() {
return FileUtils.sizeOfDirectory(rootDirectory);
}
@Override
public InputStream bzip2Project(String projectName) throws IOException {
Preconditions.checkArgument(Project.isValidProjectName(projectName));
return Tar.tar(getDotGitForProject(projectName));
}
@Override
public void remove(String projectName) throws IOException {
Preconditions.checkArgument(Project.isValidProjectName(projectName));
FileUtils.deleteDirectory(new File(rootDirectory, projectName));
}
@Override
public void unbzip2Project(
String projectName,
InputStream dataStream
) throws IOException {
Preconditions.checkArgument(Project.isValidProjectName(projectName));
Preconditions.checkState(getDirForProject(projectName).mkdirs());
Tar.untar(dataStream, getDirForProject(projectName));
}
private File getDirForProject(String projectName) {
Preconditions.checkArgument(Project.isValidProjectName(projectName));
return Paths.get(
rootDirectory.getAbsolutePath()
).resolve(
projectName
).toFile();
}
private File getDotGitForProject(String projectName) {
Preconditions.checkArgument(Project.isValidProjectName(projectName));
return Paths.get(
rootDirectory.getAbsolutePath()
).resolve(
projectName
).resolve(
".git"
).toFile();
}
private File initRootGitDirectory(String rootGitDirectoryPath) {
File rootGitDirectory = new File(rootGitDirectoryPath);
rootGitDirectory.mkdirs();
Preconditions.checkArgument(rootGitDirectory.isDirectory());
return rootGitDirectory;
}

25
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/RepoStore.java
View file

@ -1,8 +1,8 @@
package uk.ac.ic.wlgitbridge.bridge.repo;
import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.util.Collection;
/**
@ -14,9 +14,28 @@ public interface RepoStore {
File getRootDirectory();
void purgeNonexistentProjects(
Collection<String> existingProjectNames
);
long totalSize();
/*
* Tars and bzip2s the .git directory of the given project. Throws an
* IOException if the project doesn't exist. The returned stream is a copy
* of the original .git directory, which must be deleted using remove().
*/
InputStream bzip2Project(String projectName) throws IOException;
void remove(String projectName) throws IOException;
/**
* Unbzip2s the given data stream into a .git directory for projectName.
* Creates the project directory.
* If projectName already exists, throws an IOException.
* @param projectName the name of the project, e.g. abc123
* @param dataStream the data stream containing the bzipped contents.
*/
void unbzip2Project(String projectName, InputStream dataStream) throws IOException;
}

64
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/S3SwapStore.java Normal file
View file

@ -0,0 +1,64 @@
package uk.ac.ic.wlgitbridge.bridge.swap;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.model.*;
import java.io.InputStream;
/**
* Created by winston on 21/08/2016.
*/
public class S3SwapStore implements SwapStore {
private final AmazonS3 s3;
private final String bucketName;
public S3SwapStore(
String accessKey,
String secret,
String bucketName
) {
s3 = new AmazonS3Client(new BasicAWSCredentials(accessKey, secret));
this.bucketName = bucketName;
}
@Override
public void upload(
String projectName,
InputStream uploadStream,
long contentLength
) {
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentLength(contentLength);
PutObjectRequest put = new PutObjectRequest(
bucketName,
projectName,
uploadStream,
metadata
);
PutObjectResult res = s3.putObject(put);
}
@Override
public InputStream openDownloadStream(String projectName) {
GetObjectRequest get = new GetObjectRequest(
bucketName,
projectName
);
S3Object res = s3.getObject(get);
return res.getObjectContent();
}
@Override
public void remove(String projectName) {
DeleteObjectRequest del = new DeleteObjectRequest(
bucketName,
projectName
);
s3.deleteObject(del);
}
}

4
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapJob.java
View file

@ -1,11 +1,13 @@
package uk.ac.ic.wlgitbridge.bridge.swap;
import java.time.Duration;
/**
* Created by winston on 20/08/2016.
*/
public interface SwapJob {
void start(int intervalMillis);
void start(Duration interval);
void stop();

17
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapJobImpl.java
View file

@ -3,9 +3,10 @@ package uk.ac.ic.wlgitbridge.bridge.swap;
import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock;
import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore;
import uk.ac.ic.wlgitbridge.util.Util;
import java.time.Duration;
import java.util.Timer;
import java.util.concurrent.atomic.AtomicInteger;
/**
* Created by winston on 20/08/2016.
@ -19,10 +20,13 @@ public class SwapJobImpl implements SwapJob {
private final Timer timer;
final AtomicInteger swaps;
public SwapJobImpl(
ProjectLock lock,
RepoStore repoStore,
DBStore dbStore, SwapStore swapStore
DBStore dbStore,
SwapStore swapStore
) {
this.lock = lock;
@ -30,14 +34,15 @@ public class SwapJobImpl implements SwapJob {
this.swapStore = swapStore;
this.dbStore = dbStore;
timer = new Timer();
swaps = new AtomicInteger(0);
}
@Override
public void start(int intervalMillis) {
public void start(Duration interval) {
timer.scheduleAtFixedRate(
Util.makeTimerTask(this::doSwap),
uk.ac.ic.wlgitbridge.util.Timer.makeTimerTask(this::doSwap),
0,
intervalMillis
interval.toMillis()
);
}
@ -47,7 +52,7 @@ public class SwapJobImpl implements SwapJob {
}
private void doSwap() {
throw new UnsupportedOperationException();
swaps.incrementAndGet();
}
}

9
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapStore.java
View file

@ -1,7 +1,16 @@
package uk.ac.ic.wlgitbridge.bridge.swap;
import java.io.InputStream;
/**
* Created by winston on 20/08/2016.
*/
public interface SwapStore {
void upload(String projectName, InputStream uploadStream, long contentLength);
InputStream openDownloadStream(String projectName);
void remove(String projectName);
}

18
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/GitBridgeServer.java
View file

@ -24,6 +24,7 @@ import javax.servlet.DispatcherType;
import javax.servlet.Filter;
import javax.servlet.ServletException;
import java.io.File;
import java.io.InputStream;
import java.net.BindException;
import java.util.EnumSet;
@ -50,7 +51,22 @@ public class GitBridgeServer {
this.rootGitDirectoryPath = config.getRootGitDirectory();
RepoStore repoStore = new FSRepoStore(rootGitDirectoryPath);
DBStore dbStore = new SqliteDBStore(repoStore.getRootDirectory());
SwapStore swapStore = new SwapStore() {};
SwapStore swapStore = new SwapStore() {
@Override
public void upload(String projectName, InputStream uploadStream, long contentLength) {
}
@Override
public InputStream openDownloadStream(String projectName) {
return null;
}
@Override
public void remove(String projectName) {
}
};
bridgeAPI = Bridge.make(
repoStore,
dbStore,

127
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Files.java Normal file
View file

@ -0,0 +1,127 @@
package uk.ac.ic.wlgitbridge.util;
import com.google.api.client.repackaged.com.google.common.base.Preconditions;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.filefilter.TrueFileFilter;
import java.io.File;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.Set;
import java.util.stream.Collectors;
/**
* Created by winston on 23/08/2016.
*/
public class Files {
private Files() {}
public static boolean contentsAreEqual(
File f0,
File f1
) throws IOException {
try {
return uncheckedContentsAreEqual(f0, f1);
} catch (UncheckedIOException e) {
throw e.getCause();
}
}
public static void renameAll(
File fileOrDir,
String from,
String to
) {
if (fileOrDir.isDirectory()) {
File f = doRename(fileOrDir, from, to);
for (File c : f.listFiles()) {
renameAll(c, from, to);
}
} else if (fileOrDir.isFile()) {
doRename(fileOrDir, from, to);
} else {
throw new IllegalArgumentException(
"not a file or dir: " + fileOrDir
);
}
}
private static File doRename(File fileOrDir, String from, String to) {
if (!fileOrDir.getName().equals(from)) {
return fileOrDir;
}
File renamed = new File(fileOrDir.getParent(), to);
Preconditions.checkState(fileOrDir.renameTo(renamed));
return renamed;
}
private static boolean uncheckedContentsAreEqual(File f0, File f1) throws IOException {
if (f0.equals(f1)) {
return true;
}
if (!f0.isDirectory() || !f1.isDirectory()) {
return !f0.isDirectory() && !f1.isDirectory() &&
Arrays.equals(
FileUtils.readFileToByteArray(f0),
FileUtils.readFileToByteArray(f1)
);
}
Path f0Base = Paths.get(f0.getAbsolutePath());
Path f1Base = Paths.get(f1.getAbsolutePath());
Set<Path> children0 = getChildren(f0, f0Base);
Set<Path> children1 = getChildren(f1, f1Base);
if (children0.size() != children1.size()) {
return false;
}
return children0.stream(
).allMatch(c0 ->
children1.contains(c0) && childEquals(c0, f0Base, f1Base)
);
}
private static Set<Path> getChildren(File f0, Path f0Base) {
return FileUtils.listFilesAndDirs(
f0,
TrueFileFilter.TRUE,
TrueFileFilter.TRUE
).stream(
).map(
File::getAbsolutePath
).map(
Paths::get
).map(p ->
f0Base.relativize(p)
).filter(p ->
!p.toString().isEmpty()
).collect(
Collectors.toSet()
);
}
private static boolean childEquals(
Path child,
Path f0Base,
Path f1Base
) throws UncheckedIOException {
File c0 = f0Base.resolve(child).toFile();
File c1 = f1Base.resolve(child).toFile();
boolean c0IsDir = c0.isDirectory();
boolean c1IsDir = c1.isDirectory();
if (c0IsDir || c1IsDir) {
return c0IsDir && c1IsDir;
}
try {
return c0.isFile() && c1.isFile() && Arrays.equals(
FileUtils.readFileToByteArray(c0),
FileUtils.readFileToByteArray(c1)
);
} catch (IOException e) {
throw new UncheckedIOException(e);
}
}
}

13
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Project.java Normal file
View file

@ -0,0 +1,13 @@
package uk.ac.ic.wlgitbridge.util;
/**
* Created by winston on 23/08/2016.
*/
public class Project {
public static boolean isValidProjectName(String projectName) {
return projectName != null && !projectName.isEmpty()
&& !projectName.startsWith(".");
}
}

106
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Tar.java Normal file
View file

@ -0,0 +1,106 @@
package uk.ac.ic.wlgitbridge.util;
import com.google.api.client.repackaged.com.google.common.base.Preconditions;
import org.apache.commons.compress.archivers.ArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
import org.apache.commons.compress.utils.IOUtils;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.output.ByteArrayOutputStream;
import java.io.*;
import java.nio.file.Path;
import java.nio.file.Paths;
/**
* Created by winston on 23/08/2016.
*/
public class Tar {
private Tar() {}
public static InputStream tar(File fileOrDir) throws IOException {
ByteArrayOutputStream bout = new ByteArrayOutputStream();
TarArchiveOutputStream tout = new TarArchiveOutputStream(bout);
addTarEntry(
tout,
Paths.get(fileOrDir.getParentFile().getAbsolutePath()),
fileOrDir
);
tout.close();
return new ByteArrayInputStream(bout.toByteArray());
}
private static void addTarEntry(
TarArchiveOutputStream tout,
Path base,
File fileOrDir
) throws IOException {
if (fileOrDir.isDirectory()) {
addTarDir(tout, base, fileOrDir);
} else if (fileOrDir.isFile()) {
addTarFile(tout, base, fileOrDir);
} else {
throw new IllegalArgumentException(
"invalid file or dir: " + fileOrDir
);
}
}
private static void addTarDir(
TarArchiveOutputStream tout,
Path base,
File dir
) throws IOException {
Preconditions.checkArgument(dir.isDirectory());
String name = base.relativize(
Paths.get(dir.getAbsolutePath())
).toString();
ArchiveEntry entry = tout.createArchiveEntry(dir, name);
tout.putArchiveEntry(entry);
tout.closeArchiveEntry();
for (File f : dir.listFiles()) {
addTarEntry(tout, base, f);
}
}
private static void addTarFile(
TarArchiveOutputStream tout,
Path base,
File file
) throws IOException {
Preconditions.checkArgument(file.isFile());
String name = base.relativize(
Paths.get(file.getAbsolutePath())
).toString();
ArchiveEntry entry = tout.createArchiveEntry(file, name);
tout.putArchiveEntry(entry);
tout.write(FileUtils.readFileToByteArray(file));
tout.closeArchiveEntry();
}
public static void untar(InputStream tar, File parentDir) throws IOException {
TarArchiveInputStream tin = new TarArchiveInputStream(tar);
ArchiveEntry e;
while ((e = tin.getNextEntry()) != null) {
File f = new File(parentDir, e.getName());
f.setLastModified(e.getLastModifiedDate().getTime());
f.getParentFile().mkdirs();
if (e.isDirectory()) {
f.mkdir();
continue;
}
long size = e.getSize();
Preconditions.checkArgument(
size > 0 && size < Integer.MAX_VALUE,
"file too big: tar should have thrown an IOException"
);
try (OutputStream out = new FileOutputStream(f)) {
/* TarInputStream pretends each
entry's EOF is the stream's EOF */
IOUtils.copy(tin, out);
}
}
}
}

19
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Timer.java Normal file
View file

@ -0,0 +1,19 @@
package uk.ac.ic.wlgitbridge.util;
import java.util.TimerTask;
/**
* Created by winston on 23/08/2016.
*/
public class Timer {
public static TimerTask makeTimerTask(Runnable lamb) {
return new TimerTask() {
@Override
public void run() {
lamb.run();
}
};
}
}

9
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Util.java
View file

@ -19,15 +19,6 @@ public class Util {
private static String POSTBACK_URL;
private static final DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSSSSS");
public static TimerTask makeTimerTask(Runnable lamb) {
return new TimerTask() {
@Override
public void run() {
lamb.run();
}
};
}
public static String entries(int entries) {
if (entries == 1) {
return "entry";

7
services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/BridgeTest.java
View file

@ -10,6 +10,8 @@ import uk.ac.ic.wlgitbridge.bridge.snapshot.SnapshotAPI;
import uk.ac.ic.wlgitbridge.bridge.swap.SwapJob;
import uk.ac.ic.wlgitbridge.bridge.swap.SwapStore;
import java.time.Duration;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
@ -35,6 +37,7 @@ public class BridgeTest {
dbStore = mock(DBStore.class);
swapStore = mock(SwapStore.class);
snapshotAPI = mock(SnapshotAPI.class);
resourceCache = mock(ResourceCache.class);
swapJob = mock(SwapJob.class);
bridge = new Bridge(
lock,
@ -49,9 +52,9 @@ public class BridgeTest {
@Test
public void shutdownStopsSwapJob() {
bridge.startSwapJob(1000);
bridge.startSwapJob(Duration.ofSeconds(1));
bridge.doShutdown();
verify(swapJob).start(1000);
verify(swapJob).start(Duration.ofSeconds(1));
verify(swapJob).stop();
}

75
services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/repo/FSRepoStoreTest.java Normal file
View file

@ -0,0 +1,75 @@
package uk.ac.ic.wlgitbridge.bridge.repo;
import org.apache.commons.io.FileUtils;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import uk.ac.ic.wlgitbridge.util.Files;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Arrays;
import static org.junit.Assert.assertEquals;
/**
* Created by winston on 23/08/2016.
*/
public class FSRepoStoreTest {
private FSRepoStore repoStore;
private File original;
@Before
public void setup() throws IOException {
TemporaryFolder tmpFolder = new TemporaryFolder();
tmpFolder.create();
File tmp = tmpFolder.newFolder("repostore");
Path rootdir = Paths.get(
"src/test/resources/uk/ac/ic/wlgitbridge/"
+ "bridge/repo/FSRepoStoreTest/rootdir"
);
FileUtils.copyDirectory(rootdir.toFile(), tmp);
Files.renameAll(tmp, "DOTgit", ".git");
original = tmpFolder.newFolder("original");
FileUtils.copyDirectory(tmp, original);
repoStore = new FSRepoStore(tmp.getAbsolutePath());
}
@Test
public void testPurgeNonexistentProjects() {
File toDelete = new File(repoStore.getRootDirectory(), "idontexist");
File wlgb = new File(repoStore.getRootDirectory(), ".wlgb");
Assert.assertTrue(toDelete.exists());
Assert.assertTrue(wlgb.exists());
repoStore.purgeNonexistentProjects(Arrays.asList("proj1", "proj2"));
Assert.assertFalse(toDelete.exists());
Assert.assertTrue(wlgb.exists());
}
@Test
public void testTotalSize() {
assertEquals(31860, repoStore.totalSize());
}
@Test
public void zipAndUnzipShouldBeTheSame() throws IOException {
long beforeSize = repoStore.totalSize();
InputStream zipped = repoStore.bzip2Project("proj1");
repoStore.remove("proj1");
Assert.assertTrue(beforeSize > repoStore.totalSize());
repoStore.unbzip2Project("proj1", zipped);
Assert.assertEquals(beforeSize, repoStore.totalSize());
Assert.assertTrue(
Files.contentsAreEqual(
original,
repoStore.getRootDirectory()
)
);
}
}

41
services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/swap/S3SwapStoreTest.java Normal file
View file

@ -0,0 +1,41 @@
package uk.ac.ic.wlgitbridge.bridge.swap;
import org.junit.Before;
/**
* Created by winston on 21/08/2016.
*/
public class S3SwapStoreTest {
private static final String accessKey = null;
private static final String secret = null;
private static final String bucketName = "com.overleaf.testbucket";
private S3SwapStore s3;
@Before
public void setup() {
if (accessKey == null || secret == null) {
s3 = null;
return;
}
s3 = new S3SwapStore(accessKey, secret, bucketName);
}
// @Ignore
// @Test
// public void testUploadDownloadDelete() throws Exception {
// assumeNotNull(s3);
// String projName = "abc123";
// byte[] contents = "hello".getBytes();
// s3.upload(
// projName,
// new ByteArrayInputStream(contents),
// contents.length
// );
// InputStream down = s3.openDownloadStream(projName);
// s3.remove(projName);
// assertArrayEquals(contents, IOUtils.toByteArray(down));
// }
}

57
services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapJobImplTest.java Normal file
View file

@ -0,0 +1,57 @@
package uk.ac.ic.wlgitbridge.bridge.swap;
import org.junit.Before;
import org.junit.Test;
import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock;
import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore;
import java.time.Duration;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.mock;
/**
* Created by winston on 20/08/2016.
*/
public class SwapJobImplTest {
private SwapJobImpl swapJob;
private ProjectLock lock;
private RepoStore repoStore;
private DBStore dbStore;
private SwapStore swapStore;
@Before
public void setup() {
lock = mock(ProjectLock.class);
repoStore = mock(RepoStore.class);
dbStore = mock(DBStore.class);
swapStore = mock(SwapStore.class);
swapJob = new SwapJobImpl(
lock,
repoStore,
dbStore,
swapStore
);
}
@Test
public void startingTimerAlwaysCausesASwap() {
assertEquals(0, swapJob.swaps.get());
swapJob.start(Duration.ofHours(1));
while (swapJob.swaps.get() <= 0);
assertTrue(swapJob.swaps.get() > 0);
}
@Test
public void swapsHappenEveryInterval() {
assertEquals(0, swapJob.swaps.get());
swapJob.start(Duration.ofMillis(1));
while (swapJob.swaps.get() <= 1);
assertTrue(swapJob.swaps.get() > 1);
}
}

18
services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/util/ProjectTest.java Normal file
View file

@ -0,0 +1,18 @@
package uk.ac.ic.wlgitbridge.util;
import org.junit.Assert;
import org.junit.Test;
/**
* Created by winston on 23/08/2016.
*/
public class ProjectTest {
@Test
public void testValidProjectNames() {
Assert.assertFalse(Project.isValidProjectName(null));
Assert.assertFalse(Project.isValidProjectName(""));
Assert.assertFalse(Project.isValidProjectName(".wlgb"));
}
}

45
services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/util/TarTest.java Normal file
View file

@ -0,0 +1,45 @@
package uk.ac.ic.wlgitbridge.util;
import org.apache.commons.io.FileUtils;
import org.junit.Before;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Path;
import java.nio.file.Paths;
import static org.junit.Assert.assertTrue;
/**
* Created by winston on 23/08/2016.
*/
public class TarTest {
private File testDir;
@Before
public void setup() throws IOException {
TemporaryFolder tmpFolder = new TemporaryFolder();
tmpFolder.create();
testDir = tmpFolder.newFolder("testdir");
Path resdir = Paths.get(
"src/test/resources/uk/ac/ic/wlgitbridge/util/TarTest/testdir"
);
FileUtils.copyDirectory(resdir.toFile(), testDir);
}
@Test
public void tarAndUntarProducesTheSameResult() throws IOException {
InputStream tar = Tar.tar(testDir);
TemporaryFolder tmpF = new TemporaryFolder();
tmpF.create();
File parentDir = tmpF.newFolder();
Tar.untar(tar, parentDir);
File untarred = new File(parentDir, "testdir");
assertTrue(Files.contentsAreEqual(testDir, untarred));
}
}

19
services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/util/TimerTest.java Normal file
View file

@ -0,0 +1,19 @@
package uk.ac.ic.wlgitbridge.util;
import org.junit.Test;
import static org.junit.Assert.assertEquals;
/**
* Created by winston on 23/08/2016.
*/
public class TimerTest {
@Test
public void testMakeTimerTask() {
int[] iPtr = new int[] { 3 };
Timer.makeTimerTask(() -> iPtr[0] = 5).run();
assertEquals(5, iPtr[0]);
}
}

1
services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSRepoStoreTest/rootdir/proj1 Submodule

@ -0,0 +1 @@
Subproject commit e5fc0d2678ec7b9bacf0bf514bac035fa371cb6e

1
services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/bridge/repo/FSRepoStoreTest/rootdir/proj2 Submodule

@ -0,0 +1 @@
Subproject commit 6c12c073e5702530a9d06b83840d62f8a6621764

1
services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/util/TarTest/testdir/file1 Normal file
View file

@ -0,0 +1 @@
file1

1
services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/util/TarTest/testdir/file2 Normal file
View file

@ -0,0 +1 @@
file2

1
services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/util/TarTest/testdir/nest1/file1 Normal file
View file

@ -0,0 +1 @@
nest1/file1

1
services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/util/TarTest/testdir/nest1/file2 Normal file
View file

@ -0,0 +1 @@
nest1file2

1
services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/util/TarTest/testdir/nest1/file3 Normal file
View file

@ -0,0 +1 @@
nest1/file3

1
services/git-bridge/src/test/resources/uk/ac/ic/wlgitbridge/util/TarTest/testdir/nest1/nest2/file1 Normal file
View file

@ -0,0 +1 @@
nest1/nest2/file1

90
services/git-bridge/writelatex-git-bridge.iml
View file

@ -37,10 +37,6 @@
<orderEntry type="library" name="Maven: org.eclipse.jgit:org.eclipse.jgit:4.4.1.201607150455-r" level="project" />
<orderEntry type="library" name="Maven: com.jcraft:jsch:0.1.53" level="project" />
<orderEntry type="library" name="Maven: com.googlecode.javaewah:JavaEWAH:0.7.9" level="project" />
<orderEntry type="library" name="Maven: org.apache.httpcomponents:httpclient:4.3.6" level="project" />
<orderEntry type="library" name="Maven: org.apache.httpcomponents:httpcore:4.3.3" level="project" />
<orderEntry type="library" name="Maven: commons-logging:commons-logging:1.1.3" level="project" />
<orderEntry type="library" name="Maven: commons-codec:commons-codec:1.6" level="project" />
<orderEntry type="library" name="Maven: org.eclipse.jgit:org.eclipse.jgit.http.server:4.4.1.201607150455-r" level="project" />
<orderEntry type="library" name="Maven: org.xerial:sqlite-jdbc:3.8.11.2" level="project" />
<orderEntry type="library" name="Maven: joda-time:joda-time:2.9.4" level="project" />
@ -80,9 +76,9 @@
<orderEntry type="library" scope="TEST" name="Maven: com.jcraft:jzlib:1.1.3" level="project" />
<orderEntry type="library" scope="TEST" name="Maven: org.bouncycastle:bcmail-jdk15on:1.52" level="project" />
<orderEntry type="library" scope="TEST" name="Maven: org.bouncycastle:bcpkix-jdk15on:1.52" level="project" />
<orderEntry type="library" scope="TEST" name="Maven: com.fasterxml.jackson.core:jackson-core:2.5.3" level="project" />
<orderEntry type="library" scope="TEST" name="Maven: com.fasterxml.jackson.core:jackson-annotations:2.5.3" level="project" />
<orderEntry type="library" scope="TEST" name="Maven: com.fasterxml.jackson.core:jackson-databind:2.5.3" level="project" />
<orderEntry type="library" name="Maven: com.fasterxml.jackson.core:jackson-core:2.5.3" level="project" />
<orderEntry type="library" name="Maven: com.fasterxml.jackson.core:jackson-annotations:2.5.3" level="project" />
<orderEntry type="library" name="Maven: com.fasterxml.jackson.core:jackson-databind:2.5.3" level="project" />
<orderEntry type="library" scope="TEST" name="Maven: org.skyscreamer:jsonassert:1.2.3" level="project" />
<orderEntry type="library" scope="TEST" name="Maven: org.json:json:20090211" level="project" />
<orderEntry type="library" scope="TEST" name="Maven: com.github.fge:json-schema-validator:2.2.6" level="project" />
@ -97,7 +93,6 @@
<orderEntry type="library" scope="TEST" name="Maven: javax.activation:activation:1.1" level="project" />
<orderEntry type="library" scope="TEST" name="Maven: net.sf.jopt-simple:jopt-simple:4.6" level="project" />
<orderEntry type="library" scope="TEST" name="Maven: xerces:xercesImpl:2.4.0" level="project" />
<orderEntry type="library" scope="TEST" name="Maven: commons-io:commons-io:1.3.2" level="project" />
<orderEntry type="library" scope="TEST" name="Maven: io.netty:netty-buffer:4.0.34.Final" level="project" />
<orderEntry type="library" scope="TEST" name="Maven: io.netty:netty-codec:4.0.34.Final" level="project" />
<orderEntry type="library" scope="TEST" name="Maven: io.netty:netty-codec-http:4.0.34.Final" level="project" />
@ -109,5 +104,84 @@
<orderEntry type="library" scope="TEST" name="Maven: org.mock-server:mockserver-logging:3.10.4" level="project" />
<orderEntry type="library" name="Maven: org.mockito:mockito-core:1.10.19" level="project" />
<orderEntry type="library" scope="RUNTIME" name="Maven: org.objenesis:objenesis:2.1" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk:1.11.28" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-support:1.11.28" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:jmespath-java:1.0" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-simpledb:1.11.28" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-servicecatalog:1.11.28" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-simpleworkflow:1.11.28" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-storagegateway:1.11.28" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-route53:1.11.28" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-s3:1.11.28" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-importexport:1.11.28" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-sts:1.11.28" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-sqs:1.11.28" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-rds:1.11.28" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-redshift:1.11.28" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-elasticbeanstalk:1.11.28" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-glacier:1.11.28" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-iam:1.11.28" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-datapipeline:1.11.28" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-elasticloadbalancing:1.11.28" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-elasticloadbalancingv2:1.11.28" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-emr:1.11.28" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-elasticache:1.11.28" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-elastictranscoder:1.11.28" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-ec2:1.11.28" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-dynamodb:1.11.28" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-sns:1.11.28" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-cloudtrail:1.11.28" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-cloudwatch:1.11.28" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-logs:1.11.28" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-events:1.11.28" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-cognitoidentity:1.11.28" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-cognitosync:1.11.28" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-directconnect:1.11.28" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-cloudformation:1.11.28" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-cloudfront:1.11.28" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-kinesis:1.11.28" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-opsworks:1.11.28" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-ses:1.11.28" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-autoscaling:1.11.28" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-cloudsearch:1.11.28" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-cloudwatchmetrics:1.11.28" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-codedeploy:1.11.28" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-codepipeline:1.11.28" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-kms:1.11.28" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-config:1.11.28" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-lambda:1.11.28" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-ecs:1.11.28" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-ecr:1.11.28" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-cloudhsm:1.11.28" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-ssm:1.11.28" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-workspaces:1.11.28" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-machinelearning:1.11.28" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-directory:1.11.28" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-efs:1.11.28" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-codecommit:1.11.28" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-devicefarm:1.11.28" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-elasticsearch:1.11.28" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-waf:1.11.28" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-marketplacecommerceanalytics:1.11.28" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-inspector:1.11.28" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-iot:1.11.28" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-api-gateway:1.11.28" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-acm:1.11.28" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-gamelift:1.11.28" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-dms:1.11.28" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-marketplacemeteringservice:1.11.28" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-cognitoidp:1.11.28" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-discovery:1.11.28" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-applicationautoscaling:1.11.28" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-snowball:1.11.28" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-core:1.11.28" level="project" />
<orderEntry type="library" name="Maven: com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.6.6" level="project" />
<orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-swf-libraries:1.11.22" level="project" />
<orderEntry type="library" name="Maven: org.apache.httpcomponents:httpclient:4.5.2" level="project" />
<orderEntry type="library" name="Maven: org.apache.httpcomponents:httpcore:4.4.4" level="project" />
<orderEntry type="library" name="Maven: commons-logging:commons-logging:1.2" level="project" />
<orderEntry type="library" name="Maven: commons-codec:commons-codec:1.9" level="project" />
<orderEntry type="library" name="Maven: commons-io:commons-io:2.5" level="project" />
<orderEntry type="library" name="Maven: org.apache.commons:commons-compress:1.12" level="project" />
</component>
</module>