Merge pull request #37 from overleaf/msw-final

#24 Part 6/6: additional testing

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/noop/NoopDbStore.java

@@ -0,0 +1,66 @@
+package uk.ac.ic.wlgitbridge.bridge.db.noop;
+
+import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
+import uk.ac.ic.wlgitbridge.bridge.db.ProjectState;
+
+import java.sql.Timestamp;
+import java.util.List;
+
+public class NoopDbStore implements DBStore {
+
+    @Override
+    public int getNumProjects() {
+        return 0;
+    }
+
+    @Override
+    public List<String> getProjectNames() {
+        return null;
+    }
+
+    @Override
+    public void setLatestVersionForProject(String project, int versionID) {
+
+    }
+
+    @Override
+    public int getLatestVersionForProject(String project) {
+        return 0;
+    }
+
+    @Override
+    public void addURLIndexForProject(String projectName, String url, String path) {
+
+    }
+
+    @Override
+    public void deleteFilesForProject(String project, String... files) {
+
+    }
+
+    @Override
+    public String getPathForURLInProject(String projectName, String url) {
+        return null;
+    }
+
+    @Override
+    public String getOldestUnswappedProject() {
+        return null;
+    }
+
+    @Override
+    public int getNumUnswappedProjects() {
+        return 0;
+    }
+
+    @Override
+    public ProjectState getProjectState(String projectName) {
+        return null;
+    }
+
+    @Override
+    public void setLastAccessedTime(String projectName, Timestamp time) {
+
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/resource/UrlResourceCache.java

@@ -1,15 +1,15 @@
 package uk.ac.ic.wlgitbridge.bridge.resource;
 
-import com.ning.http.client.*;
+import com.ning.http.client.AsyncHttpClient;
 import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
 import uk.ac.ic.wlgitbridge.data.filestore.RawFile;
 import uk.ac.ic.wlgitbridge.data.filestore.RepositoryFile;
 import uk.ac.ic.wlgitbridge.git.exception.SizeLimitExceededException;
-import uk.ac.ic.wlgitbridge.snapshot.base.Request;
+import uk.ac.ic.wlgitbridge.io.http.ning.NingHttpClient;
+import uk.ac.ic.wlgitbridge.io.http.ning.NingHttpClientFacade;
 import uk.ac.ic.wlgitbridge.snapshot.exception.FailedConnectionException;
 import uk.ac.ic.wlgitbridge.util.Log;
 
-import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.util.List;
 import java.util.Map;
@@ -23,8 +23,15 @@ public class UrlResourceCache implements ResourceCache {
 
     private final DBStore dbStore;
 
-    public UrlResourceCache(DBStore dbStore) {
+    private final NingHttpClientFacade http;
+
+    UrlResourceCache(DBStore dbStore, NingHttpClientFacade http) {
         this.dbStore = dbStore;
+        this.http = http;
+    }
+
+    public UrlResourceCache(DBStore dbStore) {
+        this(dbStore, new NingHttpClient(new AsyncHttpClient()));
     }
 
     @Override
@@ -74,71 +81,24 @@ public class UrlResourceCache implements ResourceCache {
         byte[] contents;
         Log.info("GET -> " + url);
         try {
-            contents = Request.httpClient.prepareGet(url).execute(
+            contents = http.get(url, hs -> {
-                    new AsyncCompletionHandler<byte[]>() {
+                List<String> contentLengths
-
+                        = hs.getHeaders().get("Content-Length");
-                ByteArrayOutputStream bytes = new ByteArrayOutputStream();
+                if (!maxFileSize.isPresent()) {
-
+                    return true;
-                @Override
-                public STATE onHeadersReceived(
-                        HttpResponseHeaders headers
-                ) throws SizeLimitExceededException {
-                    List<String> contentLengths
-                            = headers.getHeaders().get("Content-Length");
-                    if (!maxFileSize.isPresent()) {
-                        return STATE.CONTINUE;
-                    }
-                    if (contentLengths.isEmpty()) {
-                        return STATE.CONTINUE;
-                    }
-                    long contentLength = Long.parseLong(contentLengths.get(0));
-                    long maxFileSize_ = maxFileSize.get();
-                    if (contentLength <= maxFileSize_) {
-                        return STATE.CONTINUE;
-                    }
-                    throw new SizeLimitExceededException(
-                            Optional.of(path), contentLength, maxFileSize_
-                    );
                 }
-
+                if (contentLengths.isEmpty()) {
-                @Override
+                    return true;
-                public STATE onBodyPartReceived(
-                        HttpResponseBodyPart bodyPart
-                ) throws Exception {
-                    bytes.write(bodyPart.getBodyPartBytes());
-                    return STATE.CONTINUE;
                 }
-
+                long contentLength = Long.parseLong(contentLengths.get(0));
-                @Override
+                long maxFileSize_ = maxFileSize.get();
-                public byte[] onCompleted(
+                if (contentLength <= maxFileSize_) {
-                        Response response
+                    return true;
-                ) throws Exception {
-                    byte[] data = bytes.toByteArray();
-                    bytes.close();
-                    Log.info(
-                            response.getStatusCode()
-                                    + " "
-                                    + response.getStatusText()
-                                    + " ("
-                                    + data.length
-                                    + "B) -> "
-                                    + url
-                    );
-                    return data;
                 }
-
+                throw new SizeLimitExceededException(
-            }).get();
+                        Optional.of(path), contentLength, maxFileSize_
-        } catch (InterruptedException e) {
+                );
-            Log.warn(
+            });
-                    "Interrupted when fetching project: "  +
-                            projectName  +
-                            ", url: " +
-                            url +
-                            ", path: " +
-                            path,
-                    e
-            );
-            throw new FailedConnectionException();
         } catch (ExecutionException e) {
             Throwable cause = e.getCause();
             if (cause instanceof SizeLimitExceededException) {

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/io/http/ning/NingHttpClient.java

@@ -0,0 +1,75 @@
+package uk.ac.ic.wlgitbridge.io.http.ning;
+
+import com.ning.http.client.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import uk.ac.ic.wlgitbridge.util.FunctionT;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.concurrent.ExecutionException;
+
+public class NingHttpClient implements NingHttpClientFacade {
+
+    private static final Logger log
+            = LoggerFactory.getLogger(NingHttpClient.class);
+
+    private final AsyncHttpClient http;
+
+    public NingHttpClient(AsyncHttpClient http) {
+        this.http = http;
+    }
+
+    @Override
+    public <E extends Exception> byte[] get(
+            String url,
+            FunctionT<HttpResponseHeaders, Boolean, E> handler
+    ) throws ExecutionException {
+        try {
+            return http
+                    .prepareGet(url)
+                    .execute(new AsyncCompletionHandler<byte[]>() {
+
+                ByteArrayOutputStream bytes = new ByteArrayOutputStream();
+
+                @Override
+                public STATE onHeadersReceived(
+                        HttpResponseHeaders headers
+                ) throws E {
+                    return handler.apply(headers)
+                            ? STATE.CONTINUE : STATE.ABORT;
+                }
+
+                @Override
+                public STATE onBodyPartReceived(
+                        HttpResponseBodyPart content
+                ) throws IOException {
+                    bytes.write(content.getBodyPartBytes());
+                    return STATE.CONTINUE;
+                }
+
+                @Override
+                public byte[] onCompleted(
+                        Response response
+                ) throws IOException {
+                    byte[] ret = bytes.toByteArray();
+                    bytes.close();
+                    log.info(
+                            response.getStatusCode()
+                                    + " "
+                                    + response.getStatusText()
+                                    + " ("
+                                    + ret.length
+                                    + "B) -> "
+                                    + url
+                    );
+                    return ret;
+                }
+
+            }).get();
+        } catch (InterruptedException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/io/http/ning/NingHttpClientFacade.java

@@ -0,0 +1,22 @@
+package uk.ac.ic.wlgitbridge.io.http.ning;
+
+import com.ning.http.client.HttpResponseHeaders;
+import uk.ac.ic.wlgitbridge.util.FunctionT;
+
+import java.util.concurrent.ExecutionException;
+
+public interface NingHttpClientFacade {
+
+    /**
+     * Performs a GET request
+     * @param url the target URL
+     * @param handler handler for the response headers. Returning false
+     *                aborts the request.
+     * @return
+     */
+    <E extends Exception> byte[] get(
+            String url,
+            FunctionT<HttpResponseHeaders, Boolean, E> handler
+    ) throws ExecutionException;
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/io/http/ning/NingHttpHeaders.java

@@ -0,0 +1,46 @@
+package uk.ac.ic.wlgitbridge.io.http.ning;
+
+import com.ning.http.client.FluentCaseInsensitiveStringsMap;
+import com.ning.http.client.HttpResponseHeaders;
+
+import java.util.*;
+
+public class NingHttpHeaders extends HttpResponseHeaders {
+
+    private final FluentCaseInsensitiveStringsMap map;
+
+    private NingHttpHeaders(FluentCaseInsensitiveStringsMap map) {
+        this.map = map;
+    }
+
+    public static NingHttpHeadersBuilder builder() {
+        return new NingHttpHeadersBuilder();
+    }
+
+    @Override
+    public FluentCaseInsensitiveStringsMap getHeaders() {
+        return map;
+    }
+
+    public static class NingHttpHeadersBuilder {
+
+        private final Map<String, Collection<String>> map;
+
+        private NingHttpHeadersBuilder() {
+            map = new HashMap<>();
+        }
+
+        public NingHttpHeadersBuilder addHeader(String key, String... values) {
+            map.computeIfAbsent(key, __ -> new ArrayList<>())
+                    .addAll(Arrays.asList(values));
+            return this;
+        }
+
+        public NingHttpHeaders build() {
+            return new NingHttpHeaders(
+                    new FluentCaseInsensitiveStringsMap(map));
+        }
+
+    }
+
+}

services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/resource/UrlResourceCacheTest.java

@@ -0,0 +1,110 @@
+package uk.ac.ic.wlgitbridge.bridge.resource;
+
+import com.ning.http.client.HttpResponseHeaders;
+import org.junit.Test;
+import uk.ac.ic.wlgitbridge.bridge.db.noop.NoopDbStore;
+import uk.ac.ic.wlgitbridge.bridge.util.CastUtil;
+import uk.ac.ic.wlgitbridge.git.exception.SizeLimitExceededException;
+import uk.ac.ic.wlgitbridge.io.http.ning.NingHttpClientFacade;
+import uk.ac.ic.wlgitbridge.io.http.ning.NingHttpHeaders;
+import uk.ac.ic.wlgitbridge.util.FunctionT;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Optional;
+import java.util.concurrent.ExecutionException;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class UrlResourceCacheTest {
+
+    private static String PROJ = "proj";
+
+    private static String URL = "http://localhost/file.jpg";
+
+    private static String NEW_PATH = "file1.jpg";
+
+    private final NingHttpClientFacade http = mock(NingHttpClientFacade.class);
+
+    private final UrlResourceCache cache
+            = new UrlResourceCache(new NoopDbStore(), http);
+
+    private static HttpResponseHeaders withContentLength(long cl) {
+        return NingHttpHeaders
+                .builder()
+                .addHeader("Content-Length", String.valueOf(cl))
+                .build();
+    }
+
+    private void respondWithContentLength(long cl, long actual)
+            throws ExecutionException {
+        when(http.get(any(), any())).thenAnswer(invoc -> {
+            Object[] args = invoc.getArguments();
+            //noinspection unchecked
+            ((FunctionT<
+                    HttpResponseHeaders, Boolean, SizeLimitExceededException
+            >) args[1]).apply(withContentLength(cl));
+            return new byte[CastUtil.assumeInt(actual)];
+        });
+    }
+
+    private void respondWithContentLength(long cl) throws ExecutionException {
+        respondWithContentLength(cl, cl);
+    }
+
+    private void getWithMaxLength(Optional<Long> max)
+        throws IOException, SizeLimitExceededException {
+        cache.get(
+                PROJ, URL, NEW_PATH, new HashMap<>(), new HashMap<>(), max);
+    }
+
+    private void getWithMaxLength(long max)
+            throws IOException, SizeLimitExceededException {
+        getWithMaxLength(Optional.of(max));
+    }
+
+    private void getWithoutLimit()
+            throws IOException, SizeLimitExceededException {
+        getWithMaxLength(Optional.empty());
+    }
+
+    @Test
+    public void getDoesNotThrowWhenContentLengthLT() throws Exception {
+        respondWithContentLength(1);
+        getWithMaxLength(2);
+    }
+
+    @Test
+    public void getDoesNotThrowWhenContentLengthEQ() throws Exception {
+        respondWithContentLength(2);
+        getWithMaxLength(2);
+    }
+
+    @Test (expected = SizeLimitExceededException.class)
+    public void getThrowsSizeLimitExceededWhenContentLengthGT()
+            throws Exception {
+        respondWithContentLength(3);
+        getWithMaxLength(2);
+    }
+
+    @Test
+    public void getWithEmptyContentIsValid() throws Exception {
+        respondWithContentLength(0);
+        getWithMaxLength(0);
+    }
+
+    @Test
+    public void getWithoutLimitDoesNotThrow() throws Exception {
+        respondWithContentLength(Integer.MAX_VALUE, 0);
+        getWithoutLimit();
+    }
+
+    @Test (expected = SizeLimitExceededException.class)
+    public void getThrowsIfActualContentTooBig() throws Exception {
+        respondWithContentLength(0, 10);
+        getWithMaxLength(5);
+    }
+
+}