diff --git a/dice-where-downloader-lib/pom.xml b/dice-where-downloader-lib/pom.xml
index 5a2966a..a067826 100644
--- a/dice-where-downloader-lib/pom.xml
+++ b/dice-where-downloader-lib/pom.xml
@@ -137,7 +137,6 @@
${wiremock.version}
test
-
diff --git a/dice-where-downloader-lib/src/main/java/technology/dice/dicewhere/downloader/destination/local/LocalFileAcceptor.java b/dice-where-downloader-lib/src/main/java/technology/dice/dicewhere/downloader/destination/local/LocalFileAcceptor.java
index 71adab3..9ba7e1c 100644
--- a/dice-where-downloader-lib/src/main/java/technology/dice/dicewhere/downloader/destination/local/LocalFileAcceptor.java
+++ b/dice-where-downloader-lib/src/main/java/technology/dice/dicewhere/downloader/destination/local/LocalFileAcceptor.java
@@ -21,7 +21,6 @@
public class LocalFileAcceptor implements FileAcceptor {
private static final Logger LOG = LoggerFactory.getLogger(LocalFileAcceptor.class);
- public static final int BUFFER = 8192;
private final Path destination;
@@ -72,9 +71,6 @@ public Optional existingFileMd5() {
try (InputStream is = Files.newInputStream(this.destination);
BufferedInputStream bis = new BufferedInputStream(is);
StreamWithMD5Decorator md5Is = StreamWithMD5Decorator.of(bis)) {
- byte[] buffer = new byte[BUFFER];
- while ((md5Is.read(buffer)) != -1) {
- }
return Optional.of(md5Is.md5());
} catch (IOException | NoSuchAlgorithmException e) {
throw new RuntimeException(
diff --git a/dice-where-downloader-lib/src/main/java/technology/dice/dicewhere/downloader/stream/StreamWithMD5Decorator.java b/dice-where-downloader-lib/src/main/java/technology/dice/dicewhere/downloader/stream/StreamWithMD5Decorator.java
index 99f03a2..9b31d2d 100644
--- a/dice-where-downloader-lib/src/main/java/technology/dice/dicewhere/downloader/stream/StreamWithMD5Decorator.java
+++ b/dice-where-downloader-lib/src/main/java/technology/dice/dicewhere/downloader/stream/StreamWithMD5Decorator.java
@@ -1,5 +1,7 @@
package technology.dice.dicewhere.downloader.stream;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.security.DigestInputStream;
@@ -10,32 +12,62 @@
public class StreamWithMD5Decorator extends InputStream {
+ private static final int BUFFER_SIZE = 8192;
+ private static final HexBinaryAdapter HEX_BINARY_ADAPTER = new HexBinaryAdapter();
+
private final MessageDigest md5;
- DigestInputStream inputStream;
+ private final DigestInputStream originalStream;
+ private final MD5Checksum md5Checksum;
+
+ private InputStream duplicatedStream;
- private StreamWithMD5Decorator(DigestInputStream inputStream, MessageDigest md5) {
- this.inputStream = inputStream;
+ private StreamWithMD5Decorator(DigestInputStream originalStream, MessageDigest md5)
+ throws IOException {
+ this.originalStream = originalStream;
this.md5 = md5;
+ consumeStream();
+ md5Checksum = generateMd5();
}
- public static StreamWithMD5Decorator of(InputStream inputStream) throws NoSuchAlgorithmException {
+ public static StreamWithMD5Decorator of(InputStream inputStream)
+ throws NoSuchAlgorithmException, IOException {
MessageDigest md5 = MessageDigest.getInstance("MD5");
DigestInputStream dis = new DigestInputStream(inputStream, md5);
return new StreamWithMD5Decorator(dis, md5);
}
- public MD5Checksum md5() {
- String hex = (new HexBinaryAdapter()).marshal(this.md5.digest());
+ private void consumeStream() throws IOException {
+ ByteArrayOutputStream tempBufferStream = new ByteArrayOutputStream();
+ byte[] data = new byte[BUFFER_SIZE];
+ int bytesRead;
+ while ((bytesRead = originalStream.read(data)) != -1) {
+ tempBufferStream.write(data, 0, bytesRead);
+ }
+ duplicatedStream = new ByteArrayInputStream(tempBufferStream.toByteArray());
+ }
+
+ private MD5Checksum generateMd5() {
+ String hex = HEX_BINARY_ADAPTER.marshal(md5.digest());
return MD5Checksum.of(hex);
}
+ public MD5Checksum md5() {
+ return md5Checksum;
+ }
+
@Override
public int read() throws IOException {
- return this.inputStream.read();
+ return duplicatedStream.read();
+ }
+
+ @Override
+ public int read(byte[] b, int off, int len) throws IOException {
+ return duplicatedStream.read(b, off, len);
}
@Override
public void close() throws IOException {
- this.inputStream.close();
+ duplicatedStream.close();
+ originalStream.close();
}
}
diff --git a/dice-where-downloader-lib/src/test/java/technology/dice/dicewhere/downloader/destination/local/LocalFileAcceptorTest.java b/dice-where-downloader-lib/src/test/java/technology/dice/dicewhere/downloader/destination/local/LocalFileAcceptorTest.java
index 121d2f7..c84bf7c 100644
--- a/dice-where-downloader-lib/src/test/java/technology/dice/dicewhere/downloader/destination/local/LocalFileAcceptorTest.java
+++ b/dice-where-downloader-lib/src/test/java/technology/dice/dicewhere/downloader/destination/local/LocalFileAcceptorTest.java
@@ -33,8 +33,7 @@
public class LocalFileAcceptorTest extends TestCase {
private static final int TEST_FILE_SIZE = 1024 * 1024;
- @ClassRule
- static WireMockRule wireMockRule = new WireMockRule(wireMockConfig().dynamicPort());
+ @ClassRule static WireMockRule wireMockRule = new WireMockRule(wireMockConfig().dynamicPort());
@BeforeClass
public static void beforeClass() {
@@ -45,16 +44,22 @@ public static void beforeClass() {
public void corruptedFileEmptyPreexistingSet() throws IOException, NoSuchAlgorithmException {
Pair tempFile = generateTempFile();
Path destinationDir = Files.createTempDirectory("dice-where");
- IpInfoSiteSource ipInfoSiteSource = new IpInfoSiteSource(
- new URL("http://localhost:" + wireMockRule.port() + "/data/file.mdb"));
- wireMockRule.stubFor(WireMock.head(UrlPattern.ANY).willReturn(
- aResponse().withStatus(HttpStatus.SC_OK)
- .withHeader("Etag", "aaa")
- .withHeader("Content-Length", Long.toString(TEST_FILE_SIZE))
- .withHeader("Last-Modified", "Thu, 01 Dec 1994 16:00:00 GMT")));
- wireMockRule.stubFor(WireMock.get(UrlPattern.ANY)
- .willReturn(aResponse().withBody(
- IOUtils.toByteArray(new FileInputStream(tempFile.getLeft().toFile())))));
+ IpInfoSiteSource ipInfoSiteSource =
+ new IpInfoSiteSource(new URL("http://localhost:" + wireMockRule.port() + "/data/file.mdb"));
+ wireMockRule.stubFor(
+ WireMock.head(UrlPattern.ANY)
+ .willReturn(
+ aResponse()
+ .withStatus(HttpStatus.SC_OK)
+ .withHeader("Etag", "aaa")
+ .withHeader("Content-Length", Long.toString(TEST_FILE_SIZE))
+ .withHeader("Last-Modified", "Thu, 01 Dec 1994 16:00:00 GMT")));
+ wireMockRule.stubFor(
+ WireMock.get(UrlPattern.ANY)
+ .willReturn(
+ aResponse()
+ .withBody(
+ IOUtils.toByteArray(new FileInputStream(tempFile.getLeft().toFile())))));
FileInfo fileInfo = ipInfoSiteSource.fileInfo();
ipInfoSiteSource.produce(new LocalFileAcceptor(destinationDir.resolve("file.mdb")), false);
@@ -68,72 +73,100 @@ public void corruptedFilePreexistingSet() throws IOException, NoSuchAlgorithmExc
Pair existingFile = generateTempFile();
Path destinationDir = Files.createTempDirectory("dice-where");
Files.copy(existingFile.getLeft(), destinationDir.resolve("existingFile.mdb"));
- IpInfoSiteSource ipInfoSiteSource = new IpInfoSiteSource(
- new URL("http://localhost:" + wireMockRule.port() + "/data/file.mdb"));
- wireMockRule.stubFor(WireMock.head(UrlPattern.ANY).willReturn(
- aResponse().withStatus(HttpStatus.SC_OK)
- .withHeader("Etag", "aaa")
- .withHeader("Content-Length", Long.toString(TEST_FILE_SIZE))
- .withHeader("Last-Modified", "Thu, 01 Dec 1994 16:00:00 GMT")));
- wireMockRule.stubFor(WireMock.get(UrlPattern.ANY)
- .willReturn(aResponse().withBody(
- IOUtils.toByteArray(new FileInputStream(tempFile.getLeft().toFile())))));
+ IpInfoSiteSource ipInfoSiteSource =
+ new IpInfoSiteSource(new URL("http://localhost:" + wireMockRule.port() + "/data/file.mdb"));
+ wireMockRule.stubFor(
+ WireMock.head(UrlPattern.ANY)
+ .willReturn(
+ aResponse()
+ .withStatus(HttpStatus.SC_OK)
+ .withHeader("Etag", "aaa")
+ .withHeader("Content-Length", Long.toString(TEST_FILE_SIZE))
+ .withHeader("Last-Modified", "Thu, 01 Dec 1994 16:00:00 GMT")));
+ wireMockRule.stubFor(
+ WireMock.get(UrlPattern.ANY)
+ .willReturn(
+ aResponse()
+ .withBody(
+ IOUtils.toByteArray(new FileInputStream(tempFile.getLeft().toFile())))));
FileInfo fileInfo = ipInfoSiteSource.fileInfo();
ipInfoSiteSource.produce(new LocalFileAcceptor(destinationDir.resolve("file.mdb")), false);
assertNotEquals(tempFile.getRight().toLowerCase(), fileInfo.getMd5Checksum().stringFormat());
assertEquals(Files.list(destinationDir).count(), 1);
assertFalse(Files.exists(destinationDir.resolve("file.mdb")));
- assertTrue(Arrays.equals(Files.readAllBytes(existingFile.getLeft()),
- Files.readAllBytes(Files.list(destinationDir).findFirst().get())));
+ assertTrue(
+ Arrays.equals(
+ Files.readAllBytes(existingFile.getLeft()),
+ Files.readAllBytes(Files.list(destinationDir).findFirst().get())));
}
@Test
public void goodFileEmptyPreexistingSet() throws IOException, NoSuchAlgorithmException {
Pair tempFile = generateTempFile();
Path destinationDir = Files.createTempDirectory("dice-where");
- IpInfoSiteSource ipInfoSiteSource = new IpInfoSiteSource(
- new URL("http://localhost:" + wireMockRule.port() + "/data/file.mdb"));
- wireMockRule.stubFor(WireMock.head(UrlPattern.ANY).willReturn(
- aResponse().withStatus(HttpStatus.SC_OK)
- .withHeader("Etag", tempFile.getRight())
- .withHeader("Content-Length", Long.toString(TEST_FILE_SIZE))
- .withHeader("Last-Modified", "Thu, 01 Dec 1994 16:00:00 GMT")));
- wireMockRule.stubFor(WireMock.get(UrlPattern.ANY)
- .willReturn(aResponse().withBody(
- IOUtils.toByteArray(new FileInputStream(tempFile.getLeft().toFile())))));
+ IpInfoSiteSource ipInfoSiteSource =
+ new IpInfoSiteSource(new URL("http://localhost:" + wireMockRule.port() + "/data/file.mdb"));
+ wireMockRule.stubFor(
+ WireMock.head(UrlPattern.ANY)
+ .willReturn(
+ aResponse()
+ .withStatus(HttpStatus.SC_OK)
+ .withHeader("Etag", tempFile.getRight())
+ .withHeader("Content-Length", Long.toString(TEST_FILE_SIZE))
+ .withHeader("Last-Modified", "Thu, 01 Dec 1994 16:00:00 GMT")));
+ wireMockRule.stubFor(
+ WireMock.get(UrlPattern.ANY)
+ .willReturn(
+ aResponse()
+ .withBody(
+ IOUtils.toByteArray(new FileInputStream(tempFile.getLeft().toFile())))));
FileInfo fileInfo = ipInfoSiteSource.fileInfo();
ipInfoSiteSource.produce(new LocalFileAcceptor(destinationDir.resolve("file.mdb")), false);
assertEquals(tempFile.getRight().toLowerCase(), fileInfo.getMd5Checksum().stringFormat());
assertEquals(1, Files.list(destinationDir).count());
- assertTrue(Arrays.equals(Files.readAllBytes(tempFile.getLeft()),
- Files.readAllBytes(destinationDir.resolve("file.mdb"))));
+ assertTrue(
+ Arrays.equals(
+ Files.readAllBytes(tempFile.getLeft()),
+ Files.readAllBytes(destinationDir.resolve("file.mdb"))));
}
@Test
public void goodFilePreexistingSet() throws IOException, NoSuchAlgorithmException {
Pair tempFile = generateTempFile();
Pair existingFile = generateTempFile();
+
Path destinationDir = Files.createTempDirectory("dice-where");
Files.copy(existingFile.getLeft(), destinationDir.resolve("existingFile.mdb"));
- IpInfoSiteSource ipInfoSiteSource = new IpInfoSiteSource(
- new URL("http://localhost:" + wireMockRule.port() + "/data/file.mdb"));
- wireMockRule.stubFor(WireMock.head(UrlPattern.ANY).willReturn(
- aResponse().withStatus(HttpStatus.SC_OK)
- .withHeader("Etag", tempFile.getRight())
- .withHeader("Content-Length", Long.toString(TEST_FILE_SIZE))
- .withHeader("Last-Modified", "Thu, 01 Dec 1994 16:00:00 GMT")));
- wireMockRule.stubFor(WireMock.get(UrlPattern.ANY)
- .willReturn(aResponse().withBody(
- IOUtils.toByteArray(new FileInputStream(tempFile.getLeft().toFile())))));
+ wireMockRule.stubFor(
+ WireMock.head(UrlPattern.ANY)
+ .willReturn(
+ aResponse()
+ .withStatus(HttpStatus.SC_OK)
+ .withHeader("Etag", tempFile.getRight())
+ .withHeader("Content-Length", Long.toString(TEST_FILE_SIZE))
+ .withHeader("Last-Modified", "Thu, 01 Dec 1994 16:00:00 GMT")));
+
+ wireMockRule.stubFor(
+ WireMock.get(UrlPattern.ANY)
+ .willReturn(
+ aResponse()
+ .withBody(
+ IOUtils.toByteArray(new FileInputStream(tempFile.getLeft().toFile())))));
+
+ IpInfoSiteSource ipInfoSiteSource =
+ new IpInfoSiteSource(new URL("http://localhost:" + wireMockRule.port() + "/data/file.mdb"));
FileInfo fileInfo = ipInfoSiteSource.fileInfo();
+
ipInfoSiteSource.produce(new LocalFileAcceptor(destinationDir.resolve("file.mdb")), false);
assertEquals(tempFile.getRight().toLowerCase(), fileInfo.getMd5Checksum().stringFormat());
assertEquals(2, Files.list(destinationDir).count());
- assertTrue(Arrays.equals(Files.readAllBytes(tempFile.getLeft()),
- Files.readAllBytes(destinationDir.resolve("file.mdb"))));
+ assertTrue(
+ Arrays.equals(
+ Files.readAllBytes(tempFile.getLeft()),
+ Files.readAllBytes(destinationDir.resolve("file.mdb"))));
}
private Pair generateTempFile() throws IOException, NoSuchAlgorithmException {
@@ -145,4 +178,4 @@ private Pair generateTempFile() throws IOException, NoSuchAlgorith
String hex = (new HexBinaryAdapter()).marshal(md.digest(contents));
return Pair.of(tempFile, hex);
}
-}
\ No newline at end of file
+}
diff --git a/dice-where-downloader-lib/src/test/java/technology/dice/dicewhere/downloader/stream/StreamWithMD5DecoratorTest.java b/dice-where-downloader-lib/src/test/java/technology/dice/dicewhere/downloader/stream/StreamWithMD5DecoratorTest.java
new file mode 100644
index 0000000..7b7841d
--- /dev/null
+++ b/dice-where-downloader-lib/src/test/java/technology/dice/dicewhere/downloader/stream/StreamWithMD5DecoratorTest.java
@@ -0,0 +1,75 @@
+package technology.dice.dicewhere.downloader.stream;
+
+import static com.github.tomakehurst.wiremock.client.WireMock.aResponse;
+import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig;
+
+import com.github.tomakehurst.wiremock.client.WireMock;
+import com.github.tomakehurst.wiremock.junit.WireMockRule;
+import com.github.tomakehurst.wiremock.matching.UrlPattern;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.nio.charset.Charset;
+import java.nio.file.Path;
+import java.security.NoSuchAlgorithmException;
+import junit.framework.TestCase;
+import org.apache.commons.io.IOUtils;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.internal.runners.JUnit4ClassRunner;
+import org.junit.runner.RunWith;
+
+@RunWith(JUnit4ClassRunner.class)
+public class StreamWithMD5DecoratorTest extends TestCase {
+
+ private static final String PATH = "/maxmind/maxmind-city-1.zip";
+
+ @ClassRule static WireMockRule wireMockRule = new WireMockRule(wireMockConfig().dynamicPort());
+
+ @BeforeClass
+ public static void beforeClass() {
+ wireMockRule.start();
+ }
+
+ @Test
+ public void shouldSuccessfullyReadAndCalculateDigestOfStream()
+ throws IOException, NoSuchAlgorithmException, URISyntaxException {
+ Path path = Path.of(getClass().getResource(PATH).toURI());
+ StreamWithMD5Decorator is = StreamWithMD5Decorator.of(new FileInputStream(path.toFile()));
+
+ String first = is.md5().stringFormat();
+ // Read from the stream
+ IOUtils.toString(is, Charset.defaultCharset());
+
+ // Assert the Stream Hash before and after
+ assertEquals(first, is.md5().stringFormat());
+ assertEquals(first, "9c7dd68c8352f1c59a33efe0dca04f06");
+ }
+
+ @Test
+ public void shouldSuccessfullyReadAndCalculateDigestOfStreamFromHttp()
+ throws IOException, NoSuchAlgorithmException, URISyntaxException {
+ Path path = Path.of(getClass().getResource(PATH).toURI());
+
+ wireMockRule.stubFor(
+ WireMock.get(UrlPattern.ANY)
+ .willReturn(
+ aResponse().withBody(IOUtils.toByteArray(new FileInputStream(path.toFile())))));
+
+ URL url = new URL("http://localhost:" + wireMockRule.port() + "/data/file.mdb");
+ HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+
+ StreamWithMD5Decorator is = StreamWithMD5Decorator.of(connection.getInputStream());
+
+ String first = is.md5().stringFormat();
+ // Read from the stream
+ IOUtils.toString(is, Charset.defaultCharset());
+
+ // Assert the Stream Hash before and after
+ assertEquals(first, is.md5().stringFormat());
+ assertEquals(first, "9c7dd68c8352f1c59a33efe0dca04f06");
+ }
+}
diff --git a/dice-where-downloader-lib/src/test/resources/maxmind/maxmind-city-1.zip b/dice-where-downloader-lib/src/test/resources/maxmind/maxmind-city-1.zip
new file mode 100644
index 0000000..a2aabb4
Binary files /dev/null and b/dice-where-downloader-lib/src/test/resources/maxmind/maxmind-city-1.zip differ