diff --git a/app/test/cc/arduino/contributions/GzippedJsonDownloaderTest.java b/app/test/cc/arduino/contributions/GzippedJsonDownloaderTest.java new file mode 100644 index 000000000..a762b6801 --- /dev/null +++ b/app/test/cc/arduino/contributions/GzippedJsonDownloaderTest.java @@ -0,0 +1,53 @@ +package cc.arduino.contributions; + +import cc.arduino.contributions.libraries.LibrariesIndex; +import cc.arduino.utils.MultiStepProgress; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.module.mrbean.MrBeanModule; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import processing.app.helpers.FileUtils; + +import java.io.File; +import java.io.FileInputStream; +import java.io.InputStream; +import java.net.URL; + +import static org.junit.Assert.assertTrue; + +public class GzippedJsonDownloaderTest { + + private File tempFolder; + private File tempFile; + private DownloadableContributionsDownloader downloader; + + @Before + public void setUp() throws Exception { + tempFolder = FileUtils.createTempFolder(); + tempFile = File.createTempFile("test", ".json"); + downloader = new DownloadableContributionsDownloader(tempFolder); + } + + @After + public void tearDown() throws Exception { + FileUtils.recursiveDelete(tempFolder); + FileUtils.recursiveDelete(tempFile); + } + + @Test + public void testJsonDownload() throws Exception { + new GZippedJsonDownloader(downloader, new URL("http://downloads.arduino.cc/libraries/library_index.json"), new URL("http://downloads.arduino.cc/libraries/library_index.json.gz")).download(tempFile, new MultiStepProgress(1), ""); + + InputStream indexIn = new FileInputStream(tempFile); + ObjectMapper mapper = new ObjectMapper(); + mapper.registerModule(new MrBeanModule()); + mapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, true); + mapper.configure(DeserializationFeature.EAGER_DESERIALIZER_FETCH, true); + mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + LibrariesIndex librariesIndex = mapper.readValue(indexIn, LibrariesIndex.class); + + assertTrue(librariesIndex != null); + } +} diff --git a/app/test/cc/arduino/contributions/JsonDownloaderTest.java b/app/test/cc/arduino/contributions/JsonDownloaderTest.java new file mode 100644 index 000000000..98cefef49 --- /dev/null +++ b/app/test/cc/arduino/contributions/JsonDownloaderTest.java @@ -0,0 +1,53 @@ +package cc.arduino.contributions; + +import cc.arduino.contributions.libraries.LibrariesIndex; +import cc.arduino.utils.MultiStepProgress; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.module.mrbean.MrBeanModule; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import processing.app.helpers.FileUtils; + +import java.io.File; +import java.io.FileInputStream; +import java.io.InputStream; +import java.net.URL; + +import static org.junit.Assert.assertTrue; + +public class JsonDownloaderTest { + + private File tempFolder; + private File tempFile; + private DownloadableContributionsDownloader downloader; + + @Before + public void setUp() throws Exception { + tempFolder = FileUtils.createTempFolder(); + tempFile = File.createTempFile("test", ".json"); + downloader = new DownloadableContributionsDownloader(tempFolder); + } + + @After + public void tearDown() throws Exception { + FileUtils.recursiveDelete(tempFolder); + FileUtils.recursiveDelete(tempFile); + } + + @Test + public void testJsonDownload() throws Exception { + new JsonDownloader(downloader, new URL("http://downloads.arduino.cc/libraries/library_index.json")).download(tempFile, new MultiStepProgress(1), ""); + + InputStream indexIn = new FileInputStream(tempFile); + ObjectMapper mapper = new ObjectMapper(); + mapper.registerModule(new MrBeanModule()); + mapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, true); + mapper.configure(DeserializationFeature.EAGER_DESERIALIZER_FETCH, true); + mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + LibrariesIndex librariesIndex = mapper.readValue(indexIn, LibrariesIndex.class); + + assertTrue(librariesIndex != null); + } +} diff --git a/arduino-core/src/cc/arduino/contributions/GZippedJsonDownloader.java b/arduino-core/src/cc/arduino/contributions/GZippedJsonDownloader.java new file mode 100644 index 000000000..5fddd5dc8 --- /dev/null +++ b/arduino-core/src/cc/arduino/contributions/GZippedJsonDownloader.java @@ -0,0 +1,50 @@ +package cc.arduino.contributions; + +import cc.arduino.utils.Progress; +import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; +import org.apache.commons.compress.compressors.gzip.GzipUtils; +import org.apache.commons.compress.utils.IOUtils; + +import java.io.*; +import java.net.URL; + +public class GZippedJsonDownloader { + + private final DownloadableContributionsDownloader downloader; + private final URL url; + private final URL gzippedUrl; + + public GZippedJsonDownloader(DownloadableContributionsDownloader downloader, URL url, URL gzippedUrl) { + this.downloader = downloader; + this.url = url; + this.gzippedUrl = gzippedUrl; + } + + public void download(File tmpFile, Progress progress, String statusText) throws Exception { + try { + new JsonDownloader(downloader, gzippedUrl).download(tmpFile, progress, statusText); + File gzipTmpFile = new File(tmpFile.getParentFile(), GzipUtils.getCompressedFilename(tmpFile.getName())); + tmpFile.renameTo(gzipTmpFile); + decompress(gzipTmpFile, tmpFile); + } catch (Exception e) { + new JsonDownloader(downloader, url).download(tmpFile, progress, statusText); + } + } + + private void decompress(File gzipTmpFile, File tmpFile) throws IOException { + OutputStream os = null; + GzipCompressorInputStream gzipIs = null; + try { + os = new FileOutputStream(tmpFile); + gzipIs = new GzipCompressorInputStream(new FileInputStream(gzipTmpFile)); + final byte[] buffer = new byte[4096]; + int n = 0; + while (-1 != (n = gzipIs.read(buffer))) { + os.write(buffer, 0, n); + } + } finally { + IOUtils.closeQuietly(os); + IOUtils.closeQuietly(gzipIs); + } + } +} diff --git a/arduino-core/src/cc/arduino/contributions/JsonDownloader.java b/arduino-core/src/cc/arduino/contributions/JsonDownloader.java new file mode 100644 index 000000000..5c889d486 --- /dev/null +++ b/arduino-core/src/cc/arduino/contributions/JsonDownloader.java @@ -0,0 +1,26 @@ +package cc.arduino.contributions; + +import cc.arduino.utils.Progress; + +import java.io.File; +import java.net.URL; + +public class JsonDownloader { + + private final DownloadableContributionsDownloader downloader; + private final URL url; + + public JsonDownloader(DownloadableContributionsDownloader downloader, URL url) { + this.downloader = downloader; + this.url = url; + } + + public void download(File tmpFile, Progress progress, String statusText) throws Exception { + try { + downloader.download(url, tmpFile, progress, statusText); + } catch (InterruptedException e) { + // Download interrupted... just exit + return; + } + } +} diff --git a/arduino-core/src/cc/arduino/contributions/libraries/LibraryInstaller.java b/arduino-core/src/cc/arduino/contributions/libraries/LibraryInstaller.java index 27e5abeb8..bf7eec121 100644 --- a/arduino-core/src/cc/arduino/contributions/libraries/LibraryInstaller.java +++ b/arduino-core/src/cc/arduino/contributions/libraries/LibraryInstaller.java @@ -29,6 +29,7 @@ package cc.arduino.contributions.libraries; import cc.arduino.contributions.DownloadableContributionsDownloader; +import cc.arduino.contributions.GZippedJsonDownloader; import cc.arduino.utils.ArchiveExtractor; import cc.arduino.utils.MultiStepProgress; import cc.arduino.utils.Progress; @@ -45,6 +46,7 @@ import static processing.app.I18n._; public class LibraryInstaller { private static final String LIBRARY_INDEX_URL; + private static final String LIBRARY_INDEX_URL_GZ; static { String externalLibraryIndexUrl = System.getProperty("LIBRARY_INDEX_URL"); @@ -53,6 +55,7 @@ public class LibraryInstaller { } else { LIBRARY_INDEX_URL = "http://downloads.arduino.cc/libraries/library_index.json"; } + LIBRARY_INDEX_URL_GZ = "http://downloads.arduino.cc/libraries/library_index.json.gz"; } private final LibrariesIndexer indexer; @@ -77,8 +80,8 @@ public class LibraryInstaller { File outputFile = indexer.getIndexFile(); File tmpFile = new File(outputFile.getAbsolutePath() + ".tmp"); try { - downloader.download(url, tmpFile, progress, - _("Downloading libraries index...")); + GZippedJsonDownloader gZippedJsonDownloader = new GZippedJsonDownloader(downloader, new URL(LIBRARY_INDEX_URL), new URL(LIBRARY_INDEX_URL_GZ)); + gZippedJsonDownloader.download(tmpFile, progress, _("Downloading libraries index...")); } catch (InterruptedException e) { // Download interrupted... just exit return; @@ -91,8 +94,7 @@ public class LibraryInstaller { if (outputFile.exists()) outputFile.delete(); if (!tmpFile.renameTo(outputFile)) - throw new Exception( - _("An error occurred while updating libraries index!")); + throw new Exception(_("An error occurred while updating libraries index!")); // Step 2: Rescan index rescanLibraryIndex(progress); diff --git a/build/shared/revisions.txt b/build/shared/revisions.txt index b542f548e..86ae17ace 100644 --- a/build/shared/revisions.txt +++ b/build/shared/revisions.txt @@ -10,6 +10,7 @@ ARDUINO 1.6.5 * Fixed a bug that made the IDE notify users of invalid libraries too many times. Thanks @Chris--A * Removed JNA. Less native stuff and less chances of incurring into an UnsatisfiedLinkError * Many new and old issues closed. Thanks to many, and @Chris--A in particular +* Faster libraries list update [libraries] * LiquidCrystal fixes. Thanks @newbie15