rooty/Arduino
1
0
Fork 0
mirror of https://github.com/arduino/Arduino.git synced 2025-01-17 06:52:18 +01:00
Code Issues Releases Activity

Faster library list downloading by downloading gzipped version

Browse Source
This commit is contained in:
Federico Fissore 2015-05-21 17:01:36 +02:00
parent 365b0bdc94
commit 94b16a550e
6 changed files with 189 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
app/test/cc/arduino/contributions/GzippedJsonDownloaderTest.java Normal file
View File

@ -0,0 +1,53 @@
package cc.arduino.contributions;
import cc.arduino.contributions.libraries.LibrariesIndex;
import cc.arduino.utils.MultiStepProgress;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.module.mrbean.MrBeanModule;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import processing.app.helpers.FileUtils;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.net.URL;
import static org.junit.Assert.assertTrue;
public class GzippedJsonDownloaderTest {
private File tempFolder;
private File tempFile;
private DownloadableContributionsDownloader downloader;
@Before
public void setUp() throws Exception {
tempFolder = FileUtils.createTempFolder();
tempFile = File.createTempFile("test", ".json");
downloader = new DownloadableContributionsDownloader(tempFolder);
}
@After
public void tearDown() throws Exception {
FileUtils.recursiveDelete(tempFolder);
FileUtils.recursiveDelete(tempFile);
}
@Test
public void testJsonDownload() throws Exception {
new GZippedJsonDownloader(downloader, new URL("http://downloads.arduino.cc/libraries/library_index.json"), new URL("http://downloads.arduino.cc/libraries/library_index.json.gz")).download(tempFile, new MultiStepProgress(1), "");
InputStream indexIn = new FileInputStream(tempFile);
ObjectMapper mapper = new ObjectMapper();
mapper.registerModule(new MrBeanModule());
mapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, true);
mapper.configure(DeserializationFeature.EAGER_DESERIALIZER_FETCH, true);
mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
LibrariesIndex librariesIndex = mapper.readValue(indexIn, LibrariesIndex.class);
assertTrue(librariesIndex != null);
}
}

53
app/test/cc/arduino/contributions/JsonDownloaderTest.java Normal file
View File

@ -0,0 +1,53 @@
package cc.arduino.contributions;
import cc.arduino.contributions.libraries.LibrariesIndex;
import cc.arduino.utils.MultiStepProgress;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.module.mrbean.MrBeanModule;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import processing.app.helpers.FileUtils;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.net.URL;
import static org.junit.Assert.assertTrue;
public class JsonDownloaderTest {
private File tempFolder;
private File tempFile;
private DownloadableContributionsDownloader downloader;
@Before
public void setUp() throws Exception {
tempFolder = FileUtils.createTempFolder();
tempFile = File.createTempFile("test", ".json");
downloader = new DownloadableContributionsDownloader(tempFolder);
}
@After
public void tearDown() throws Exception {
FileUtils.recursiveDelete(tempFolder);
FileUtils.recursiveDelete(tempFile);
}
@Test
public void testJsonDownload() throws Exception {
new JsonDownloader(downloader, new URL("http://downloads.arduino.cc/libraries/library_index.json")).download(tempFile, new MultiStepProgress(1), "");
InputStream indexIn = new FileInputStream(tempFile);
ObjectMapper mapper = new ObjectMapper();
mapper.registerModule(new MrBeanModule());
mapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, true);
mapper.configure(DeserializationFeature.EAGER_DESERIALIZER_FETCH, true);
mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
LibrariesIndex librariesIndex = mapper.readValue(indexIn, LibrariesIndex.class);
assertTrue(librariesIndex != null);
}
}

50
arduino-core/src/cc/arduino/contributions/GZippedJsonDownloader.java Normal file
View File

@ -0,0 +1,50 @@
package cc.arduino.contributions;
import cc.arduino.utils.Progress;
import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
import org.apache.commons.compress.compressors.gzip.GzipUtils;
import org.apache.commons.compress.utils.IOUtils;
import java.io.*;
import java.net.URL;
public class GZippedJsonDownloader {
private final DownloadableContributionsDownloader downloader;
private final URL url;
private final URL gzippedUrl;
public GZippedJsonDownloader(DownloadableContributionsDownloader downloader, URL url, URL gzippedUrl) {
this.downloader = downloader;
this.url = url;
this.gzippedUrl = gzippedUrl;
}
public void download(File tmpFile, Progress progress, String statusText) throws Exception {
try {
new JsonDownloader(downloader, gzippedUrl).download(tmpFile, progress, statusText);
File gzipTmpFile = new File(tmpFile.getParentFile(), GzipUtils.getCompressedFilename(tmpFile.getName()));
tmpFile.renameTo(gzipTmpFile);
decompress(gzipTmpFile, tmpFile);
} catch (Exception e) {
new JsonDownloader(downloader, url).download(tmpFile, progress, statusText);
}
}
private void decompress(File gzipTmpFile, File tmpFile) throws IOException {
OutputStream os = null;
GzipCompressorInputStream gzipIs = null;
try {
os = new FileOutputStream(tmpFile);
gzipIs = new GzipCompressorInputStream(new FileInputStream(gzipTmpFile));
final byte[] buffer = new byte[4096];
int n = 0;
while (-1 != (n = gzipIs.read(buffer))) {
os.write(buffer, 0, n);
}
} finally {
IOUtils.closeQuietly(os);
IOUtils.closeQuietly(gzipIs);
}
}
}

26
arduino-core/src/cc/arduino/contributions/JsonDownloader.java Normal file
View File

@ -0,0 +1,26 @@
package cc.arduino.contributions;
import cc.arduino.utils.Progress;
import java.io.File;
import java.net.URL;
public class JsonDownloader {
private final DownloadableContributionsDownloader downloader;
private final URL url;
public JsonDownloader(DownloadableContributionsDownloader downloader, URL url) {
this.downloader = downloader;
this.url = url;
}
public void download(File tmpFile, Progress progress, String statusText) throws Exception {
try {
downloader.download(url, tmpFile, progress, statusText);
} catch (InterruptedException e) {
// Download interrupted... just exit
return;
}
}
}

10
arduino-core/src/cc/arduino/contributions/libraries/LibraryInstaller.java
View File

@ -29,6 +29,7 @@
package cc.arduino.contributions.libraries;
import cc.arduino.contributions.DownloadableContributionsDownloader;
import cc.arduino.contributions.GZippedJsonDownloader;
import cc.arduino.utils.ArchiveExtractor;
import cc.arduino.utils.MultiStepProgress;
import cc.arduino.utils.Progress;
@ -45,6 +46,7 @@ import static processing.app.I18n._;
public class LibraryInstaller {
private static final String LIBRARY_INDEX_URL;
private static final String LIBRARY_INDEX_URL_GZ;
static {
String externalLibraryIndexUrl = System.getProperty("LIBRARY_INDEX_URL");
@ -53,6 +55,7 @@ public class LibraryInstaller {
} else {
LIBRARY_INDEX_URL = "http://downloads.arduino.cc/libraries/library_index.json";
}
LIBRARY_INDEX_URL_GZ = "http://downloads.arduino.cc/libraries/library_index.json.gz";
}
private final LibrariesIndexer indexer;
@ -77,8 +80,8 @@ public class LibraryInstaller {
File outputFile = indexer.getIndexFile();
File tmpFile = new File(outputFile.getAbsolutePath() + ".tmp");
try {
downloader.download(url, tmpFile, progress,
_("Downloading libraries index..."));
GZippedJsonDownloader gZippedJsonDownloader = new GZippedJsonDownloader(downloader, new URL(LIBRARY_INDEX_URL), new URL(LIBRARY_INDEX_URL_GZ));
gZippedJsonDownloader.download(tmpFile, progress, _("Downloading libraries index..."));
} catch (InterruptedException e) {
// Download interrupted... just exit
return;
@ -91,8 +94,7 @@ public class LibraryInstaller {
if (outputFile.exists())
outputFile.delete();
if (!tmpFile.renameTo(outputFile))
throw new Exception(
_("An error occurred while updating libraries index!"));
throw new Exception(_("An error occurred while updating libraries index!"));
// Step 2: Rescan index
rescanLibraryIndex(progress);

1
build/shared/revisions.txt
View File

@ -10,6 +10,7 @@ ARDUINO 1.6.5
* Fixed a bug that made the IDE notify users of invalid libraries too many times. Thanks @Chris--A
* Removed JNA. Less native stuff and less chances of incurring into an UnsatisfiedLinkError
* Many new and old issues closed. Thanks to many, and @Chris--A in particular
* Faster libraries list update
[libraries]
* LiquidCrystal fixes. Thanks @newbie15