mirror of
https://github.com/arduino/Arduino.git
synced 2025-01-20 09:52:13 +01:00
Add the file downloader cache to make faster the library/boards manager
This commit is contained in:
Mattia Bertorello
parent
00a7546fb4
commit
6592c42dcf
@ -29,26 +29,26 @@
|
||||
|
||||
package cc.arduino.utils.network;
|
||||
|
||||
import cc.arduino.net.CustomProxySelector;
|
||||
import org.apache.commons.codec.binary.Base64;
|
||||
import org.apache.commons.compress.utils.IOUtils;
|
||||
|
||||
import processing.app.BaseNoGui;
|
||||
import processing.app.PreferencesData;
|
||||
import processing.app.helpers.FileUtils;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.RandomAccessFile;
|
||||
import java.net.HttpURLConnection;
|
||||
import java.net.Proxy;
|
||||
import java.net.SocketTimeoutException;
|
||||
import java.net.URL;
|
||||
import java.net.*;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.Observable;
|
||||
import java.util.Optional;
|
||||
import java.util.logging.Level;
|
||||
import java.util.logging.Logger;
|
||||
|
||||
public class FileDownloader extends Observable {
|
||||
private static Logger log = Logger
|
||||
.getLogger(FileDownloader.class.getName());
|
||||
|
||||
public enum Status {
|
||||
CONNECTING, //
|
||||
@ -68,15 +68,12 @@ public class FileDownloader extends Observable {
|
||||
private final File outputFile;
|
||||
private InputStream stream = null;
|
||||
private Exception error;
|
||||
private String userAgent;
|
||||
|
||||
public FileDownloader(URL url, File file) {
|
||||
downloadUrl = url;
|
||||
outputFile = file;
|
||||
downloaded = 0;
|
||||
initialSize = 0;
|
||||
userAgent = "ArduinoIDE/" + BaseNoGui.VERSION_NAME + " Java/"
|
||||
+ System.getProperty("java.version");
|
||||
}
|
||||
|
||||
public long getInitialSize() {
|
||||
@ -144,12 +141,37 @@ public class FileDownloader extends Observable {
|
||||
}
|
||||
|
||||
private void downloadFile(boolean noResume) throws InterruptedException {
|
||||
RandomAccessFile file = null;
|
||||
RandomAccessFile randomAccessOutputFile = null;
|
||||
|
||||
try {
|
||||
setStatus(Status.CONNECTING);
|
||||
|
||||
final File settingsFolder = BaseNoGui.getPlatform().getSettingsFolder();
|
||||
final String cacheFolder = Paths.get(settingsFolder.getPath(), "cache").toString();
|
||||
final FileDownloaderCache fileDownloaderCache =
|
||||
new FileDownloaderCache(cacheFolder, downloadUrl);
|
||||
|
||||
final boolean isChanged = fileDownloaderCache.checkIfTheFileIsChanged();
|
||||
|
||||
if (!isChanged) {
|
||||
try {
|
||||
final Optional<File> fileFromCache =
|
||||
fileDownloaderCache.getFileFromCache();
|
||||
if (fileFromCache.isPresent()) {
|
||||
FileUtils.copyFile(fileFromCache.get(), outputFile);
|
||||
setStatus(Status.COMPLETE);
|
||||
return;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.log(Level.WARNING,
|
||||
"Cannot get the file from the cache, will be downloaded a new one ", e.getCause());
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
// Open file and seek to the end of it
|
||||
file = new RandomAccessFile(outputFile, "rw");
|
||||
initialSize = file.length();
|
||||
randomAccessOutputFile = new RandomAccessFile(outputFile, "rw");
|
||||
initialSize = randomAccessOutputFile.length();
|
||||
|
||||
if (noResume && initialSize > 0) {
|
||||
// delete file and restart downloading
|
||||
@ -157,49 +179,11 @@ public class FileDownloader extends Observable {
|
||||
initialSize = 0;
|
||||
}
|
||||
|
||||
file.seek(initialSize);
|
||||
randomAccessOutputFile.seek(initialSize);
|
||||
|
||||
setStatus(Status.CONNECTING);
|
||||
|
||||
Proxy proxy = new CustomProxySelector(PreferencesData.getMap()).getProxyFor(downloadUrl.toURI());
|
||||
if ("true".equals(System.getProperty("DEBUG"))) {
|
||||
System.err.println("Using proxy " + proxy);
|
||||
}
|
||||
|
||||
HttpURLConnection connection = (HttpURLConnection) downloadUrl.openConnection(proxy);
|
||||
connection.setRequestProperty("User-agent", userAgent);
|
||||
if (downloadUrl.getUserInfo() != null) {
|
||||
String auth = "Basic " + new String(new Base64().encode(downloadUrl.getUserInfo().getBytes()));
|
||||
connection.setRequestProperty("Authorization", auth);
|
||||
}
|
||||
|
||||
connection.setRequestProperty("Range", "bytes=" + initialSize + "-");
|
||||
connection.setConnectTimeout(5000);
|
||||
setDownloaded(0);
|
||||
|
||||
// Connect
|
||||
connection.connect();
|
||||
int resp = connection.getResponseCode();
|
||||
|
||||
if (resp == HttpURLConnection.HTTP_MOVED_PERM || resp == HttpURLConnection.HTTP_MOVED_TEMP) {
|
||||
URL newUrl = new URL(connection.getHeaderField("Location"));
|
||||
|
||||
proxy = new CustomProxySelector(PreferencesData.getMap()).getProxyFor(newUrl.toURI());
|
||||
|
||||
// open the new connnection again
|
||||
connection = (HttpURLConnection) newUrl.openConnection(proxy);
|
||||
connection.setRequestProperty("User-agent", userAgent);
|
||||
if (downloadUrl.getUserInfo() != null) {
|
||||
String auth = "Basic " + new String(new Base64().encode(downloadUrl.getUserInfo().getBytes()));
|
||||
connection.setRequestProperty("Authorization", auth);
|
||||
}
|
||||
|
||||
connection.setRequestProperty("Range", "bytes=" + initialSize + "-");
|
||||
connection.setConnectTimeout(5000);
|
||||
|
||||
connection.connect();
|
||||
resp = connection.getResponseCode();
|
||||
}
|
||||
final HttpURLConnection connection = new HttpConnectionManager(downloadUrl)
|
||||
.makeConnection((c) -> setDownloaded(0));
|
||||
final int resp = connection.getResponseCode();
|
||||
|
||||
if (resp < 200 || resp >= 300) {
|
||||
throw new IOException("Received invalid http status code from server: " + resp);
|
||||
@ -221,11 +205,11 @@ public class FileDownloader extends Observable {
|
||||
if (read == -1)
|
||||
break;
|
||||
|
||||
file.write(buffer, 0, read);
|
||||
randomAccessOutputFile.write(buffer, 0, read);
|
||||
setDownloaded(getDownloaded() + read);
|
||||
|
||||
if (Thread.interrupted()) {
|
||||
file.close();
|
||||
randomAccessOutputFile.close();
|
||||
throw new InterruptedException();
|
||||
}
|
||||
}
|
||||
@ -234,6 +218,8 @@ public class FileDownloader extends Observable {
|
||||
if (getDownloaded() < getDownloadSize())
|
||||
throw new Exception("Incomplete download");
|
||||
}
|
||||
// Set the cache whe it finish to download the file
|
||||
fileDownloaderCache.fillCache(outputFile);
|
||||
setStatus(Status.COMPLETE);
|
||||
} catch (InterruptedException e) {
|
||||
setStatus(Status.CANCELLED);
|
||||
@ -249,7 +235,7 @@ public class FileDownloader extends Observable {
|
||||
setError(e);
|
||||
|
||||
} finally {
|
||||
IOUtils.closeQuietly(file);
|
||||
IOUtils.closeQuietly(randomAccessOutputFile);
|
||||
|
||||
synchronized (this) {
|
||||
IOUtils.closeQuietly(stream);
|
||||
|
||||
@ -0,0 +1,100 @@
|
||||
package cc.arduino.utils.network;
|
||||
|
||||
import com.sun.istack.internal.NotNull;
|
||||
import processing.app.PreferencesData;
|
||||
import processing.app.helpers.FileUtils;
|
||||
|
||||
import javax.script.ScriptException;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.net.HttpURLConnection;
|
||||
import java.net.ProtocolException;
|
||||
import java.net.URISyntaxException;
|
||||
import java.net.URL;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.Optional;
|
||||
import java.util.logging.Logger;
|
||||
|
||||
public class FileDownloaderCache {
|
||||
private static Logger log = Logger
|
||||
.getLogger(FileDownloaderCache.class.getName());
|
||||
private final String cacheFolder;
|
||||
private final URL remoteURL;
|
||||
private final Path cacheFilePath;
|
||||
// Will be initialized by call the checkIfTheFileIsChanged function
|
||||
private String eTag;
|
||||
|
||||
// BaseNoGui.getSettingsFolder()
|
||||
public FileDownloaderCache(@NotNull String cacheFolder, @NotNull URL remoteURL) {
|
||||
this.cacheFolder = cacheFolder;
|
||||
this.remoteURL = remoteURL;
|
||||
String[] splitPath = remoteURL.getPath().split("/");
|
||||
if (splitPath.length > 0) {
|
||||
this.cacheFilePath = Paths.get(cacheFolder, splitPath[splitPath.length - 1]);
|
||||
} else {
|
||||
this.cacheFilePath = null;
|
||||
}
|
||||
}
|
||||
|
||||
public boolean checkIfTheFileIsChanged()
|
||||
throws NoSuchMethodException, ScriptException, IOException,
|
||||
URISyntaxException {
|
||||
|
||||
final HttpURLConnection headRequest = new HttpConnectionManager(remoteURL)
|
||||
.makeConnection((connection) -> {
|
||||
try {
|
||||
connection.setRequestMethod("HEAD");
|
||||
} catch (ProtocolException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
});
|
||||
final int responseCode = headRequest.getResponseCode();
|
||||
// Something bad is happening return a conservative true to try to download the file
|
||||
if (responseCode < 200 || responseCode >= 300) {
|
||||
log.warning("The head request return a bad response code " + responseCode);
|
||||
return true;
|
||||
}
|
||||
|
||||
final String remoteETag = headRequest.getHeaderField("ETag");
|
||||
final String localETag = PreferencesData.get(getPreferencesDataKey());
|
||||
|
||||
// If the header doesn't exist or the local cache doesn't exist you need to download the file
|
||||
if (remoteETag == null || localETag == null) {
|
||||
return true;
|
||||
}
|
||||
eTag = remoteETag.trim().replace("\"", "");
|
||||
|
||||
// If are different means that the file is change
|
||||
return !eTag.equals(localETag);
|
||||
}
|
||||
|
||||
public Optional<File> getFileFromCache() {
|
||||
if (Optional.ofNullable(cacheFilePath).isPresent()) {
|
||||
if (Files.exists(cacheFilePath)) {
|
||||
return Optional.of(new File(cacheFilePath.toUri()));
|
||||
}
|
||||
}
|
||||
return Optional.empty();
|
||||
|
||||
}
|
||||
|
||||
public void fillCache(File fileToCache) throws Exception {
|
||||
if (Optional.ofNullable(eTag).isPresent() &&
|
||||
Optional.ofNullable(cacheFilePath).isPresent()) {
|
||||
|
||||
PreferencesData.set(getPreferencesDataKey(), eTag);
|
||||
// If the cache directory does not exist create it
|
||||
final Path cachePath = Paths.get(cacheFolder);
|
||||
if (!Files.exists(cachePath)) {
|
||||
Files.createDirectory(cachePath);
|
||||
}
|
||||
FileUtils.copyFile(fileToCache, cacheFilePath.toFile());
|
||||
}
|
||||
}
|
||||
|
||||
private String getPreferencesDataKey() {
|
||||
return "cache.file." + remoteURL.getPath();
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,114 @@
|
||||
package cc.arduino.utils.network;
|
||||
|
||||
import cc.arduino.net.CustomProxySelector;
|
||||
import org.apache.commons.codec.binary.Base64;
|
||||
import processing.app.BaseNoGui;
|
||||
import processing.app.PreferencesData;
|
||||
|
||||
import javax.script.ScriptException;
|
||||
import java.io.IOException;
|
||||
import java.net.HttpURLConnection;
|
||||
import java.net.Proxy;
|
||||
import java.net.URISyntaxException;
|
||||
import java.net.URL;
|
||||
import java.util.function.Consumer;
|
||||
import java.util.logging.Level;
|
||||
import java.util.logging.Logger;
|
||||
|
||||
public class HttpConnectionManager {
|
||||
private static Logger log = Logger
|
||||
.getLogger(HttpConnectionManager.class.getName());
|
||||
private final URL requestURL;
|
||||
private final String userAgent;
|
||||
private int connectTimeout;
|
||||
|
||||
|
||||
public HttpConnectionManager(URL requestURL) {
|
||||
this.requestURL = requestURL;
|
||||
final String defaultUserAgent = String.format(
|
||||
"ArduinoIDE/%s (%s; %s; %s; %s) Java/%s (%s)",
|
||||
BaseNoGui.VERSION_NAME,
|
||||
System.getProperty("os.name"),
|
||||
System.getProperty("os.version"),
|
||||
System.getProperty("os.arch"),
|
||||
System.getProperty("user.language"),
|
||||
System.getProperty("java.version"),
|
||||
System.getProperty("java.vendor")
|
||||
);
|
||||
this.userAgent = PreferencesData.get("http.user_agent", defaultUserAgent);
|
||||
try {
|
||||
this.connectTimeout =
|
||||
Integer.parseInt(
|
||||
PreferencesData.get("http.connection_timeout", "5000"));
|
||||
} catch (NumberFormatException e) {
|
||||
log.log(Level.WARNING,
|
||||
"Cannot parse the http.connection_timeout configuration switch to default 5000 milliseconds", e.getCause());
|
||||
this.connectTimeout = 5000;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public HttpURLConnection makeConnection(Consumer<HttpURLConnection> beforeConnection)
|
||||
throws URISyntaxException, NoSuchMethodException, IOException,
|
||||
ScriptException {
|
||||
return makeConnection(this.requestURL, 0, beforeConnection);
|
||||
}
|
||||
|
||||
private HttpURLConnection makeConnection(URL requestURL, int movedTimes,
|
||||
Consumer<HttpURLConnection> beforeConnection)
|
||||
throws NoSuchMethodException, ScriptException, IOException,
|
||||
URISyntaxException {
|
||||
log.info("Prepare http request to " + requestURL);
|
||||
if (requestURL == null) {
|
||||
log.warning("Invalid request url is null");
|
||||
throw new RuntimeException("Invalid request url is null");
|
||||
}
|
||||
if (movedTimes > 3) {
|
||||
log.warning("Too many redirect " + requestURL);
|
||||
throw new RuntimeException("Too many redirect " + requestURL);
|
||||
}
|
||||
|
||||
Proxy proxy = new CustomProxySelector(PreferencesData.getMap())
|
||||
.getProxyFor(requestURL.toURI());
|
||||
if ("true".equals(System.getProperty("DEBUG"))) {
|
||||
System.err.println("Using proxy " + proxy);
|
||||
}
|
||||
|
||||
HttpURLConnection connection = (HttpURLConnection) requestURL
|
||||
.openConnection(proxy);
|
||||
connection.setRequestProperty("User-agent", userAgent);
|
||||
if (requestURL.getUserInfo() != null) {
|
||||
String auth = "Basic " + new String(
|
||||
new Base64().encode(requestURL.getUserInfo().getBytes()));
|
||||
connection.setRequestProperty("Authorization", auth);
|
||||
}
|
||||
|
||||
int initialSize = 0;
|
||||
connection.setRequestProperty("Range", "bytes=" + initialSize + "-");
|
||||
connection.setConnectTimeout(connectTimeout);
|
||||
beforeConnection.accept(connection);
|
||||
|
||||
// Connect
|
||||
log.info("Connect to " + requestURL);
|
||||
|
||||
connection.connect();
|
||||
int resp = connection.getResponseCode();
|
||||
|
||||
if (resp == HttpURLConnection.HTTP_MOVED_PERM
|
||||
|| resp == HttpURLConnection.HTTP_MOVED_TEMP) {
|
||||
|
||||
URL newUrl = new URL(connection.getHeaderField("Location"));
|
||||
log.info("The response code was a 301,302 so try again with the new URL " + newUrl);
|
||||
|
||||
return this.makeConnection(newUrl, movedTimes + 1, beforeConnection);
|
||||
}
|
||||
log.info("The response code was" + resp);
|
||||
|
||||
return connection;
|
||||
}
|
||||
|
||||
public void setConnectTimeout(int connectTimeout) {
|
||||
this.connectTimeout = connectTimeout;
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user