Add the file downloader cache to make faster the library/boards manager

arduino-core/src/cc/arduino/utils/network/FileDownloader.java

@@ -29,26 +29,26 @@
 
 package cc.arduino.utils.network;
 
-import cc.arduino.net.CustomProxySelector;
-import org.apache.commons.codec.binary.Base64;
 import org.apache.commons.compress.utils.IOUtils;
 
 import processing.app.BaseNoGui;
-import processing.app.PreferencesData;
+import processing.app.helpers.FileUtils;
 
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.RandomAccessFile;
-import java.net.HttpURLConnection;
+import java.net.*;
-import java.net.Proxy;
-import java.net.SocketTimeoutException;
-import java.net.URL;
 import java.nio.file.Files;
 import java.nio.file.Paths;
 import java.util.Observable;
+import java.util.Optional;
+import java.util.logging.Level;
+import java.util.logging.Logger;
 
 public class FileDownloader extends Observable {
+  private static Logger log = Logger
+    .getLogger(FileDownloader.class.getName());
 
   public enum Status {
     CONNECTING, //
@@ -68,15 +68,12 @@ public class FileDownloader extends Observable {
   private final File outputFile;
   private InputStream stream = null;
   private Exception error;
-  private String userAgent;
 
   public FileDownloader(URL url, File file) {
     downloadUrl = url;
     outputFile = file;
     downloaded = 0;
     initialSize = 0;
-    userAgent = "ArduinoIDE/" + BaseNoGui.VERSION_NAME + " Java/"
-                + System.getProperty("java.version");
   }
 
   public long getInitialSize() {
@@ -144,12 +141,37 @@ public class FileDownloader extends Observable {
   }
 
   private void downloadFile(boolean noResume) throws InterruptedException {
-    RandomAccessFile file = null;
+    RandomAccessFile randomAccessOutputFile = null;
 
     try {
+      setStatus(Status.CONNECTING);
+
+      final File settingsFolder = BaseNoGui.getPlatform().getSettingsFolder();
+      final String cacheFolder = Paths.get(settingsFolder.getPath(), "cache").toString();
+      final FileDownloaderCache fileDownloaderCache =
+        new FileDownloaderCache(cacheFolder, downloadUrl);
+
+      final boolean isChanged = fileDownloaderCache.checkIfTheFileIsChanged();
+
+      if (!isChanged) {
+        try {
+          final Optional<File> fileFromCache =
+            fileDownloaderCache.getFileFromCache();
+          if (fileFromCache.isPresent()) {
+            FileUtils.copyFile(fileFromCache.get(), outputFile);
+            setStatus(Status.COMPLETE);
+            return;
+          }
+        } catch (Exception e) {
+          log.log(Level.WARNING,
+            "Cannot get the file from the cache, will be downloaded a new one ", e.getCause());
+
+        }
+      }
+
       // Open file and seek to the end of it
-      file = new RandomAccessFile(outputFile, "rw");
+      randomAccessOutputFile = new RandomAccessFile(outputFile, "rw");
-      initialSize = file.length();
+      initialSize = randomAccessOutputFile.length();
 
       if (noResume && initialSize > 0) {
         // delete file and restart downloading
@@ -157,49 +179,11 @@ public class FileDownloader extends Observable {
         initialSize = 0;
       }
 
-      file.seek(initialSize);
+      randomAccessOutputFile.seek(initialSize);
 
-      setStatus(Status.CONNECTING);
+      final HttpURLConnection connection = new HttpConnectionManager(downloadUrl)
-
+        .makeConnection((c) -> setDownloaded(0));
-      Proxy proxy = new CustomProxySelector(PreferencesData.getMap()).getProxyFor(downloadUrl.toURI());
+      final int resp = connection.getResponseCode();
-      if ("true".equals(System.getProperty("DEBUG"))) {
-        System.err.println("Using proxy " + proxy);
-      }
-
-      HttpURLConnection connection = (HttpURLConnection) downloadUrl.openConnection(proxy);
-      connection.setRequestProperty("User-agent", userAgent);
-      if (downloadUrl.getUserInfo() != null) {
-        String auth = "Basic " + new String(new Base64().encode(downloadUrl.getUserInfo().getBytes()));
-        connection.setRequestProperty("Authorization", auth);
-      }
-
-      connection.setRequestProperty("Range", "bytes=" + initialSize + "-");
-      connection.setConnectTimeout(5000);
-      setDownloaded(0);
-
-      // Connect
-      connection.connect();
-      int resp = connection.getResponseCode();
-
-      if (resp == HttpURLConnection.HTTP_MOVED_PERM || resp == HttpURLConnection.HTTP_MOVED_TEMP) {
-        URL newUrl = new URL(connection.getHeaderField("Location"));
-
-        proxy = new CustomProxySelector(PreferencesData.getMap()).getProxyFor(newUrl.toURI());
-
-        // open the new connnection again
-        connection = (HttpURLConnection) newUrl.openConnection(proxy);
-        connection.setRequestProperty("User-agent", userAgent);
-        if (downloadUrl.getUserInfo() != null) {
-          String auth = "Basic " + new String(new Base64().encode(downloadUrl.getUserInfo().getBytes()));
-          connection.setRequestProperty("Authorization", auth);
-        }
-
-        connection.setRequestProperty("Range", "bytes=" + initialSize + "-");
-        connection.setConnectTimeout(5000);
-
-        connection.connect();
-        resp = connection.getResponseCode();
-      }
 
       if (resp < 200 || resp >= 300) {
         throw new IOException("Received invalid http status code from server: " + resp);
@@ -221,11 +205,11 @@ public class FileDownloader extends Observable {
         if (read == -1)
           break;
 
-        file.write(buffer, 0, read);
+        randomAccessOutputFile.write(buffer, 0, read);
         setDownloaded(getDownloaded() + read);
 
         if (Thread.interrupted()) {
-          file.close();
+          randomAccessOutputFile.close();
           throw new InterruptedException();
         }
       }
@@ -234,6 +218,8 @@ public class FileDownloader extends Observable {
         if (getDownloaded() < getDownloadSize())
           throw new Exception("Incomplete download");
       }
+      // Set the cache whe it finish to download the file
+      fileDownloaderCache.fillCache(outputFile);
       setStatus(Status.COMPLETE);
     } catch (InterruptedException e) {
       setStatus(Status.CANCELLED);
@@ -249,7 +235,7 @@ public class FileDownloader extends Observable {
       setError(e);
 
     } finally {
-      IOUtils.closeQuietly(file);
+      IOUtils.closeQuietly(randomAccessOutputFile);
 
       synchronized (this) {
         IOUtils.closeQuietly(stream);

arduino-core/src/cc/arduino/utils/network/FileDownloaderCache.java

@@ -0,0 +1,100 @@
+package cc.arduino.utils.network;
+
+import com.sun.istack.internal.NotNull;
+import processing.app.PreferencesData;
+import processing.app.helpers.FileUtils;
+
+import javax.script.ScriptException;
+import java.io.File;
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.ProtocolException;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Optional;
+import java.util.logging.Logger;
+
+public class FileDownloaderCache {
+  private static Logger log = Logger
+    .getLogger(FileDownloaderCache.class.getName());
+  private final String cacheFolder;
+  private final URL remoteURL;
+  private final Path cacheFilePath;
+  // Will be initialized by call the checkIfTheFileIsChanged function
+  private String eTag;
+
+  // BaseNoGui.getSettingsFolder()
+  public FileDownloaderCache(@NotNull String cacheFolder, @NotNull URL remoteURL) {
+    this.cacheFolder = cacheFolder;
+    this.remoteURL = remoteURL;
+    String[] splitPath = remoteURL.getPath().split("/");
+    if (splitPath.length > 0) {
+      this.cacheFilePath = Paths.get(cacheFolder, splitPath[splitPath.length - 1]);
+    } else {
+      this.cacheFilePath = null;
+    }
+  }
+
+  public boolean checkIfTheFileIsChanged()
+    throws NoSuchMethodException, ScriptException, IOException,
+    URISyntaxException {
+
+    final HttpURLConnection headRequest = new HttpConnectionManager(remoteURL)
+      .makeConnection((connection) -> {
+        try {
+          connection.setRequestMethod("HEAD");
+        } catch (ProtocolException e) {
+          throw new RuntimeException(e);
+        }
+      });
+    final int responseCode = headRequest.getResponseCode();
+    // Something bad is happening return a conservative true to try to download the file
+    if (responseCode < 200 || responseCode >= 300) {
+      log.warning("The head request return a bad response code " + responseCode);
+      return true;
+    }
+
+    final String remoteETag = headRequest.getHeaderField("ETag");
+    final String localETag = PreferencesData.get(getPreferencesDataKey());
+
+    // If the header doesn't exist or the local cache doesn't exist you need to download the file
+    if (remoteETag == null || localETag == null) {
+      return true;
+    }
+    eTag = remoteETag.trim().replace("\"", "");
+
+    // If are different means that the file is change
+    return !eTag.equals(localETag);
+  }
+
+  public Optional<File> getFileFromCache() {
+    if (Optional.ofNullable(cacheFilePath).isPresent()) {
+      if (Files.exists(cacheFilePath)) {
+        return Optional.of(new File(cacheFilePath.toUri()));
+      }
+    }
+    return Optional.empty();
+
+  }
+
+  public void fillCache(File fileToCache) throws Exception {
+    if (Optional.ofNullable(eTag).isPresent() &&
+      Optional.ofNullable(cacheFilePath).isPresent()) {
+
+      PreferencesData.set(getPreferencesDataKey(), eTag);
+      // If the cache directory does not exist create it
+      final Path cachePath = Paths.get(cacheFolder);
+      if (!Files.exists(cachePath)) {
+        Files.createDirectory(cachePath);
+      }
+      FileUtils.copyFile(fileToCache, cacheFilePath.toFile());
+    }
+  }
+
+  private String getPreferencesDataKey() {
+    return "cache.file." + remoteURL.getPath();
+  }
+}

arduino-core/src/cc/arduino/utils/network/HttpConnectionManager.java

@@ -0,0 +1,114 @@
+package cc.arduino.utils.network;
+
+import cc.arduino.net.CustomProxySelector;
+import org.apache.commons.codec.binary.Base64;
+import processing.app.BaseNoGui;
+import processing.app.PreferencesData;
+
+import javax.script.ScriptException;
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.Proxy;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.util.function.Consumer;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+public class HttpConnectionManager {
+  private static Logger log = Logger
+    .getLogger(HttpConnectionManager.class.getName());
+  private final URL requestURL;
+  private final String userAgent;
+  private int connectTimeout;
+
+
+  public HttpConnectionManager(URL requestURL) {
+    this.requestURL = requestURL;
+    final String defaultUserAgent = String.format(
+      "ArduinoIDE/%s (%s; %s; %s; %s) Java/%s (%s)",
+      BaseNoGui.VERSION_NAME,
+      System.getProperty("os.name"),
+      System.getProperty("os.version"),
+      System.getProperty("os.arch"),
+      System.getProperty("user.language"),
+      System.getProperty("java.version"),
+      System.getProperty("java.vendor")
+    );
+    this.userAgent = PreferencesData.get("http.user_agent", defaultUserAgent);
+    try {
+      this.connectTimeout =
+        Integer.parseInt(
+          PreferencesData.get("http.connection_timeout", "5000"));
+    } catch (NumberFormatException e) {
+      log.log(Level.WARNING,
+        "Cannot parse the http.connection_timeout configuration switch to default 5000 milliseconds", e.getCause());
+      this.connectTimeout = 5000;
+    }
+
+  }
+
+  public HttpURLConnection makeConnection(Consumer<HttpURLConnection> beforeConnection)
+    throws URISyntaxException, NoSuchMethodException, IOException,
+    ScriptException {
+    return makeConnection(this.requestURL, 0, beforeConnection);
+  }
+
+  private HttpURLConnection makeConnection(URL requestURL, int movedTimes,
+                                           Consumer<HttpURLConnection> beforeConnection)
+    throws NoSuchMethodException, ScriptException, IOException,
+    URISyntaxException {
+    log.info("Prepare http request to " + requestURL);
+    if (requestURL == null) {
+      log.warning("Invalid request url is null");
+      throw new RuntimeException("Invalid request url is null");
+    }
+    if (movedTimes > 3) {
+      log.warning("Too many redirect " + requestURL);
+      throw new RuntimeException("Too many redirect " + requestURL);
+    }
+
+    Proxy proxy = new CustomProxySelector(PreferencesData.getMap())
+      .getProxyFor(requestURL.toURI());
+    if ("true".equals(System.getProperty("DEBUG"))) {
+      System.err.println("Using proxy " + proxy);
+    }
+
+    HttpURLConnection connection = (HttpURLConnection) requestURL
+      .openConnection(proxy);
+    connection.setRequestProperty("User-agent", userAgent);
+    if (requestURL.getUserInfo() != null) {
+      String auth = "Basic " + new String(
+        new Base64().encode(requestURL.getUserInfo().getBytes()));
+      connection.setRequestProperty("Authorization", auth);
+    }
+
+    int initialSize = 0;
+    connection.setRequestProperty("Range", "bytes=" + initialSize + "-");
+    connection.setConnectTimeout(connectTimeout);
+    beforeConnection.accept(connection);
+
+    // Connect
+    log.info("Connect to " + requestURL);
+
+    connection.connect();
+    int resp = connection.getResponseCode();
+
+    if (resp == HttpURLConnection.HTTP_MOVED_PERM
+        || resp == HttpURLConnection.HTTP_MOVED_TEMP) {
+
+      URL newUrl = new URL(connection.getHeaderField("Location"));
+      log.info("The response code was a 301,302 so try again with the new URL " + newUrl);
+
+      return this.makeConnection(newUrl, movedTimes + 1, beforeConnection);
+    }
+    log.info("The response code was" + resp);
+
+    return connection;
+  }
+
+  public void setConnectTimeout(int connectTimeout) {
+    this.connectTimeout = connectTimeout;
+  }
+}
+