diff --git a/arduino-core/src/cc/arduino/utils/network/CacheControl.java b/arduino-core/src/cc/arduino/utils/network/CacheControl.java
new file mode 100644
index 000000000..432624356
--- /dev/null
+++ b/arduino-core/src/cc/arduino/utils/network/CacheControl.java
@@ -0,0 +1,236 @@
+package cc.arduino.utils.network;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Represents a HTTP Cache-Control response header and parses it from string.
+ *
+ * Note: This class ignores 1#field-name parameter for
+ * private and no-cache directive and cache extensions.
+ *
+ * @see HTTP/1.1 section 14.9
+ */
+public class CacheControl {
+
+
+ // copied from org.apache.abdera.protocol.util.CacheControlUtil
+ private static final Pattern PATTERN
+ = Pattern.compile("\\s*([\\w\\-]+)\\s*(=)?\\s*(\\-?\\d+|\\\"([^\"\\\\]*(\\\\.[^\"\\\\]*)*)+\\\")?\\s*");
+
+ /**
+ * Corresponds to the max-age cache control directive.
+ * The default value is -1, i.e. not specified.
+ *
+ * @see HTTP/1.1 section 14.9.3
+ */
+ private int maxAge = -1;
+
+ /**
+ * Corresponds to the s-maxage cache control directive.
+ * The default value is -1, i.e. not specified.
+ *
+ * @see HTTP/1.1 section 14.9.3
+ */
+ private int sMaxAge = -1;
+
+ /**
+ * Whether the must-revalidate directive is specified.
+ * The default value is false.
+ *
+ * @see HTTP/1.1 section 14.9.4
+ */
+ private boolean isMustRevalidate = false;
+
+ /**
+ * Whether the no-cache directive is specified.
+ * The default value is false.
+ *
+ * @see HTTP/1.1 section 14.9.1
+ */
+ private boolean isNoCache = false;
+
+ /**
+ * Whether the no-store directive is specified.
+ * The default value is false.
+ *
+ * @see HTTP/1.1 section 14.9.2
+ */
+ private boolean isNoStore = false;
+
+ /**
+ * Whether the no-transform directive is specified.
+ * The default value is false.
+ *
+ * @see HTTP/1.1 section 14.9.5
+ */
+ private boolean isNoTransform = false;
+
+ /**
+ * Whether the private directive is specified.
+ * The default value is false.
+ *
+ * @see HTTP/1.1 section 14.9.1
+ */
+ private boolean isPrivate = false;
+
+ /**
+ * Whether the public directive is specified.
+ * The default value is false.
+ *
+ * @see HTTP/1.1 section 14.9.1
+ */
+ private boolean isPublic = false;
+
+ /**
+ * Whether the proxy-revalidate directive is specified.
+ * The default value is false.
+ *
+ * @see HTTP/1.1 section 14.9.4
+ */
+ private boolean isProxyRevalidate = false;
+
+
+ /**
+ * Creates a new instance of CacheControl by parsing the supplied string.
+ *
+ * @param value A value the Cache-Control header.
+ */
+ public static CacheControl valueOf(String value) {
+ CacheControl cc = new CacheControl();
+
+ if (value != null) {
+ Matcher matcher = PATTERN.matcher(value);
+ while (matcher.find()) {
+ switch (matcher.group(1).toLowerCase()) {
+ case "max-age":
+ cc.setMaxAge(Integer.parseInt(matcher.group(3))); break;
+ case "s-maxage":
+ cc.setSMaxAge(Integer.parseInt(matcher.group(3))); break;
+ case "must-revalidate":
+ cc.setMustRevalidate(true); break;
+ case "no-cache":
+ cc.setNoCache(true); break;
+ case "no-store":
+ cc.setNoStore(true); break;
+ case "no-transform":
+ cc.setNoTransform(true); break;
+ case "private":
+ cc.setPrivate(true); break;
+ case "public":
+ cc.setPublic(true); break;
+ case "proxy-revalidate":
+ cc.setProxyRevalidate(true); break;
+ default: //ignore
+ }
+ }
+ }
+ return cc;
+ }
+
+ /**
+ * Returns max-age, or s-maxage according to whether
+ * considering a shared cache, or a private cache. If shared cache and the
+ * s-maxage is negative (i.e. not set), then returns
+ * max-age instead.
+ *
+ * @param sharedCache true for a shared cache,
+ * or false for a private cache
+ * @return A {@link #maxAge}, or {@link #sMaxAge} according to the given
+ * sharedCache argument.
+ */
+ public int getMaxAge(boolean sharedCache) {
+ if (sharedCache) {
+ return sMaxAge >= 0 ? sMaxAge : maxAge;
+ } else {
+ return maxAge;
+ }
+ }
+
+ public void setMaxAge(int maxAge) {
+ this.maxAge = maxAge;
+ }
+
+ public int getMaxAge() {
+ return maxAge;
+ }
+
+ public int getSMaxAge() {
+ return sMaxAge;
+ }
+
+ public void setSMaxAge(int sMaxAge) {
+ this.sMaxAge = sMaxAge;
+ }
+
+ public boolean isMustRevalidate() {
+ return isMustRevalidate;
+ }
+
+ public void setMustRevalidate(boolean mustRevalidate) {
+ isMustRevalidate = mustRevalidate;
+ }
+
+ public boolean isNoCache() {
+ return isNoCache;
+ }
+
+ public void setNoCache(boolean noCache) {
+ isNoCache = noCache;
+ }
+
+ public boolean isNoStore() {
+ return isNoStore;
+ }
+
+ public void setNoStore(boolean noStore) {
+ isNoStore = noStore;
+ }
+
+ public boolean isNoTransform() {
+ return isNoTransform;
+ }
+
+ public void setNoTransform(boolean noTransform) {
+ isNoTransform = noTransform;
+ }
+
+ public boolean isPrivate() {
+ return isPrivate;
+ }
+
+ public void setPrivate(boolean aPrivate) {
+ isPrivate = aPrivate;
+ }
+
+ public boolean isPublic() {
+ return isPublic;
+ }
+
+ public void setPublic(boolean aPublic) {
+ isPublic = aPublic;
+ }
+
+ public boolean isProxyRevalidate() {
+ return isProxyRevalidate;
+ }
+
+ public void setProxyRevalidate(boolean proxyRevalidate) {
+ isProxyRevalidate = proxyRevalidate;
+ }
+
+ @Override
+ public String toString() {
+ return "CacheControl{" +
+ "maxAge=" + maxAge +
+ ", sMaxAge=" + sMaxAge +
+ ", isMustRevalidate=" + isMustRevalidate +
+ ", isNoCache=" + isNoCache +
+ ", isNoStore=" + isNoStore +
+ ", isNoTransform=" + isNoTransform +
+ ", isPrivate=" + isPrivate +
+ ", isPublic=" + isPublic +
+ ", isProxyRevalidate=" + isProxyRevalidate +
+ '}';
+ }
+}
diff --git a/arduino-core/src/cc/arduino/utils/network/FileDownloader.java b/arduino-core/src/cc/arduino/utils/network/FileDownloader.java
index ef671cdff..674277041 100644
--- a/arduino-core/src/cc/arduino/utils/network/FileDownloader.java
+++ b/arduino-core/src/cc/arduino/utils/network/FileDownloader.java
@@ -30,9 +30,8 @@
package cc.arduino.utils.network;
import org.apache.commons.compress.utils.IOUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import processing.app.BaseNoGui;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
import processing.app.helpers.FileUtils;
import java.io.File;
@@ -48,7 +47,7 @@ import java.util.Observable;
import java.util.Optional;
public class FileDownloader extends Observable {
- private static Logger log = LoggerFactory.getLogger(FileDownloader.class);
+ private static Logger log = LogManager.getLogger(FileDownloader.class);
public enum Status {
CONNECTING, //
@@ -146,15 +145,14 @@ public class FileDownloader extends Observable {
try {
setStatus(Status.CONNECTING);
- final File settingsFolder = BaseNoGui.getPlatform().getSettingsFolder();
- final String cacheFolder = Paths.get(settingsFolder.getPath(), "cache").toString();
- final FileDownloaderCache fileDownloaderCache = FileDownloaderCache.getFileCached(cacheFolder, downloadUrl);
+ final Optional fileCached = FileDownloaderCache.getFileCached(downloadUrl);
- if (!fileDownloaderCache.isChange()) {
+ if (fileCached.isPresent() && !fileCached.get().isChange()) {
try {
final Optional fileFromCache =
- fileDownloaderCache.getFileFromCache();
+ fileCached.get().getFileFromCache();
if (fileFromCache.isPresent()) {
+ log.info("No need to download using cached file: {}", fileCached.get());
FileUtils.copyFile(fileFromCache.get(), outputFile);
setStatus(Status.COMPLETE);
return;
@@ -195,7 +193,7 @@ public class FileDownloader extends Observable {
synchronized (this) {
stream = connection.getInputStream();
}
- byte buffer[] = new byte[10240];
+ byte[] buffer = new byte[10240];
while (status == Status.DOWNLOADING) {
int read = stream.read(buffer);
if (read == -1)
@@ -216,7 +214,9 @@ public class FileDownloader extends Observable {
}
// Set the cache whe it finish to download the file
IOUtils.closeQuietly(randomAccessOutputFile);
- fileDownloaderCache.fillCache(outputFile);
+ if (fileCached.isPresent()) {
+ fileCached.get().updateCacheFile(outputFile);
+ }
setStatus(Status.COMPLETE);
} catch (InterruptedException e) {
setStatus(Status.CANCELLED);
diff --git a/arduino-core/src/cc/arduino/utils/network/FileDownloaderCache.java b/arduino-core/src/cc/arduino/utils/network/FileDownloaderCache.java
index 0d11510c2..32e425abb 100644
--- a/arduino-core/src/cc/arduino/utils/network/FileDownloaderCache.java
+++ b/arduino-core/src/cc/arduino/utils/network/FileDownloaderCache.java
@@ -1,8 +1,16 @@
package cc.arduino.utils.network;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import processing.app.PreferencesData;
+import cc.arduino.utils.FileHash;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import processing.app.BaseNoGui;
import processing.app.helpers.FileUtils;
import javax.script.ScriptException;
@@ -15,91 +23,289 @@ import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
-import java.util.Optional;
+import java.security.NoSuchAlgorithmException;
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.*;
+import java.util.function.BiFunction;
+import java.util.function.Function;
+import java.util.stream.Collectors;
public class FileDownloaderCache {
- private static Logger log = LoggerFactory.getLogger(FileDownloaderCache.class);
- private final Path cacheFilePath;
- private final String remoteETag;
- private final String preferencesDataKey;
+ private final static Logger log = LogManager
+ .getLogger(FileDownloaderCache.class);
+ private static Map cachedFiles = Collections
+ .synchronizedMap(new HashMap<>());
+ private static String cacheFolder;
- // BaseNoGui.getSettingsFolder()
- private FileDownloaderCache(Path cacheFilePath, String remoteETag, String preferencesDataKey) {
- this.cacheFilePath = cacheFilePath;
- this.remoteETag = remoteETag;
- this.preferencesDataKey = preferencesDataKey;
+ static {
+ final File settingsFolder;
+ try {
+ settingsFolder = BaseNoGui.getPlatform().getSettingsFolder();
+ cacheFolder = Paths.get(settingsFolder.getPath(), "cache")
+ .toString();
+ final Path pathCacheInfo = getCachedInfoPath();
+ if (Files.exists(pathCacheInfo)) {
+ ObjectMapper mapper = new ObjectMapper();
+ final JsonNode jsonNode = mapper.readTree(pathCacheInfo.toFile());
+
+ // Read the files array
+ TypeReference> typeRef = new TypeReference>() {
+ };
+ final List files = mapper
+ .readValue(mapper.treeAsTokens(jsonNode.get("files")), typeRef);
+
+ // Create a map with the remote url as a key and the file cache info as a value
+ cachedFiles = Collections
+ .synchronizedMap(files.stream().collect(
+ Collectors.toMap(FileCached::getRemoteURL, Function.identity())));
+ }
+ } catch (Exception e) {
+ log.error("Cannot initialized the cache", e);
+ }
}
- public static FileDownloaderCache getFileCached(String cacheFolder, URL remoteURL)
- throws IOException, NoSuchMethodException, ScriptException, URISyntaxException {
+ public static Optional getFileCached(URL remoteURL)
+ throws URISyntaxException, NoSuchMethodException, ScriptException,
+ IOException {
- final String[] splitPath = remoteURL.getPath().split("/");
- final String preferencesDataKey = "cache.file." + remoteURL.getPath();
- final Path cacheFilePath;
- if (splitPath.length > 0) {
- cacheFilePath = Paths.get(cacheFolder, splitPath);
+ final Optional fileCachedOpt;
+
+ // The file info must exist in the cachedFiles map but also the real file must exist in the file system
+ if (cachedFiles.containsKey(remoteURL.toString()) && cachedFiles.get(remoteURL.toString()).exists()) {
+ fileCachedOpt = Optional.of(cachedFiles.get(remoteURL.toString()));
} else {
- cacheFilePath = null;
- }
-
- final HttpURLConnection headRequest = new HttpConnectionManager(remoteURL)
- .makeConnection((connection) -> {
- try {
- connection.setRequestMethod("HEAD");
- } catch (ProtocolException e) {
- log.error("Invalid protocol", e);
+ // Update
+ fileCachedOpt = FileDownloaderCache.updateCacheInfo(remoteURL, (remoteETagClean, cacheControl) -> {
+ // Check cache control data
+ if (cacheControl.isNoCache() || cacheControl.isMustRevalidate()) {
+ log.warn("The file must not be cache due to cache control header {}",
+ cacheControl);
+ return Optional.empty();
}
+
+ final String[] splitPath = remoteURL.getPath().split("/");
+ final Path cacheFilePath;
+ if (splitPath.length > 0) {
+ Deque addFirstRemoteURL = new LinkedList<>(Arrays.asList(splitPath));
+ addFirstRemoteURL.addFirst(remoteURL.getHost());
+ cacheFilePath = Paths.get(cacheFolder, addFirstRemoteURL.toArray(new String[0]));
+ return Optional.of(
+ new FileCached(remoteETagClean, remoteURL.toString(), cacheFilePath.toString(),
+ cacheControl));
+ }
+ log.warn("The remote path as no file name {}", remoteURL);
+ return Optional.empty();
});
+ }
+ return fileCachedOpt;
+ }
+
+ private static Optional updateCacheInfo(URL remoteURL, BiFunction> getNewFile)
+ throws URISyntaxException, NoSuchMethodException, ScriptException,
+ IOException {
+ // Update the headers of the cached file
+ final HttpURLConnection headRequest = new HttpConnectionManager(
+ remoteURL).makeConnection((connection) -> {
+ try {
+ connection.setRequestMethod("HEAD");
+ } catch (ProtocolException e) {
+ log.error("Invalid protocol", e);
+ }
+ });
final int responseCode = headRequest.getResponseCode();
headRequest.disconnect();
// Something bad is happening return a conservative true to try to download the file
if (responseCode < 200 || responseCode >= 300) {
log.warn("The head request return a bad response code " + responseCode);
// if something bad happend
- return new FileDownloaderCache(cacheFilePath, null, preferencesDataKey);
+ return Optional.empty();
}
-
- final String remoteETag = headRequest.getHeaderField("ETag");
- String remoteETagClean = null;
- if (remoteETag != null) {
- remoteETagClean = remoteETag.trim().replace("\"", "");
+ // Get all the useful headers
+ String remoteETag = headRequest.getHeaderField("ETag");
+ String cacheControlHeader = headRequest.getHeaderField("Cache-Control");
+ if (remoteETag != null && cacheControlHeader != null) {
+ final String remoteETagClean = remoteETag.trim().replace("\"", "");
+ final CacheControl cacheControl = CacheControl.valueOf(cacheControlHeader);
+ return getNewFile.apply(remoteETagClean, cacheControl);
}
-
- return new FileDownloaderCache(cacheFilePath, remoteETagClean, preferencesDataKey);
+ log.warn("The head request do not return the ETag {} or the Cache-Control {}", remoteETag, cacheControlHeader);
+ return Optional.empty();
}
- public boolean isChange() {
+ private static void updateCacheFilesInfo() throws IOException {
+ if (cachedFiles != null) {
+ synchronized (cachedFiles) {
+ ObjectMapper mapper = new ObjectMapper();
+ mapper.enable(SerializationFeature.INDENT_OUTPUT);
+ final ObjectNode objectNode = mapper.createObjectNode();
+ objectNode.putArray("files").addAll(
+ cachedFiles.values().stream()
+ .map((v) -> mapper.convertValue(v, JsonNode.class))
+ .collect(Collectors.toList()));
+ // Create the path Arduino15/cache
+ Path cachedFileInfo = getCachedInfoPath();
+ if (Files.notExists(cachedFileInfo)) {
+ Files.createDirectories(cachedFileInfo.getParent());
+ }
+ mapper.writeValue(cachedFileInfo.toFile(), objectNode);
+ }
+ }
+ }
- final String localETag = PreferencesData.get(preferencesDataKey);
+ private static Path getCachedInfoPath() {
+ return Paths.get(cacheFolder, "cache.json");
+ }
- // If the header doesn't exist or the local cache doesn't exist you need to download the file
- if (cacheFilePath == null || remoteETag == null || localETag == null) {
+ @JsonIgnoreProperties(ignoreUnknown = true)
+ static class FileCached {
+ private String eTag;
+ @JsonIgnore
+ private final String lastETag;
+ private final String remoteURL;
+ private final String localPath;
+ private String md5;
+ private String createdAt;
+ private final CacheControl cacheControl;
+
+ FileCached() {
+ this.lastETag = null;
+ this.remoteURL = null;
+ this.localPath = null;
+ this.cacheControl = null;
+ }
+
+ FileCached(String lastETag, String remoteURL, String localPath, CacheControl cacheControl) {
+ this.lastETag = lastETag;
+ this.remoteURL = remoteURL;
+ this.localPath = localPath;
+ this.cacheControl = cacheControl;
+ }
+
+ @JsonIgnore
+ public boolean isChange() {
+ // Check if the file is expire
+ if (this.getExpiresTime().isAfter(LocalDateTime.now())) {
+ log.info("The file \"{}\" is no expire, the etag will not be checked. Expire time: {}", localPath, this.getExpiresTime().toString());
+ return false;
+ }
+
+ if (lastETag != null) {
+ // If are different means that the file is change
+ return !lastETag.equals(eTag);
+ }
return true;
}
- // If are different means that the file is change
- return !remoteETag.equals(localETag);
- }
-
- public Optional getFileFromCache() {
- if (Optional.ofNullable(cacheFilePath).isPresent() && Files.exists(cacheFilePath)) {
- return Optional.of(new File(cacheFilePath.toUri()));
- }
- return Optional.empty();
-
- }
-
- public void fillCache(File fileToCache) throws Exception {
- if (Optional.ofNullable(remoteETag).isPresent() &&
- Optional.ofNullable(cacheFilePath).isPresent()) {
-
- PreferencesData.set(preferencesDataKey, remoteETag);
- // If the cache directory does not exist create it
- if (!Files.exists(cacheFilePath.getParent())) {
- Files.createDirectories(cacheFilePath.getParent());
+ @JsonIgnore
+ public boolean exists() throws IOException {
+ if (localPath != null && Files.exists(Paths.get(localPath))) {
+ try {
+ final String md5Local = FileHash.hash(Paths.get(localPath).toFile(), "MD5");
+ if (md5Local.equals(md5)) {
+ return true;
+ }
+ } catch (NoSuchAlgorithmException e) {
+ log.error("MD5 algorithm is not supported", e);
+ }
}
- FileUtils.copyFile(fileToCache, cacheFilePath.toFile());
+ return false;
+ }
+
+ @JsonIgnore
+ public Optional getFileFromCache() {
+ if (localPath != null && Files.exists(Paths.get(localPath))) {
+ return Optional.of(new File(localPath));
+ }
+ return Optional.empty();
+
+ }
+
+ public void updateCacheFile(File fileToCache) throws Exception {
+ if (Optional.ofNullable(lastETag).isPresent() && Optional
+ .ofNullable(localPath).isPresent()) {
+ Path cacheFilePath = Paths.get(localPath);
+
+ // If the cache directory does not exist create it
+ if (!Files.exists(cacheFilePath.getParent())) {
+ Files.createDirectories(cacheFilePath.getParent());
+ }
+ final File cacheFile = cacheFilePath.toFile();
+ FileUtils.copyFile(fileToCache, cacheFile);
+ eTag = lastETag;
+ createdAt = LocalDateTime.now().format(DateTimeFormatter.ISO_DATE_TIME);
+ updateMD5();
+ }
+ log.info("Update cache file: {}", this);
+ cachedFiles.put(remoteURL, this);
+ updateCacheFilesInfo();
+ }
+
+ private void updateMD5() throws IOException, NoSuchAlgorithmException {
+ if (localPath != null) {
+ md5 = FileHash.hash(Paths.get(localPath).toFile(), "MD5");
+ }
+ }
+
+ @JsonIgnore
+ public LocalDateTime getExpiresTime() {
+ final int maxAge;
+ if (cacheControl != null) {
+ maxAge = cacheControl.getMaxAge();
+ } else {
+ maxAge = 0;
+ }
+ if (createdAt != null) {
+ return LocalDateTime.parse(createdAt, DateTimeFormatter.ISO_DATE_TIME)
+ .plusSeconds(maxAge);
+ }
+ return LocalDateTime.now();
+
+ }
+
+ public String getExpires() {
+ return getExpiresTime().toString();
+ }
+
+ public String getMD5() {
+ return md5;
+ }
+
+ public String geteTag() {
+ return eTag;
+ }
+
+ public String getRemoteURL() {
+ return remoteURL;
+ }
+
+ public String getLocalPath() {
+ return localPath;
+ }
+
+ public void setMd5(String md5) {
+ this.md5 = md5;
+ }
+
+ public String getCreatedAt() {
+ return createdAt;
+ }
+
+ public CacheControl getCacheControl() {
+ return cacheControl;
+ }
+
+ @Override
+ public String toString() {
+ return "FileCached{" +
+ "eTag='" + eTag + '\'' +
+ ", lastETag='" + lastETag + '\'' +
+ ", remoteURL='" + remoteURL + '\'' +
+ ", localPath='" + localPath + '\'' +
+ ", md5='" + md5 + '\'' +
+ ", createdAt='" + createdAt + '\'' +
+ ", cacheControl=" + cacheControl +
+ '}';
}
}
-
}