rooty/Arduino
1
0
Fork 0
mirror of https://github.com/arduino/Arduino.git synced 2025-01-31 20:52:13 +01:00
Code Issues Releases Activity

Fix portable mode and make the fileCached immutable

Browse Source
This commit is contained in:
Mattia Bertorello 2019-07-07 13:02:27 +02:00
parent fa77c15e8e
commit 4a944df758
No known key found for this signature in database
GPG Key ID: CE1FB2BE91770F24
4 changed files with 177 additions and 114 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
arduino-core/src/cc/arduino/contributions/DownloadableContributionsDownloader.java
View File

@ -170,6 +170,7 @@ public class DownloadableContributionsDownloader {
log.info("The domain is not selected to verify the signature. will be copied into this path {}, packageIndex url: {}", packageIndex, packageIndexUrl); log.info("The domain is not selected to verify the signature. will be copied into this path {}, packageIndex url: {}", packageIndex, packageIndexUrl);
} }
} catch (Exception e) { } catch (Exception e) {
log.error("Cannot download the package index from {} the package will be discard", packageIndexUrl, e);
throw e; throw e;
} finally { } finally {
// Delete useless temp file // Delete useless temp file

2
arduino-core/src/cc/arduino/contributions/libraries/LibraryInstaller.java
View File

@ -48,8 +48,6 @@ import java.io.IOException;
import java.net.URL; import java.net.URL;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.StandardCopyOption; import java.nio.file.StandardCopyOption;
import java.util.LinkedList;
import java.util.List;
import java.util.Optional; import java.util.Optional;
import static processing.app.I18n.tr; import static processing.app.I18n.tr;

12
arduino-core/src/cc/arduino/utils/network/FileDownloader.java
View File

@ -147,7 +147,7 @@ public class FileDownloader extends Observable {
final Optional<FileDownloaderCache.FileCached> fileCached = FileDownloaderCache.getFileCached(downloadUrl); final Optional<FileDownloaderCache.FileCached> fileCached = FileDownloaderCache.getFileCached(downloadUrl);
if (fileCached.isPresent() && !fileCached.get().isChange()) { if (fileCached.isPresent() && fileCached.get().isNotChange()) {
try { try {
final Optional<File> fileFromCache = final Optional<File> fileFromCache =
fileCached.get().getFileFromCache(); fileCached.get().getFileFromCache();
@ -156,11 +156,17 @@ public class FileDownloader extends Observable {
FileUtils.copyFile(fileFromCache.get(), outputFile); FileUtils.copyFile(fileFromCache.get(), outputFile);
setStatus(Status.COMPLETE); setStatus(Status.COMPLETE);
return; return;
} else {
log.info(
"The file in the cache is not in the path or the md5 validation failed: path={}, file exist={}, md5 validation={}",
fileCached.get().getLocalPath(), fileCached.get().exists(), fileCached.get().md5Check());
} }
} catch (Exception e) { } catch (Exception e) {
log.warn( log.warn(
"Cannot get the file from the cache, will be downloaded a new one ", e.getCause()); "Cannot get the file from the cache, will be downloaded a new one ", e);
} }
} else {
log.info("The file is change {}", fileCached);
} }
initialSize = outputFile.length(); initialSize = outputFile.length();
@ -226,10 +232,12 @@ public class FileDownloader extends Observable {
} catch (SocketTimeoutException e) { } catch (SocketTimeoutException e) {
setStatus(Status.CONNECTION_TIMEOUT_ERROR); setStatus(Status.CONNECTION_TIMEOUT_ERROR);
setError(e); setError(e);
log.error(e);
} catch (Exception e) { } catch (Exception e) {
setStatus(Status.ERROR); setStatus(Status.ERROR);
setError(e); setError(e);
log.error(e);
} finally { } finally {
IOUtils.closeQuietly(randomAccessOutputFile); IOUtils.closeQuietly(randomAccessOutputFile);

276
arduino-core/src/cc/arduino/utils/network/FileDownloaderCache.java
View File

@ -63,20 +63,34 @@ import java.util.function.Function;
import java.util.stream.Collectors; import java.util.stream.Collectors;
public class FileDownloaderCache { public class FileDownloaderCache {
private final static String CACHE_ENABLE_PREFERENCE_KEY = "cache.enable";
private final static Logger log = LogManager private final static Logger log = LogManager
.getLogger(FileDownloaderCache.class); .getLogger(FileDownloaderCache.class);
private static Map<String, FileCached> cachedFiles = Collections private final static Map<String, FileCached> cachedFiles = Collections
.synchronizedMap(new HashMap<>()); .synchronizedMap(new HashMap<>());
private static String cacheFolder; private final static String cacheFolder;
private static boolean enableCache; private static boolean enableCache;
static { static {
enableCache = Boolean.valueOf(PreferencesData.get(CACHE_ENABLE_PREFERENCE_KEY, "true"));
if (!enableCache) {
log.info("The cache is disable cache.enable=false");
}
PreferencesData.set(CACHE_ENABLE_PREFERENCE_KEY, Boolean.toString(enableCache));
final File settingsFolder; final File settingsFolder;
try { settingsFolder = BaseNoGui.getSettingsFolder();
settingsFolder = BaseNoGui.getPlatform().getSettingsFolder(); if (settingsFolder != null) {
cacheFolder = Paths.get(settingsFolder.getPath(), "cache") cacheFolder = Paths.get(settingsFolder.getPath(), "cache")
.toString(); .toString();
final Path pathCacheInfo = getCachedInfoPath(); } else {
enableCache = false;
cacheFolder = null;
log.error("The cache will disable because the setting folder is null, cannot generate the cache path");
}
final Path pathCacheInfo = getCachedInfoPath();
log.info("Cache folder {}", cacheFolder);
try {
if (Files.exists(pathCacheInfo)) { if (Files.exists(pathCacheInfo)) {
ObjectMapper mapper = new ObjectMapper(); ObjectMapper mapper = new ObjectMapper();
final JsonNode jsonNode = mapper.readTree(pathCacheInfo.toFile()); final JsonNode jsonNode = mapper.readTree(pathCacheInfo.toFile());
@ -87,54 +101,72 @@ public class FileDownloaderCache {
final List<FileCached> files = mapper final List<FileCached> files = mapper
.readValue(mapper.treeAsTokens(jsonNode.get("files")), typeRef); .readValue(mapper.treeAsTokens(jsonNode.get("files")), typeRef);
// Create a map with the remote url as a key and the file cache info as a value // Update the map with the remote url as a key and the file cache info as a value
cachedFiles = Collections cachedFiles.putAll(Collections
.synchronizedMap(files.stream().collect( .synchronizedMap(files
Collectors.toMap(FileCached::getRemoteURL, Function.identity()))); .stream()
.filter(FileCached::exists)
.collect(Collectors.toMap(FileCached::getRemoteURL, Function.identity()))
)
);
log.info("Number of file already in the cache {}", cachedFiles.size());
} }
} catch (Exception e) { } catch (Exception e) {
log.error("Cannot initialized the cache", e); log.error("Cannot initialized the cache", e);
} }
enableCache = Boolean.valueOf(PreferencesData.get("cache.enable", "true"));
if (!enableCache) {
log.info("The cache is disable cache.enable=false");
}
} }
public static Optional<FileCached> getFileCached(URL remoteURL) public static Optional<FileCached> getFileCached(final URL remoteURL)
throws URISyntaxException, NoSuchMethodException, ScriptException, throws URISyntaxException, NoSuchMethodException, ScriptException,
IOException { IOException {
// Return always and empty file if the cache is not enable
if (!enableCache) {
log.info("The cache is not enable.");
return Optional.empty();
}
final String[] splitPath = remoteURL.getPath().split("/"); final String[] splitPath = remoteURL.getPath().split("/");
if (splitPath.length == 0) { if (splitPath.length == 0) {
log.warn("The remote path as no file name {}", remoteURL); log.warn("The remote path as no file name {}", remoteURL);
return Optional.empty(); return Optional.empty();
} }
// Create the path where the cached file should exist
final Deque<String> addFirstRemoteURL = new LinkedList<>(Arrays.asList(splitPath));
addFirstRemoteURL.addFirst(remoteURL.getHost());
final Path cacheFilePath = Paths.get(cacheFolder, addFirstRemoteURL.toArray(new String[0]));
// Take from the cache the file info or build from scratch // Take from the cache the file info or build from scratch
final FileCached fileCachedOpt = Optional.ofNullable(cachedFiles.get(remoteURL.toString())) final FileCached fileCached = Optional.ofNullable(cachedFiles.get(remoteURL.toString()))
.orElseGet(() -> { .orElseGet(() -> new FileCached(remoteURL.toString(), cacheFilePath.toString()));
Deque<String> addFirstRemoteURL = new LinkedList<>(Arrays.asList(splitPath));
addFirstRemoteURL.addFirst(remoteURL.getHost());
final Path cacheFilePath = Paths.get(cacheFolder, addFirstRemoteURL.toArray(new String[0]));
return new FileCached(remoteURL.toString(), cacheFilePath.toString());
});
// If the file is change of the cache is disable run the HEAD request to check if the file is changed // If the file is change of the cache is disable run the HEAD request to check if the file is changed
if (fileCachedOpt.isChange() || !enableCache) { if (fileCached.isExpire() || !fileCached.exists()) {
// Update remote etag and cache control header // Update remote etag and cache control header
return FileDownloaderCache.updateCacheInfo(remoteURL, (remoteETagClean, cacheControl) -> { final Optional<FileCached> fileCachedInfoUpdated =
// Check cache control data FileDownloaderCache.updateCacheInfo(remoteURL, (remoteETagClean, cacheControl) -> {
if (cacheControl.isNoCache() || cacheControl.isMustRevalidate() || cacheControl.isNoStore()) { // Check cache control data
log.warn("The file {} must not be cache due to cache control header {}", if (cacheControl.isNoCache() || cacheControl.isMustRevalidate() || cacheControl.isNoStore()) {
remoteURL, cacheControl); log.warn("The file {} must not be cache due to cache control header {}",
return Optional.empty(); remoteURL, cacheControl);
} return Optional.empty();
fileCachedOpt.setLastETag(remoteETagClean); }
fileCachedOpt.setCacheControl(cacheControl); log.info("Update cached info of {}, createdAt {}, previous eTag {}, last eTag {}, cache control header {} ",
return Optional.of(fileCachedOpt); remoteURL, fileCached.createdAt, fileCached.eTag, remoteETagClean, cacheControl);
}); final FileCached fileCachedUpdateETag = new FileCached(
remoteURL.toString(),
cacheFilePath.toString(),
fileCached.eTag,
remoteETagClean, // Set the lastETag
fileCached.md5,
cacheControl // Set the new cache control
);
cachedFiles.put(remoteURL.toString(), fileCachedUpdateETag);
return Optional.of(fileCachedUpdateETag);
});
FileDownloaderCache.updateCacheFilesInfo();
return fileCachedInfoUpdated;
} }
return Optional.of(fileCachedOpt); return Optional.of(fileCached);
} }
private static Optional<FileCached> updateCacheInfo(URL remoteURL, BiFunction<String, CacheControl, Optional<FileCached>> getNewFile) private static Optional<FileCached> updateCacheInfo(URL remoteURL, BiFunction<String, CacheControl, Optional<FileCached>> getNewFile)
@ -169,27 +201,24 @@ public class FileDownloaderCache {
return Optional.empty(); return Optional.empty();
} }
private static void updateCacheFilesInfo() throws IOException { private synchronized static void updateCacheFilesInfo() throws IOException {
if (cachedFiles != null) { ObjectMapper mapper = new ObjectMapper();
synchronized (cachedFiles) { // Generate a pretty json
ObjectMapper mapper = new ObjectMapper(); mapper.enable(SerializationFeature.INDENT_OUTPUT);
// Generate a pretty json final ObjectNode objectNode = mapper.createObjectNode();
mapper.enable(SerializationFeature.INDENT_OUTPUT); // Generate a json {"files":[...{files_info}...]}
final ObjectNode objectNode = mapper.createObjectNode(); objectNode.putArray("files").addAll(
// Generate a json {"files":[...{files_info}...]} cachedFiles.values().stream()
objectNode.putArray("files").addAll( .map((v) -> mapper.convertValue(v, JsonNode.class))
cachedFiles.values().stream() .collect(Collectors.toList()));
.map((v) -> mapper.convertValue(v, JsonNode.class)) // Create the path Arduino15/cache
.collect(Collectors.toList())); Path cachedFileInfo = getCachedInfoPath();
// Create the path Arduino15/cache if (Files.notExists(cachedFileInfo)) {
Path cachedFileInfo = getCachedInfoPath(); Files.createDirectories(cachedFileInfo.getParent());
if (Files.notExists(cachedFileInfo)) {
Files.createDirectories(cachedFileInfo.getParent());
}
// Write to cache.json
mapper.writeValue(cachedFileInfo.toFile(), objectNode);
}
} }
log.info("Update cache file info in {}, number of cached files is {}", cachedFileInfo.toFile(), cachedFiles.size());
// Write to cache.json
mapper.writeValue(cachedFileInfo.toFile(), objectNode);
} }
private static Path getCachedInfoPath() { private static Path getCachedInfoPath() {
@ -200,27 +229,59 @@ public class FileDownloaderCache {
static class FileCached { static class FileCached {
private final String remoteURL; private final String remoteURL;
private final String localPath; private final String localPath;
private String eTag; private final String eTag;
private String lastETag; private final String lastETag;
private String md5; private final String md5;
private String createdAt; private final String createdAt;
private CacheControl cacheControl; private final CacheControl cacheControl;
FileCached() { FileCached() {
this.remoteURL = null; this.remoteURL = null;
this.localPath = null; this.localPath = null;
lastETag = null;
eTag = null;
md5 = null;
createdAt = null;
cacheControl = null;
} }
FileCached(String remoteURL, String localPath) { FileCached(String remoteURL, String localPath) {
this.remoteURL = remoteURL; this.remoteURL = remoteURL;
this.localPath = localPath; this.localPath = localPath;
lastETag = null;
eTag = null;
md5 = null;
createdAt = LocalDateTime.now().format(DateTimeFormatter.ISO_DATE_TIME);
cacheControl = null;
}
public FileCached(String remoteURL, String localPath, String eTag, String lastETag, String md5, CacheControl cacheControl) {
this.remoteURL = remoteURL;
this.localPath = localPath;
this.eTag = eTag;
this.lastETag = lastETag;
this.md5 = md5;
this.createdAt = LocalDateTime.now().format(DateTimeFormatter.ISO_DATE_TIME);
this.cacheControl = cacheControl;
}
@JsonIgnore
public boolean isExpire() {
// Check if the file is expire
return this.getExpiresTime().isBefore(LocalDateTime.now());
}
@JsonIgnore
public boolean isNotChange() {
return !isChange();
} }
@JsonIgnore @JsonIgnore
public boolean isChange() { public boolean isChange() {
// Check if the file is expire // Check if the file is expire
if (this.getExpiresTime().isAfter(LocalDateTime.now())) { if (!isExpire()) {
log.info("The file \"{}\" is no expire, the etag will not be checked. Expire time: {}", localPath, this.getExpiresTime().toString()); log.debug("The file \"{}\" is no expire, the eTag will not be checked. Expire time: {}", localPath,
this.getExpiresTime().format(DateTimeFormatter.ISO_DATE_TIME));
return false; return false;
} }
@ -232,52 +293,63 @@ public class FileDownloaderCache {
} }
@JsonIgnore @JsonIgnore
public boolean exists() throws IOException { public boolean exists() {
if (localPath != null && Files.exists(Paths.get(localPath))) { return localPath != null && Files.exists(Paths.get(localPath));
try {
final String md5Local = FileHash.hash(Paths.get(localPath).toFile(), "MD5");
if (md5Local.equals(md5)) {
return true;
}
} catch (NoSuchAlgorithmException e) {
log.error("MD5 algorithm is not supported", e);
}
}
return false;
} }
@JsonIgnore @JsonIgnore
public Optional<File> getFileFromCache() { public Optional<File> getFileFromCache() {
if (localPath != null && Files.exists(Paths.get(localPath))) { if (md5Check()) {
return Optional.of(new File(localPath)); return Optional.of(Paths.get(localPath).toFile());
} }
return Optional.empty(); return Optional.empty();
} }
public void updateCacheFile(File fileToCache) throws Exception { public synchronized void updateCacheFile(File fileToCache) throws Exception {
if (Optional.ofNullable(lastETag).isPresent() && Optional Path cacheFilePath = Paths.get(localPath);
.ofNullable(localPath).isPresent()) {
Path cacheFilePath = Paths.get(localPath);
// If the cache directory does not exist create it // If the cache directory does not exist create it
if (!Files.exists(cacheFilePath.getParent())) { if (!Files.exists(cacheFilePath.getParent())) {
Files.createDirectories(cacheFilePath.getParent()); Files.createDirectories(cacheFilePath.getParent());
}
final File cacheFile = cacheFilePath.toFile();
FileUtils.copyFile(fileToCache, cacheFile);
eTag = lastETag;
createdAt = LocalDateTime.now().format(DateTimeFormatter.ISO_DATE_TIME);
updateMD5();
} }
log.info("Update cache file: {}", this); FileUtils.copyFile(fileToCache, cacheFilePath.toFile());
cachedFiles.put(remoteURL, this); final String md5 = this.calculateMD5();
final String eTag;
if (lastETag == null) {
log.warn("The eTag was not calculate this time, is not the right behaviour fileCached={}, md5={}", this, md5);
eTag = this.eTag;
} else {
eTag = this.lastETag;
}
FileCached newFileCached = new FileCached(
this.remoteURL,
this.localPath,
eTag, // Initialize the right eTag with the last eTag because the file was updated
eTag,
md5,
this.cacheControl
);
log.info("Update cache file: {}", newFileCached);
cachedFiles.put(remoteURL, newFileCached);
updateCacheFilesInfo(); updateCacheFilesInfo();
} }
private void updateMD5() throws IOException, NoSuchAlgorithmException { private String calculateMD5() throws IOException, NoSuchAlgorithmException {
if (localPath != null) { if (exists()) {
md5 = FileHash.hash(Paths.get(localPath).toFile(), "MD5"); return FileHash.hash(Paths.get(localPath).toFile(), "MD5");
}
return null;
}
@JsonIgnore
public boolean md5Check() {
try {
return !Objects.isNull(getMD5()) && Objects.equals(calculateMD5(), getMD5());
} catch (Exception e) {
log.error("Fail to calculate the MD5. file={}", this, e);
return false;
} }
} }
@ -317,10 +389,6 @@ public class FileDownloaderCache {
return localPath; return localPath;
} }
public void setMd5(String md5) {
this.md5 = md5;
}
public String getCreatedAt() { public String getCreatedAt() {
return createdAt; return createdAt;
} }
@ -329,18 +397,6 @@ public class FileDownloaderCache {
return cacheControl; return cacheControl;
} }
public void seteTag(String eTag) {
this.eTag = eTag;
}
public void setLastETag(String lastETag) {
this.lastETag = lastETag;
}
public void setCacheControl(CacheControl cacheControl) {
this.cacheControl = cacheControl;
}
@Override @Override
public String toString() { public String toString() {
return "FileCached{" + return "FileCached{" +