diff --git a/arduino-core/src/cc/arduino/contributions/DownloadableContributionsDownloader.java b/arduino-core/src/cc/arduino/contributions/DownloadableContributionsDownloader.java index 66cf172cc..ba92f8905 100644 --- a/arduino-core/src/cc/arduino/contributions/DownloadableContributionsDownloader.java +++ b/arduino-core/src/cc/arduino/contributions/DownloadableContributionsDownloader.java @@ -126,7 +126,7 @@ public class DownloadableContributionsDownloader { } public void download(URL url, File tmpFile, Progress progress, String statusText, ProgressListener progressListener, boolean noResume, boolean allowCache) throws Exception { - FileDownloader downloader = new FileDownloader(url, tmpFile, allowCache); + final FileDownloader downloader = new FileDownloader(url, tmpFile, allowCache); downloader.addObserver((o, arg) -> { FileDownloader me = (FileDownloader) o; String msg = ""; @@ -148,22 +148,24 @@ public class DownloadableContributionsDownloader { public void downloadIndexAndSignature(MultiStepProgress progress, URL packageIndexUrl, ProgressListener progressListener, SignatureVerifier signatureVerifier) throws Exception { // Extract the file name from the url - String indexFileName = FilenameUtils.getName(packageIndexUrl.getPath()); - File packageIndex = BaseNoGui.indexer.getIndexFile(indexFileName); + final String indexFileName = FilenameUtils.getName(packageIndexUrl.getPath()); + final File packageIndex = BaseNoGui.indexer.getIndexFile(indexFileName); final String statusText = tr("Downloading platforms index..."); // Create temp files - File packageIndexTemp = File.createTempFile(indexFileName, ".tmp"); + final File packageIndexTemp = File.createTempFile(indexFileName, ".tmp"); try { // Download package index download(packageIndexUrl, packageIndexTemp, progress, statusText, progressListener, true, true); + final URL signatureUrl = new URL(packageIndexUrl.toString() + ".sig"); if (verifyDomain(packageIndexUrl)) { - URL signatureUrl = new URL(packageIndexUrl.toString() + ".sig"); - if (checkSignature(progress, signatureUrl, progressListener, signatureVerifier, statusText, packageIndexTemp)) { Files.move(packageIndexTemp.toPath(), packageIndex.toPath(), StandardCopyOption.REPLACE_EXISTING); + } else { + log.info("The cached files have been removed. {} {}", packageIndexUrl, signatureUrl); + FileDownloader.invalidateFiles(packageIndexUrl, signatureUrl); } } else { // Move the package index to the destination when the signature is not necessary @@ -196,10 +198,17 @@ public class DownloadableContributionsDownloader { public boolean checkSignature(MultiStepProgress progress, URL signatureUrl, ProgressListener progressListener, SignatureVerifier signatureVerifier, String statusText, File fileToVerify) throws Exception { + final boolean allowInsecurePackages = + PreferencesData.getBoolean("allow_insecure_packages", false); + if (allowInsecurePackages) { + log.info("Allow insecure packages is true the signature will be skip and return always verified"); + return true; + } + // Signature file name - String signatureFileName = FilenameUtils.getName(signatureUrl.getPath()); - File packageIndexSignature = BaseNoGui.indexer.getIndexFile(signatureFileName); - File packageIndexSignatureTemp = File.createTempFile(signatureFileName, ".tmp"); + final String signatureFileName = FilenameUtils.getName(signatureUrl.getPath()); + final File packageIndexSignature = BaseNoGui.indexer.getIndexFile(signatureFileName); + final File packageIndexSignatureTemp = File.createTempFile(signatureFileName, ".tmp"); try { @@ -207,7 +216,7 @@ public class DownloadableContributionsDownloader { download(signatureUrl, packageIndexSignatureTemp, progress, statusText, progressListener, true); // Verify the signature before move the files - boolean signatureVerified = signatureVerifier.isSigned(fileToVerify, packageIndexSignatureTemp); + final boolean signatureVerified = signatureVerifier.isSigned(fileToVerify, packageIndexSignatureTemp); if (signatureVerified) { log.info("Signature verified. url={}, signature url={}, file to verify={}, signature file={}", signatureUrl, signatureUrl, fileToVerify, packageIndexSignatureTemp); // Move if the signature is ok diff --git a/arduino-core/src/cc/arduino/contributions/libraries/LibraryInstaller.java b/arduino-core/src/cc/arduino/contributions/libraries/LibraryInstaller.java index cbab19c89..f820f2bb8 100644 --- a/arduino-core/src/cc/arduino/contributions/libraries/LibraryInstaller.java +++ b/arduino-core/src/cc/arduino/contributions/libraries/LibraryInstaller.java @@ -36,6 +36,7 @@ import cc.arduino.contributions.GZippedJsonDownloader; import cc.arduino.contributions.ProgressListener; import cc.arduino.utils.ArchiveExtractor; import cc.arduino.utils.MultiStepProgress; +import cc.arduino.utils.network.FileDownloader; import org.apache.commons.io.FilenameUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -74,9 +75,10 @@ public class LibraryInstaller { String signatureFileName = FilenameUtils.getName(new URL(Constants.LIBRARY_INDEX_URL).getPath()); File libraryIndexTemp = File.createTempFile(signatureFileName, ".tmp"); final URL libraryURL = new URL(Constants.LIBRARY_INDEX_URL); + final URL libraryGzURL = new URL(Constants.LIBRARY_INDEX_URL_GZ); final String statusText = tr("Downloading libraries index..."); try { - GZippedJsonDownloader gZippedJsonDownloader = new GZippedJsonDownloader(downloader, libraryURL, new URL(Constants.LIBRARY_INDEX_URL_GZ)); + GZippedJsonDownloader gZippedJsonDownloader = new GZippedJsonDownloader(downloader, libraryURL, libraryGzURL); gZippedJsonDownloader.download(libraryIndexTemp, progress, statusText, progressListener, true); } catch (InterruptedException e) { // Download interrupted... just exit @@ -98,7 +100,8 @@ public class LibraryInstaller { // Step 3: Rescan index rescanLibraryIndex(progress, progressListener); } else { - log.error("Fail to verify the signature of {}", libraryURL); + FileDownloader.invalidateFiles(libraryGzURL, libraryURL, signatureUrl); + log.error("Fail to verify the signature of {} the cached files have been removed", libraryURL); } } else { log.info("The domain is not selected to verify the signature. library index: {}", signatureUrl); diff --git a/arduino-core/src/cc/arduino/utils/network/FileDownloader.java b/arduino-core/src/cc/arduino/utils/network/FileDownloader.java index e191b8f50..6f4bada2a 100644 --- a/arduino-core/src/cc/arduino/utils/network/FileDownloader.java +++ b/arduino-core/src/cc/arduino/utils/network/FileDownloader.java @@ -34,15 +34,18 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import processing.app.helpers.FileUtils; +import javax.script.ScriptException; import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.RandomAccessFile; import java.net.HttpURLConnection; import java.net.SocketTimeoutException; +import java.net.URISyntaxException; import java.net.URL; import java.nio.file.Files; import java.nio.file.Paths; +import java.util.Arrays; import java.util.Observable; import java.util.Optional; @@ -137,26 +140,49 @@ public class FileDownloader extends Observable { } } + public static void invalidateFiles(URL... filesUrl) { + // For each file delete the file cached if exist + Arrays.stream(filesUrl).forEach(url -> { + try { + FileDownloaderCache.getFileCached(url).ifPresent(fileCached -> { + try { + log.info("Invalidate this file {} that comes from {}", fileCached.getLocalPath(), fileCached.getRemoteURL()); + fileCached.invalidateCache(); + } catch (Exception e) { + log.warn("Fail to invalidate cache", e); + } + }); + } catch (URISyntaxException | NoSuchMethodException | ScriptException | IOException e) { + log.warn("Fail to get the file cached during the file invalidation", e); + } + }); + + } + private void downloadFile(boolean noResume) throws InterruptedException { try { setStatus(Status.CONNECTING); - final Optional fileCached = FileDownloaderCache.getFileCached(downloadUrl); - if (fileCached.isPresent() && fileCached.get().isNotChange()) { - final Optional fileFromCache = getFileCached(fileCached.get()); - if (fileFromCache.isPresent()) { + final Optional fileCachedOpt = FileDownloaderCache.getFileCached(downloadUrl); + if (fileCachedOpt.isPresent()) { + final FileDownloaderCache.FileCached fileCached = fileCachedOpt.get(); + + final Optional fileFromCache = getFileCached(fileCached); + if (fileCached.isNotChange() && fileFromCache.isPresent()) { // Copy the cached file in the destination file FileUtils.copyFile(fileFromCache.get(), outputFile); } else { openConnectionAndFillTheFile(noResume); if (allowCache) { - fileCached.get().updateCacheFile(outputFile); + fileCached.updateCacheFile(outputFile); } else { log.info("The file {} was not cached because allow cache is false", downloadUrl); } } + } else { + openConnectionAndFillTheFile(noResume); } setStatus(Status.COMPLETE); diff --git a/arduino-core/src/cc/arduino/utils/network/FileDownloaderCache.java b/arduino-core/src/cc/arduino/utils/network/FileDownloaderCache.java index 743de8d71..9c8648708 100644 --- a/arduino-core/src/cc/arduino/utils/network/FileDownloaderCache.java +++ b/arduino-core/src/cc/arduino/utils/network/FileDownloaderCache.java @@ -117,7 +117,7 @@ public class FileDownloaderCache { } } - public static Optional getFileCached(final URL remoteURL) + static Optional getFileCached(final URL remoteURL) throws URISyntaxException, NoSuchMethodException, ScriptException, IOException { // Return always and empty file if the cache is not enable @@ -268,7 +268,8 @@ public class FileDownloaderCache { @JsonIgnore public boolean isExpire() { // Check if the file is expire - return this.getExpiresTime().isBefore(LocalDateTime.now()); + final LocalDateTime now = LocalDateTime.now(); + return this.getExpiresTime().isBefore(now) || this.getExpiresTime().isEqual(now); } @JsonIgnore @@ -298,7 +299,7 @@ public class FileDownloaderCache { } @JsonIgnore - public Optional getFileFromCache() { + Optional getFileFromCache() { if (md5Check()) { return Optional.of(Paths.get(localPath).toFile()); } @@ -306,7 +307,7 @@ public class FileDownloaderCache { } - public synchronized void updateCacheFile(File fileToCache) throws Exception { + synchronized void updateCacheFile(File fileToCache) throws Exception { Path cacheFilePath = Paths.get(localPath); // If the cache directory does not exist create it @@ -336,6 +337,11 @@ public class FileDownloaderCache { } + synchronized void invalidateCache() throws IOException { + cachedFiles.remove(remoteURL); + Files.deleteIfExists(Paths.get(localPath)); + } + private String calculateMD5() throws IOException, NoSuchAlgorithmException { if (exists()) { return FileHash.hash(Paths.get(localPath).toFile(), "MD5"); @@ -344,7 +350,7 @@ public class FileDownloaderCache { } @JsonIgnore - public boolean md5Check() { + boolean md5Check() { try { return !Objects.isNull(getMD5()) && Objects.equals(calculateMD5(), getMD5()); } catch (Exception e) { @@ -354,7 +360,7 @@ public class FileDownloaderCache { } @JsonIgnore - public LocalDateTime getExpiresTime() { + LocalDateTime getExpiresTime() { final int maxAge; if (cacheControl != null) { maxAge = cacheControl.getMaxAge();