mirror of
https://github.com/arduino/Arduino.git
synced 2024-11-29 10:24:12 +01:00
Merge pull request #9023 from mattiabertorello/add-file-cache
Add file downloader cache to make faster the library/boards manager
This commit is contained in:
Martino Facchin
GitHub
commit
324a9bcbbb
@ -4,7 +4,6 @@
|
||||
<classpathentry kind="src" path="app/test"/>
|
||||
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
|
||||
<classpathentry kind="lib" path="app/lib/apple.jar"/>
|
||||
<classpathentry kind="lib" path="app/lib/ecj.jar"/>
|
||||
<classpathentry kind="lib" path="app/test-lib/junit-4.11.jar"/>
|
||||
<classpathentry kind="lib" path="app/test-lib/fest-assert-1.2.jar"/>
|
||||
<classpathentry kind="lib" path="app/test-lib/fest-reflect-1.2.jar"/>
|
||||
@ -16,10 +15,10 @@
|
||||
<classpathentry kind="lib" path="app/lib/commons-httpclient-3.1.jar"/>
|
||||
<classpathentry kind="lib" path="app/lib/commons-logging-1.0.4.jar"/>
|
||||
<classpathentry kind="lib" path="app/lib/commons-net-3.3.jar"/>
|
||||
<classpathentry kind="lib" path="app/lib/jmdns-3.5.1.jar"/>
|
||||
<classpathentry kind="lib" path="app/lib/jmdns-3.5.3.jar"/>
|
||||
<classpathentry kind="lib" path="app/lib/slf4j-api-1.7.22.jar"/>
|
||||
<classpathentry kind="lib" path="app/lib/jsch-0.1.50.jar"/>
|
||||
<classpathentry kind="lib" path="app/lib/jssc-2.8.0.jar"/>
|
||||
<classpathentry kind="lib" path="app/lib/jssc-2.8.0-arduino3.jar"/>
|
||||
<classpathentry kind="lib" path="app/lib/bcpg-jdk15on-152.jar"/>
|
||||
<classpathentry kind="lib" path="app/lib/bcprov-jdk15on-152.jar"/>
|
||||
<classpathentry kind="lib" path="app/lib/jackson-core-2.9.5.jar"/>
|
||||
|
||||
@ -39,6 +39,8 @@
|
||||
<classpathentry kind="lib" path="lib/jmdns-3.5.3.jar"/>
|
||||
<classpathentry kind="lib" path="lib/slf4j-api-1.7.22.jar"/>
|
||||
<classpathentry kind="lib" path="lib/slf4j-simple-1.7.22.jar"/>
|
||||
<classpathentry kind="lib" path="lib/log4j-api-2.12.0.jar"/>
|
||||
<classpathentry kind="lib" path="lib/log4j-core-2.12.0.jar"/>
|
||||
<classpathentry kind="lib" path="lib/jsch-0.1.50.jar"/>
|
||||
<classpathentry kind="lib" path="lib/jssc-2.8.0-arduino3.jar"/>
|
||||
<classpathentry kind="lib" path="lib/rsyntaxtextarea-3.0.3-SNAPSHOT.jar"/>
|
||||
|
||||
@ -80,6 +80,10 @@
|
||||
includeAntRuntime="false"
|
||||
debug="true"
|
||||
classpathref="class.path" />
|
||||
<!-- If you want to add files in the jars -->
|
||||
<copy todir="bin" overwrite="true" verbose="true">
|
||||
<fileset dir="src" includes="log4j2.xml" />
|
||||
</copy>
|
||||
</target>
|
||||
|
||||
<target name="test" depends="compile" description="Runs the test">
|
||||
|
||||
BIN
app/lib/commons-io-2.6.jar
Normal file
BIN
app/lib/commons-io-2.6.jar
Normal file
Binary file not shown.
BIN
app/lib/log4j-api-2.12.0.jar
Normal file
BIN
app/lib/log4j-api-2.12.0.jar
Normal file
Binary file not shown.
BIN
app/lib/log4j-core-2.12.0.jar
Normal file
BIN
app/lib/log4j-core-2.12.0.jar
Normal file
Binary file not shown.
29
app/src/log4j2.xml
Normal file
29
app/src/log4j2.xml
Normal file
@ -0,0 +1,29 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<Configuration status="warn" name="Arduino" packages="cc.arduino">
|
||||
<Appenders>
|
||||
|
||||
<!-- Console Appender -->
|
||||
<Console name="Console" target="SYSTEM_ERR">
|
||||
<PatternLayout pattern="%d{yyyy-MM-dd'T'HH:mm:ss.SSSXXX}{UTC} %p %c{1.} [%t] %m%n" />
|
||||
</Console>
|
||||
|
||||
<!-- Rolling File Appender -->
|
||||
<RollingFile name="RollingFile" fileName="${sys:log4j.saveDirectory}/logs/application.log"
|
||||
filePattern="${sys:log4j.saveDirectory}/logs/application-%d{MM-dd-yyyy}-%i.log.gz"
|
||||
ignoreExceptions="false">
|
||||
<PatternLayout>
|
||||
<Pattern>%d{yyyy-MM-dd'T'HH:mm:ss.SSSXXX}{UTC} %p %c{1.} [%t] %m%n</Pattern>
|
||||
</PatternLayout>
|
||||
<Policies>
|
||||
<SizeBasedTriggeringPolicy size="50 MB"/>
|
||||
</Policies>
|
||||
<DefaultRolloverStrategy max="20"/>
|
||||
</RollingFile>
|
||||
</Appenders>
|
||||
<Loggers>
|
||||
<Root level="debug">
|
||||
<AppenderRef ref="Console" level="info" />
|
||||
<AppenderRef ref="RollingFile"/>
|
||||
</Root>
|
||||
</Loggers>
|
||||
</Configuration>
|
||||
@ -26,9 +26,11 @@ import cc.arduino.Compiler;
|
||||
import cc.arduino.Constants;
|
||||
import cc.arduino.UpdatableBoardsLibsFakeURLsHandler;
|
||||
import cc.arduino.UploaderUtils;
|
||||
import cc.arduino.packages.Uploader;
|
||||
import cc.arduino.contributions.*;
|
||||
import cc.arduino.contributions.libraries.*;
|
||||
import cc.arduino.contributions.libraries.ContributedLibrary;
|
||||
import cc.arduino.contributions.libraries.LibrariesIndexer;
|
||||
import cc.arduino.contributions.libraries.LibraryInstaller;
|
||||
import cc.arduino.contributions.libraries.LibraryOfSameTypeComparator;
|
||||
import cc.arduino.contributions.libraries.ui.LibraryManagerUI;
|
||||
import cc.arduino.contributions.packages.ContributedPlatform;
|
||||
import cc.arduino.contributions.packages.ContributionInstaller;
|
||||
@ -36,20 +38,17 @@ import cc.arduino.contributions.packages.ContributionsIndexer;
|
||||
import cc.arduino.contributions.packages.ui.ContributionManagerUI;
|
||||
import cc.arduino.files.DeleteFilesOnShutdown;
|
||||
import cc.arduino.packages.DiscoveryManager;
|
||||
import cc.arduino.packages.Uploader;
|
||||
import cc.arduino.view.Event;
|
||||
import cc.arduino.view.JMenuUtils;
|
||||
import cc.arduino.view.SplashScreenHelper;
|
||||
|
||||
import com.github.zafarkhaja.semver.Version;
|
||||
import org.apache.commons.compress.utils.IOUtils;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
||||
import com.github.zafarkhaja.semver.Version;
|
||||
|
||||
import processing.app.debug.TargetBoard;
|
||||
import processing.app.debug.TargetPackage;
|
||||
import processing.app.debug.TargetPlatform;
|
||||
import processing.app.helpers.*;
|
||||
import processing.app.helpers.OSUtils;
|
||||
import processing.app.helpers.filefilters.OnlyDirs;
|
||||
import processing.app.helpers.filefilters.OnlyFilesWithExtension;
|
||||
import processing.app.javax.swing.filechooser.FileNameExtensionFilter;
|
||||
@ -67,9 +66,9 @@ import javax.swing.*;
|
||||
import java.awt.*;
|
||||
import java.awt.event.*;
|
||||
import java.io.*;
|
||||
import java.util.*;
|
||||
import java.util.List;
|
||||
import java.util.Timer;
|
||||
import java.util.*;
|
||||
import java.util.logging.Handler;
|
||||
import java.util.logging.Level;
|
||||
import java.util.logging.Logger;
|
||||
@ -208,6 +207,8 @@ public class Base {
|
||||
BaseNoGui.getPlatform().init();
|
||||
|
||||
BaseNoGui.initPortableFolder();
|
||||
// This configure the logs root folder
|
||||
System.setProperty("log4j.saveDirectory", BaseNoGui.getSettingsFolder().getAbsolutePath());
|
||||
|
||||
// Look for a possible "--preferences-file" parameter and load preferences
|
||||
BaseNoGui.initParameters(args);
|
||||
@ -286,8 +287,9 @@ public class Base {
|
||||
pdeKeywords = new PdeKeywords();
|
||||
pdeKeywords.reload();
|
||||
|
||||
contributionInstaller = new ContributionInstaller(BaseNoGui.getPlatform(), new GPGDetachedSignatureVerifier());
|
||||
libraryInstaller = new LibraryInstaller(BaseNoGui.getPlatform());
|
||||
final GPGDetachedSignatureVerifier gpgDetachedSignatureVerifier = new GPGDetachedSignatureVerifier();
|
||||
contributionInstaller = new ContributionInstaller(BaseNoGui.getPlatform(), gpgDetachedSignatureVerifier);
|
||||
libraryInstaller = new LibraryInstaller(BaseNoGui.getPlatform(), gpgDetachedSignatureVerifier);
|
||||
|
||||
parser.parseArgumentsPhase2();
|
||||
|
||||
@ -301,7 +303,7 @@ public class Base {
|
||||
if (parser.isInstallBoard()) {
|
||||
ContributionsIndexer indexer = new ContributionsIndexer(
|
||||
BaseNoGui.getSettingsFolder(), BaseNoGui.getHardwareFolder(),
|
||||
BaseNoGui.getPlatform(), new GPGDetachedSignatureVerifier());
|
||||
BaseNoGui.getPlatform(), gpgDetachedSignatureVerifier);
|
||||
ProgressListener progressListener = new ConsoleProgressListener();
|
||||
|
||||
List<String> downloadedPackageIndexFiles = contributionInstaller.updateIndex(progressListener);
|
||||
|
||||
@ -8,6 +8,8 @@
|
||||
<classpathentry kind="lib" path="lib/jmdns-3.5.3.jar"/>
|
||||
<classpathentry kind="lib" path="lib/slf4j-api-1.7.22.jar"/>
|
||||
<classpathentry kind="lib" path="lib/slf4j-simple-1.7.22.jar"/>
|
||||
<classpathentry kind="lib" path="lib/log4j-api-2.12.0.jar"/>
|
||||
<classpathentry kind="lib" path="lib/log4j-core-2.12.0.jar"/>
|
||||
<classpathentry kind="lib" path="lib/jssc-2.8.0-arduino3.jar"/>
|
||||
<classpathentry kind="lib" path="lib/jsch-0.1.50.jar"/>
|
||||
<classpathentry kind="lib" path="lib/commons-exec-1.1.jar"/>
|
||||
|
||||
BIN
arduino-core/lib/commons-io-2.6.jar
Normal file
BIN
arduino-core/lib/commons-io-2.6.jar
Normal file
Binary file not shown.
BIN
arduino-core/lib/log4j-api-2.12.0.jar
Normal file
BIN
arduino-core/lib/log4j-api-2.12.0.jar
Normal file
Binary file not shown.
BIN
arduino-core/lib/log4j-core-2.12.0.jar
Normal file
BIN
arduino-core/lib/log4j-core-2.12.0.jar
Normal file
Binary file not shown.
@ -30,20 +30,25 @@
|
||||
package cc.arduino.contributions;
|
||||
|
||||
import cc.arduino.utils.FileHash;
|
||||
import cc.arduino.utils.MultiStepProgress;
|
||||
import cc.arduino.utils.Progress;
|
||||
import cc.arduino.utils.network.FileDownloader;
|
||||
import org.apache.commons.io.FilenameUtils;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import processing.app.BaseNoGui;
|
||||
import processing.app.PreferencesData;
|
||||
|
||||
import java.io.File;
|
||||
import java.net.URL;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.LinkOption;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.nio.file.*;
|
||||
import java.util.Collection;
|
||||
|
||||
import static processing.app.I18n.format;
|
||||
import static processing.app.I18n.tr;
|
||||
|
||||
public class DownloadableContributionsDownloader {
|
||||
private static Logger log = LogManager.getLogger(DownloadableContributionsDownloader.class);
|
||||
|
||||
private final File stagingFolder;
|
||||
|
||||
@ -51,11 +56,11 @@ public class DownloadableContributionsDownloader {
|
||||
stagingFolder = _stagingFolder;
|
||||
}
|
||||
|
||||
public File download(DownloadableContribution contribution, Progress progress, final String statusText, ProgressListener progressListener) throws Exception {
|
||||
return download(contribution, progress, statusText, progressListener, false);
|
||||
public File download(DownloadableContribution contribution, Progress progress, final String statusText, ProgressListener progressListener, boolean allowCache) throws Exception {
|
||||
return download(contribution, progress, statusText, progressListener, false, allowCache);
|
||||
}
|
||||
|
||||
public File download(DownloadableContribution contribution, Progress progress, final String statusText, ProgressListener progressListener, boolean noResume) throws Exception {
|
||||
public File download(DownloadableContribution contribution, Progress progress, final String statusText, ProgressListener progressListener, boolean noResume, boolean allowCache) throws Exception {
|
||||
URL url = new URL(contribution.getUrl());
|
||||
Path outputFile = Paths.get(stagingFolder.getAbsolutePath(), contribution.getArchiveFileName());
|
||||
|
||||
@ -70,7 +75,7 @@ public class DownloadableContributionsDownloader {
|
||||
while (true) {
|
||||
// Need to download or resume downloading?
|
||||
if (!Files.isRegularFile(outputFile, LinkOption.NOFOLLOW_LINKS) || (Files.size(outputFile) < contribution.getSize())) {
|
||||
download(url, outputFile.toFile(), progress, statusText, progressListener, noResume);
|
||||
download(url, outputFile.toFile(), progress, statusText, progressListener, noResume, allowCache);
|
||||
downloaded = true;
|
||||
}
|
||||
|
||||
@ -116,12 +121,12 @@ public class DownloadableContributionsDownloader {
|
||||
return algo != null && !algo.isEmpty();
|
||||
}
|
||||
|
||||
public void download(URL url, File tmpFile, Progress progress, String statusText, ProgressListener progressListener) throws Exception {
|
||||
download(url, tmpFile, progress, statusText, progressListener, false);
|
||||
public void download(URL url, File tmpFile, Progress progress, String statusText, ProgressListener progressListener, boolean allowCache) throws Exception {
|
||||
download(url, tmpFile, progress, statusText, progressListener, false, allowCache);
|
||||
}
|
||||
|
||||
public void download(URL url, File tmpFile, Progress progress, String statusText, ProgressListener progressListener, boolean noResume) throws Exception {
|
||||
FileDownloader downloader = new FileDownloader(url, tmpFile);
|
||||
public void download(URL url, File tmpFile, Progress progress, String statusText, ProgressListener progressListener, boolean noResume, boolean allowCache) throws Exception {
|
||||
final FileDownloader downloader = new FileDownloader(url, tmpFile, allowCache);
|
||||
downloader.addObserver((o, arg) -> {
|
||||
FileDownloader me = (FileDownloader) o;
|
||||
String msg = "";
|
||||
@ -140,4 +145,94 @@ public class DownloadableContributionsDownloader {
|
||||
}
|
||||
}
|
||||
|
||||
public void downloadIndexAndSignature(MultiStepProgress progress, URL packageIndexUrl, ProgressListener progressListener, SignatureVerifier signatureVerifier) throws Exception {
|
||||
|
||||
// Extract the file name from the url
|
||||
final String indexFileName = FilenameUtils.getName(packageIndexUrl.getPath());
|
||||
final File packageIndex = BaseNoGui.indexer.getIndexFile(indexFileName);
|
||||
|
||||
final String statusText = tr("Downloading platforms index...");
|
||||
|
||||
// Create temp files
|
||||
final File packageIndexTemp = File.createTempFile(indexFileName, ".tmp");
|
||||
try {
|
||||
// Download package index
|
||||
download(packageIndexUrl, packageIndexTemp, progress, statusText, progressListener, true, true);
|
||||
final URL signatureUrl = new URL(packageIndexUrl.toString() + ".sig");
|
||||
|
||||
if (verifyDomain(packageIndexUrl)) {
|
||||
if (checkSignature(progress, signatureUrl, progressListener, signatureVerifier, statusText, packageIndexTemp)) {
|
||||
Files.move(packageIndexTemp.toPath(), packageIndex.toPath(), StandardCopyOption.REPLACE_EXISTING);
|
||||
} else {
|
||||
log.info("The cached files have been removed. {} {}", packageIndexUrl, signatureUrl);
|
||||
FileDownloader.invalidateFiles(packageIndexUrl, signatureUrl);
|
||||
}
|
||||
} else {
|
||||
// Move the package index to the destination when the signature is not necessary
|
||||
Files.move(packageIndexTemp.toPath(), packageIndex.toPath(), StandardCopyOption.REPLACE_EXISTING);
|
||||
log.info("The domain is not selected to verify the signature. will be copied into this path {}, packageIndex url: {}", packageIndex, packageIndexUrl);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("Cannot download the package index from {} the package will be discard", packageIndexUrl, e);
|
||||
throw e;
|
||||
} finally {
|
||||
// Delete useless temp file
|
||||
Files.deleteIfExists(packageIndexTemp.toPath());
|
||||
}
|
||||
}
|
||||
|
||||
public boolean verifyDomain(URL url) {
|
||||
final Collection<String> domain = PreferencesData.
|
||||
getCollection("http.signature_verify_domains");
|
||||
if (domain.size() == 0) {
|
||||
// Default domain
|
||||
domain.add("downloads.arduino.cc");
|
||||
}
|
||||
if (domain.contains(url.getHost())) {
|
||||
return true;
|
||||
} else {
|
||||
log.info("The domain is not selected to verify the signature. domain list: {}, url: {}", domain, url);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public boolean checkSignature(MultiStepProgress progress, URL signatureUrl, ProgressListener progressListener, SignatureVerifier signatureVerifier, String statusText, File fileToVerify) throws Exception {
|
||||
|
||||
final boolean allowInsecurePackages =
|
||||
PreferencesData.getBoolean("allow_insecure_packages", false);
|
||||
if (allowInsecurePackages) {
|
||||
log.info("Allow insecure packages is true the signature will be skip and return always verified");
|
||||
return true;
|
||||
}
|
||||
|
||||
// Signature file name
|
||||
final String signatureFileName = FilenameUtils.getName(signatureUrl.getPath());
|
||||
final File packageIndexSignature = BaseNoGui.indexer.getIndexFile(signatureFileName);
|
||||
final File packageIndexSignatureTemp = File.createTempFile(signatureFileName, ".tmp");
|
||||
|
||||
|
||||
try {
|
||||
// Download signature
|
||||
download(signatureUrl, packageIndexSignatureTemp, progress, statusText, progressListener, true);
|
||||
|
||||
// Verify the signature before move the files
|
||||
final boolean signatureVerified = signatureVerifier.isSigned(fileToVerify, packageIndexSignatureTemp);
|
||||
if (signatureVerified) {
|
||||
log.info("Signature verified. url={}, signature url={}, file to verify={}, signature file={}", signatureUrl, signatureUrl, fileToVerify, packageIndexSignatureTemp);
|
||||
// Move if the signature is ok
|
||||
Files.move(packageIndexSignatureTemp.toPath(), packageIndexSignature.toPath(), StandardCopyOption.REPLACE_EXISTING);
|
||||
} else {
|
||||
log.error("{} file signature verification failed. File ignored.", signatureUrl);
|
||||
System.err.println(format(tr("{0} file signature verification failed. File ignored."), signatureUrl.toString()));
|
||||
}
|
||||
return signatureVerified;
|
||||
} catch (Exception e) {
|
||||
log.error("Cannot download the signature from {} the package will be discard", signatureUrl, e);
|
||||
throw e;
|
||||
} finally {
|
||||
Files.deleteIfExists(packageIndexSignatureTemp.toPath());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -29,13 +29,16 @@
|
||||
|
||||
package cc.arduino.contributions;
|
||||
|
||||
import cc.arduino.Constants;
|
||||
import cc.arduino.utils.Progress;
|
||||
import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
|
||||
import org.apache.commons.compress.compressors.gzip.GzipUtils;
|
||||
import org.apache.commons.compress.utils.IOUtils;
|
||||
import org.apache.commons.io.FilenameUtils;
|
||||
|
||||
import java.io.*;
|
||||
import java.net.URL;
|
||||
import java.nio.file.Files;
|
||||
|
||||
public class GZippedJsonDownloader {
|
||||
|
||||
@ -49,18 +52,22 @@ public class GZippedJsonDownloader {
|
||||
this.gzippedUrl = gzippedUrl;
|
||||
}
|
||||
|
||||
public void download(File tmpFile, Progress progress, String statusText, ProgressListener progressListener) throws Exception {
|
||||
public void download(File tmpFile, Progress progress, String statusText, ProgressListener progressListener, boolean allowCache) throws Exception {
|
||||
File gzipTmpFile = null;
|
||||
try {
|
||||
File gzipTmpFile = new File(tmpFile.getParentFile(), GzipUtils.getCompressedFilename(tmpFile.getName()));
|
||||
String tmpFileName = FilenameUtils.getName(new URL(Constants.LIBRARY_INDEX_URL_GZ).getPath());
|
||||
gzipTmpFile = File.createTempFile(tmpFileName, GzipUtils.getCompressedFilename(tmpFile.getName()));
|
||||
// remove eventual leftovers from previous downloads
|
||||
if (gzipTmpFile.exists()) {
|
||||
gzipTmpFile.delete();
|
||||
}
|
||||
new JsonDownloader(downloader, gzippedUrl).download(gzipTmpFile, progress, statusText, progressListener);
|
||||
Files.deleteIfExists(gzipTmpFile.toPath());
|
||||
|
||||
new JsonDownloader(downloader, gzippedUrl).download(gzipTmpFile, progress, statusText, progressListener, allowCache);
|
||||
decompress(gzipTmpFile, tmpFile);
|
||||
gzipTmpFile.delete();
|
||||
} catch (Exception e) {
|
||||
new JsonDownloader(downloader, url).download(tmpFile, progress, statusText, progressListener);
|
||||
new JsonDownloader(downloader, url).download(tmpFile, progress, statusText, progressListener, allowCache);
|
||||
} finally {
|
||||
if (gzipTmpFile != null) {
|
||||
Files.deleteIfExists(gzipTmpFile.toPath());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -44,9 +44,9 @@ public class JsonDownloader {
|
||||
this.url = url;
|
||||
}
|
||||
|
||||
public void download(File tmpFile, Progress progress, String statusText, ProgressListener progressListener) throws Exception {
|
||||
public void download(File tmpFile, Progress progress, String statusText, ProgressListener progressListener, boolean allowCache) throws Exception {
|
||||
try {
|
||||
downloader.download(url, tmpFile, progress, statusText, progressListener);
|
||||
downloader.download(url, tmpFile, progress, statusText, progressListener, allowCache);
|
||||
} catch (InterruptedException e) {
|
||||
// Download interrupted... just exit
|
||||
}
|
||||
|
||||
@ -50,6 +50,15 @@ public abstract class SignatureVerifier {
|
||||
}
|
||||
}
|
||||
|
||||
public boolean isSigned(File indexFile, File signature) {
|
||||
try {
|
||||
return verify(indexFile, signature, new File(BaseNoGui.getContentFile("lib"), "public.gpg.key"));
|
||||
} catch (Exception e) {
|
||||
BaseNoGui.showWarning(e.getMessage(), e.getMessage(), e);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
protected abstract boolean verify(File signedFile, File signature, File publicKey) throws IOException;
|
||||
|
||||
}
|
||||
|
||||
@ -31,10 +31,15 @@ package cc.arduino.contributions.libraries;
|
||||
|
||||
import cc.arduino.Constants;
|
||||
import cc.arduino.contributions.DownloadableContributionsDownloader;
|
||||
import cc.arduino.contributions.GPGDetachedSignatureVerifier;
|
||||
import cc.arduino.contributions.GZippedJsonDownloader;
|
||||
import cc.arduino.contributions.ProgressListener;
|
||||
import cc.arduino.utils.ArchiveExtractor;
|
||||
import cc.arduino.utils.MultiStepProgress;
|
||||
import cc.arduino.utils.network.FileDownloader;
|
||||
import org.apache.commons.io.FilenameUtils;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import processing.app.BaseNoGui;
|
||||
import processing.app.I18n;
|
||||
import processing.app.Platform;
|
||||
@ -43,6 +48,8 @@ import processing.app.helpers.FileUtils;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.net.URL;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.StandardCopyOption;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
@ -50,11 +57,14 @@ import java.util.Optional;
|
||||
import static processing.app.I18n.tr;
|
||||
|
||||
public class LibraryInstaller {
|
||||
private static Logger log = LogManager.getLogger(LibraryInstaller.class);
|
||||
|
||||
private final Platform platform;
|
||||
private final GPGDetachedSignatureVerifier signatureVerifier;
|
||||
|
||||
public LibraryInstaller(Platform platform) {
|
||||
public LibraryInstaller(Platform platform, GPGDetachedSignatureVerifier signatureVerifier) {
|
||||
this.platform = platform;
|
||||
this.signatureVerifier = signatureVerifier;
|
||||
}
|
||||
|
||||
public synchronized void updateIndex(ProgressListener progressListener) throws Exception {
|
||||
@ -63,29 +73,42 @@ public class LibraryInstaller {
|
||||
DownloadableContributionsDownloader downloader = new DownloadableContributionsDownloader(BaseNoGui.librariesIndexer.getStagingFolder());
|
||||
// Step 1: Download index
|
||||
File outputFile = BaseNoGui.librariesIndexer.getIndexFile();
|
||||
File tmpFile = new File(outputFile.getAbsolutePath() + ".tmp");
|
||||
// Create temp files
|
||||
String signatureFileName = FilenameUtils.getName(new URL(Constants.LIBRARY_INDEX_URL).getPath());
|
||||
File libraryIndexTemp = File.createTempFile(signatureFileName, ".tmp");
|
||||
final URL libraryURL = new URL(Constants.LIBRARY_INDEX_URL);
|
||||
final URL libraryGzURL = new URL(Constants.LIBRARY_INDEX_URL_GZ);
|
||||
final String statusText = tr("Downloading libraries index...");
|
||||
try {
|
||||
GZippedJsonDownloader gZippedJsonDownloader = new GZippedJsonDownloader(downloader, new URL(Constants.LIBRARY_INDEX_URL), new URL(Constants.LIBRARY_INDEX_URL_GZ));
|
||||
gZippedJsonDownloader.download(tmpFile, progress, tr("Downloading libraries index..."), progressListener);
|
||||
GZippedJsonDownloader gZippedJsonDownloader = new GZippedJsonDownloader(downloader, libraryURL, libraryGzURL);
|
||||
gZippedJsonDownloader.download(libraryIndexTemp, progress, statusText, progressListener, true);
|
||||
} catch (InterruptedException e) {
|
||||
// Download interrupted... just exit
|
||||
return;
|
||||
}
|
||||
progress.stepDone();
|
||||
|
||||
// TODO: Check downloaded index
|
||||
|
||||
// Replace old index with the updated one
|
||||
if (outputFile.exists())
|
||||
outputFile.delete();
|
||||
if (!tmpFile.renameTo(outputFile))
|
||||
throw new Exception(tr("An error occurred while updating libraries index!"));
|
||||
URL signatureUrl = new URL(libraryURL.toString() + ".sig");
|
||||
if (downloader.verifyDomain(signatureUrl)) {
|
||||
if (downloader.checkSignature(progress, signatureUrl, progressListener, signatureVerifier, statusText, libraryIndexTemp)) {
|
||||
// Replace old index with the updated one
|
||||
if (libraryIndexTemp.length() > 0) {
|
||||
Files.move(libraryIndexTemp.toPath(), outputFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
|
||||
}
|
||||
} else {
|
||||
FileDownloader.invalidateFiles(libraryGzURL, libraryURL, signatureUrl);
|
||||
log.error("Fail to verify the signature of {} the cached files have been removed", libraryURL);
|
||||
}
|
||||
} else {
|
||||
log.info("The domain is not selected to verify the signature. library index: {}", signatureUrl);
|
||||
}
|
||||
|
||||
// Step 2: Parse index
|
||||
BaseNoGui.librariesIndexer.parseIndex();
|
||||
|
||||
// Step 3: Rescan index
|
||||
rescanLibraryIndex(progress, progressListener);
|
||||
|
||||
}
|
||||
|
||||
public void install(ContributedLibrary lib, ProgressListener progressListener) throws Exception {
|
||||
@ -129,7 +152,7 @@ public class LibraryInstaller {
|
||||
|
||||
// Step 1: Download library
|
||||
try {
|
||||
downloader.download(lib, progress, I18n.format(tr("Downloading library: {0}"), lib.getName()), progressListener);
|
||||
downloader.download(lib, progress, I18n.format(tr("Downloading library: {0}"), lib.getName()), progressListener, false);
|
||||
} catch (InterruptedException e) {
|
||||
// Download interrupted... just exit
|
||||
return;
|
||||
|
||||
@ -41,6 +41,9 @@ import org.apache.commons.exec.CommandLine;
|
||||
import org.apache.commons.exec.DefaultExecutor;
|
||||
import org.apache.commons.exec.Executor;
|
||||
import org.apache.commons.exec.PumpStreamHandler;
|
||||
import org.apache.commons.io.FilenameUtils;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import processing.app.BaseNoGui;
|
||||
import processing.app.I18n;
|
||||
import processing.app.Platform;
|
||||
@ -62,6 +65,7 @@ import static processing.app.I18n.format;
|
||||
import static processing.app.I18n.tr;
|
||||
|
||||
public class ContributionInstaller {
|
||||
private static Logger log = LogManager.getLogger(ContributionInstaller.class);
|
||||
|
||||
private final Platform platform;
|
||||
private final SignatureVerifier signatureVerifier;
|
||||
@ -98,7 +102,7 @@ public class ContributionInstaller {
|
||||
// Download all
|
||||
try {
|
||||
// Download platform
|
||||
downloader.download(contributedPlatform, progress, tr("Downloading boards definitions."), progressListener);
|
||||
downloader.download(contributedPlatform, progress, tr("Downloading boards definitions."), progressListener, false);
|
||||
progress.stepDone();
|
||||
|
||||
// Download tools
|
||||
@ -106,7 +110,7 @@ public class ContributionInstaller {
|
||||
for (ContributedTool tool : tools) {
|
||||
String msg = format(tr("Downloading tools ({0}/{1})."), i, tools.size());
|
||||
i++;
|
||||
downloader.download(tool.getDownloadableContribution(platform), progress, msg, progressListener);
|
||||
downloader.download(tool.getDownloadableContribution(platform), progress, msg, progressListener, false);
|
||||
progress.stepDone();
|
||||
}
|
||||
} catch (InterruptedException e) {
|
||||
@ -122,10 +126,10 @@ public class ContributionInstaller {
|
||||
// all the temporary folders and abort installation.
|
||||
|
||||
List<Map.Entry<ContributedToolReference, ContributedTool>> resolvedToolReferences = contributedPlatform
|
||||
.getResolvedToolReferences().entrySet().stream()
|
||||
.filter((entry) -> !entry.getValue().isInstalled()
|
||||
|| entry.getValue().isBuiltIn())
|
||||
.collect(Collectors.toList());
|
||||
.getResolvedToolReferences().entrySet().stream()
|
||||
.filter((entry) -> !entry.getValue().isInstalled()
|
||||
|| entry.getValue().isBuiltIn())
|
||||
.collect(Collectors.toList());
|
||||
|
||||
int i = 1;
|
||||
for (Map.Entry<ContributedToolReference, ContributedTool> entry : resolvedToolReferences) {
|
||||
@ -268,6 +272,8 @@ public class ContributionInstaller {
|
||||
Files.delete(destFolder.getParentFile().toPath());
|
||||
} catch (Exception e) {
|
||||
// ignore
|
||||
log.info("The directory is not empty there is another version installed. directory {}",
|
||||
destFolder.getParentFile().toPath(), e);
|
||||
}
|
||||
}
|
||||
|
||||
@ -278,74 +284,48 @@ public class ContributionInstaller {
|
||||
return errors;
|
||||
}
|
||||
|
||||
public synchronized List<String> updateIndex(ProgressListener progressListener) throws Exception {
|
||||
public synchronized List<String> updateIndex(ProgressListener progressListener) {
|
||||
MultiStepProgress progress = new MultiStepProgress(1);
|
||||
|
||||
final DownloadableContributionsDownloader downloader = new DownloadableContributionsDownloader(BaseNoGui.indexer.getStagingFolder());
|
||||
|
||||
final Set<String> packageIndexURLs = new HashSet<>(
|
||||
PreferencesData.getCollection(Constants.PREF_BOARDS_MANAGER_ADDITIONAL_URLS)
|
||||
);
|
||||
packageIndexURLs.add(Constants.PACKAGE_INDEX_URL);
|
||||
List<String> downloadedPackageIndexFilesAccumulator = new LinkedList<>();
|
||||
downloadIndexAndSignature(progress, downloadedPackageIndexFilesAccumulator, Constants.PACKAGE_INDEX_URL, progressListener);
|
||||
|
||||
Set<String> packageIndexURLs = new HashSet<>();
|
||||
String additionalURLs = PreferencesData.get(Constants.PREF_BOARDS_MANAGER_ADDITIONAL_URLS, "");
|
||||
if (!"".equals(additionalURLs)) {
|
||||
packageIndexURLs.addAll(Arrays.asList(additionalURLs.split(",")));
|
||||
}
|
||||
|
||||
for (String packageIndexURL : packageIndexURLs) {
|
||||
for (String packageIndexURLString : packageIndexURLs) {
|
||||
try {
|
||||
downloadIndexAndSignature(progress, downloadedPackageIndexFilesAccumulator, packageIndexURL, progressListener);
|
||||
// Extract the file name from the URL
|
||||
final URL packageIndexURL = new URL(packageIndexURLString);
|
||||
String indexFileName = FilenameUtils.getName(packageIndexURL.getPath());
|
||||
downloadedPackageIndexFilesAccumulator.add(BaseNoGui.indexer.getIndexFile(indexFileName).getName());
|
||||
|
||||
log.info("Start download and signature check of={}", packageIndexURLs);
|
||||
downloader.downloadIndexAndSignature(progress, packageIndexURL, progressListener, signatureVerifier);
|
||||
} catch (Exception e) {
|
||||
log.error(e.getMessage(), e);
|
||||
System.err.println(e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
progress.stepDone();
|
||||
|
||||
log.info("Downloaded package index URL={}", packageIndexURLs);
|
||||
return downloadedPackageIndexFilesAccumulator;
|
||||
}
|
||||
|
||||
private void downloadIndexAndSignature(MultiStepProgress progress, List<String> downloadedPackagedIndexFilesAccumulator, String packageIndexUrl, ProgressListener progressListener) throws Exception {
|
||||
File packageIndex = download(progress, packageIndexUrl, progressListener);
|
||||
downloadedPackagedIndexFilesAccumulator.add(packageIndex.getName());
|
||||
try {
|
||||
File packageIndexSignature = download(progress, packageIndexUrl + ".sig", progressListener);
|
||||
boolean signatureVerified = signatureVerifier.isSigned(packageIndex);
|
||||
if (signatureVerified) {
|
||||
downloadedPackagedIndexFilesAccumulator.add(packageIndexSignature.getName());
|
||||
} else {
|
||||
downloadedPackagedIndexFilesAccumulator.remove(packageIndex.getName());
|
||||
Files.delete(packageIndex.toPath());
|
||||
Files.delete(packageIndexSignature.toPath());
|
||||
System.err.println(I18n.format(tr("{0} file signature verification failed. File ignored."), packageIndexUrl));
|
||||
}
|
||||
} catch (Exception e) {
|
||||
//ignore errors
|
||||
}
|
||||
}
|
||||
|
||||
private File download(MultiStepProgress progress, String packageIndexUrl, ProgressListener progressListener) throws Exception {
|
||||
String statusText = tr("Downloading platforms index...");
|
||||
URL url = new URL(packageIndexUrl);
|
||||
String[] urlPathParts = url.getFile().split("/");
|
||||
File outputFile = BaseNoGui.indexer.getIndexFile(urlPathParts[urlPathParts.length - 1]);
|
||||
File tmpFile = new File(outputFile.getAbsolutePath() + ".tmp");
|
||||
DownloadableContributionsDownloader downloader = new DownloadableContributionsDownloader(BaseNoGui.indexer.getStagingFolder());
|
||||
boolean noResume = true;
|
||||
downloader.download(url, tmpFile, progress, statusText, progressListener, noResume);
|
||||
|
||||
Files.deleteIfExists(outputFile.toPath());
|
||||
Files.move(tmpFile.toPath(), outputFile.toPath());
|
||||
|
||||
return outputFile;
|
||||
}
|
||||
|
||||
public synchronized void deleteUnknownFiles(List<String> downloadedPackageIndexFiles) throws IOException {
|
||||
File preferencesFolder = BaseNoGui.indexer.getIndexFile(".").getParentFile();
|
||||
File[] additionalPackageIndexFiles = preferencesFolder.listFiles(new PackageIndexFilenameFilter(Constants.DEFAULT_INDEX_FILE_NAME));
|
||||
if (additionalPackageIndexFiles == null) {
|
||||
return;
|
||||
}
|
||||
log.info("Check unknown files. Additional package index folder files={}, Additional package index url downloaded={}", downloadedPackageIndexFiles, additionalPackageIndexFiles);
|
||||
|
||||
for (File additionalPackageIndexFile : additionalPackageIndexFiles) {
|
||||
if (!downloadedPackageIndexFiles.contains(additionalPackageIndexFile.getName())) {
|
||||
log.info("Delete this unknown file={} because not included in this list={}", additionalPackageIndexFile, additionalPackageIndexFiles);
|
||||
Files.delete(additionalPackageIndexFile.toPath());
|
||||
}
|
||||
}
|
||||
|
||||
119
arduino-core/src/cc/arduino/utils/network/CacheControl.java
Normal file
119
arduino-core/src/cc/arduino/utils/network/CacheControl.java
Normal file
@ -0,0 +1,119 @@
|
||||
/*
|
||||
* This file is part of Arduino.
|
||||
*
|
||||
* Copyright 2019 Arduino LLC (http://www.arduino.cc/)
|
||||
*
|
||||
* Arduino is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation; either version 2 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
*
|
||||
* As a special exception, you may use this file as part of a free software
|
||||
* library without restriction. Specifically, if other files instantiate
|
||||
* templates or use macros or inline functions from this file, or you compile
|
||||
* this file and link it with other files to produce an executable, this
|
||||
* file does not by itself cause the resulting executable to be covered by
|
||||
* the GNU General Public License. This exception does not however
|
||||
* invalidate any other reasons why the executable file might be covered by
|
||||
* the GNU General Public License.
|
||||
*/
|
||||
|
||||
|
||||
package cc.arduino.utils.network;
|
||||
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
public class CacheControl {
|
||||
|
||||
|
||||
// see org.apache.abdera.protocol.util.CacheControlUtil
|
||||
private static final Pattern PATTERN
|
||||
= Pattern.compile("\\s*([\\w\\-]+)\\s*(=)?\\s*(\\-?\\d+|\\\"([^\"\\\\]*(\\\\.[^\"\\\\]*)*)+\\\")?\\s*");
|
||||
|
||||
private int maxAge = -1;
|
||||
|
||||
private boolean isMustRevalidate = false;
|
||||
|
||||
private boolean isNoCache = false;
|
||||
|
||||
private boolean isNoStore = false;
|
||||
|
||||
|
||||
public static CacheControl valueOf(String value) {
|
||||
CacheControl cc = new CacheControl();
|
||||
|
||||
if (value != null) {
|
||||
Matcher matcher = PATTERN.matcher(value);
|
||||
while (matcher.find()) {
|
||||
switch (matcher.group(1).toLowerCase()) {
|
||||
case "max-age":
|
||||
cc.setMaxAge(Integer.parseInt(matcher.group(3)));
|
||||
break;
|
||||
case "must-revalidate":
|
||||
cc.setMustRevalidate(true);
|
||||
break;
|
||||
case "no-cache":
|
||||
cc.setNoCache(true);
|
||||
break;
|
||||
case "no-store":
|
||||
cc.setNoStore(true);
|
||||
break;
|
||||
default: //ignore
|
||||
}
|
||||
}
|
||||
}
|
||||
return cc;
|
||||
}
|
||||
|
||||
public void setMaxAge(int maxAge) {
|
||||
this.maxAge = maxAge;
|
||||
}
|
||||
|
||||
public int getMaxAge() {
|
||||
return maxAge;
|
||||
}
|
||||
|
||||
public boolean isMustRevalidate() {
|
||||
return isMustRevalidate;
|
||||
}
|
||||
|
||||
public void setMustRevalidate(boolean mustRevalidate) {
|
||||
isMustRevalidate = mustRevalidate;
|
||||
}
|
||||
|
||||
public boolean isNoCache() {
|
||||
return isNoCache;
|
||||
}
|
||||
|
||||
public void setNoCache(boolean noCache) {
|
||||
isNoCache = noCache;
|
||||
}
|
||||
|
||||
public boolean isNoStore() {
|
||||
return isNoStore;
|
||||
}
|
||||
|
||||
public void setNoStore(boolean noStore) {
|
||||
isNoStore = noStore;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "CacheControl{" +
|
||||
"maxAge=" + maxAge +
|
||||
", isMustRevalidate=" + isMustRevalidate +
|
||||
", isNoCache=" + isNoCache +
|
||||
", isNoStore=" + isNoStore +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
@ -29,26 +29,28 @@
|
||||
|
||||
package cc.arduino.utils.network;
|
||||
|
||||
import cc.arduino.net.CustomProxySelector;
|
||||
import org.apache.commons.codec.binary.Base64;
|
||||
import org.apache.commons.compress.utils.IOUtils;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import processing.app.helpers.FileUtils;
|
||||
|
||||
import processing.app.BaseNoGui;
|
||||
import processing.app.PreferencesData;
|
||||
|
||||
import javax.script.ScriptException;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.RandomAccessFile;
|
||||
import java.net.HttpURLConnection;
|
||||
import java.net.Proxy;
|
||||
import java.net.SocketTimeoutException;
|
||||
import java.net.URISyntaxException;
|
||||
import java.net.URL;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.Arrays;
|
||||
import java.util.Observable;
|
||||
import java.util.Optional;
|
||||
|
||||
public class FileDownloader extends Observable {
|
||||
private static Logger log = LogManager.getLogger(FileDownloader.class);
|
||||
|
||||
public enum Status {
|
||||
CONNECTING, //
|
||||
@ -66,17 +68,15 @@ public class FileDownloader extends Observable {
|
||||
private final URL downloadUrl;
|
||||
|
||||
private final File outputFile;
|
||||
private InputStream stream = null;
|
||||
private final boolean allowCache;
|
||||
private Exception error;
|
||||
private String userAgent;
|
||||
|
||||
public FileDownloader(URL url, File file) {
|
||||
downloadUrl = url;
|
||||
outputFile = file;
|
||||
downloaded = 0;
|
||||
initialSize = 0;
|
||||
userAgent = "ArduinoIDE/" + BaseNoGui.VERSION_NAME + " Java/"
|
||||
+ System.getProperty("java.version");
|
||||
public FileDownloader(URL url, File file, boolean allowCache) {
|
||||
this.downloadUrl = url;
|
||||
this.outputFile = file;
|
||||
this.allowCache = allowCache;
|
||||
this.downloaded = 0;
|
||||
this.initialSize = 0;
|
||||
}
|
||||
|
||||
public long getInitialSize() {
|
||||
@ -121,9 +121,6 @@ public class FileDownloader extends Observable {
|
||||
notifyObservers();
|
||||
}
|
||||
|
||||
public void download() throws InterruptedException {
|
||||
download(false);
|
||||
}
|
||||
|
||||
public void download(boolean noResume) throws InterruptedException {
|
||||
if ("file".equals(downloadUrl.getProtocol())) {
|
||||
@ -143,98 +140,48 @@ public class FileDownloader extends Observable {
|
||||
}
|
||||
}
|
||||
|
||||
public static void invalidateFiles(URL... filesUrl) {
|
||||
// For each file delete the file cached if exist
|
||||
Arrays.stream(filesUrl).forEach(url -> {
|
||||
try {
|
||||
FileDownloaderCache.getFileCached(url).ifPresent(fileCached -> {
|
||||
try {
|
||||
log.info("Invalidate this file {} that comes from {}", fileCached.getLocalPath(), fileCached.getRemoteURL());
|
||||
fileCached.invalidateCache();
|
||||
} catch (Exception e) {
|
||||
log.warn("Fail to invalidate cache", e);
|
||||
}
|
||||
});
|
||||
} catch (URISyntaxException | NoSuchMethodException | ScriptException | IOException e) {
|
||||
log.warn("Fail to get the file cached during the file invalidation", e);
|
||||
}
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
private void downloadFile(boolean noResume) throws InterruptedException {
|
||||
RandomAccessFile file = null;
|
||||
|
||||
try {
|
||||
// Open file and seek to the end of it
|
||||
file = new RandomAccessFile(outputFile, "rw");
|
||||
initialSize = file.length();
|
||||
|
||||
if (noResume && initialSize > 0) {
|
||||
// delete file and restart downloading
|
||||
Files.delete(outputFile.toPath());
|
||||
initialSize = 0;
|
||||
}
|
||||
|
||||
file.seek(initialSize);
|
||||
|
||||
setStatus(Status.CONNECTING);
|
||||
|
||||
Proxy proxy = new CustomProxySelector(PreferencesData.getMap()).getProxyFor(downloadUrl.toURI());
|
||||
if ("true".equals(System.getProperty("DEBUG"))) {
|
||||
System.err.println("Using proxy " + proxy);
|
||||
}
|
||||
final Optional<FileDownloaderCache.FileCached> fileCachedOpt = FileDownloaderCache.getFileCached(downloadUrl, allowCache);
|
||||
if (fileCachedOpt.isPresent()) {
|
||||
final FileDownloaderCache.FileCached fileCached = fileCachedOpt.get();
|
||||
|
||||
HttpURLConnection connection = (HttpURLConnection) downloadUrl.openConnection(proxy);
|
||||
connection.setRequestProperty("User-agent", userAgent);
|
||||
if (downloadUrl.getUserInfo() != null) {
|
||||
String auth = "Basic " + new String(new Base64().encode(downloadUrl.getUserInfo().getBytes()));
|
||||
connection.setRequestProperty("Authorization", auth);
|
||||
}
|
||||
final Optional<File> fileFromCache = getFileCached(fileCached);
|
||||
if (fileCached.isNotChange() && fileFromCache.isPresent()) {
|
||||
// Copy the cached file in the destination file
|
||||
FileUtils.copyFile(fileFromCache.get(), outputFile);
|
||||
} else {
|
||||
openConnectionAndFillTheFile(noResume);
|
||||
|
||||
connection.setRequestProperty("Range", "bytes=" + initialSize + "-");
|
||||
connection.setConnectTimeout(5000);
|
||||
setDownloaded(0);
|
||||
|
||||
// Connect
|
||||
connection.connect();
|
||||
int resp = connection.getResponseCode();
|
||||
|
||||
if (resp == HttpURLConnection.HTTP_MOVED_PERM || resp == HttpURLConnection.HTTP_MOVED_TEMP) {
|
||||
URL newUrl = new URL(connection.getHeaderField("Location"));
|
||||
|
||||
proxy = new CustomProxySelector(PreferencesData.getMap()).getProxyFor(newUrl.toURI());
|
||||
|
||||
// open the new connnection again
|
||||
connection = (HttpURLConnection) newUrl.openConnection(proxy);
|
||||
connection.setRequestProperty("User-agent", userAgent);
|
||||
if (downloadUrl.getUserInfo() != null) {
|
||||
String auth = "Basic " + new String(new Base64().encode(downloadUrl.getUserInfo().getBytes()));
|
||||
connection.setRequestProperty("Authorization", auth);
|
||||
fileCached.updateCacheFile(outputFile);
|
||||
}
|
||||
|
||||
connection.setRequestProperty("Range", "bytes=" + initialSize + "-");
|
||||
connection.setConnectTimeout(5000);
|
||||
|
||||
connection.connect();
|
||||
resp = connection.getResponseCode();
|
||||
}
|
||||
|
||||
if (resp < 200 || resp >= 300) {
|
||||
throw new IOException("Received invalid http status code from server: " + resp);
|
||||
}
|
||||
|
||||
// Check for valid content length.
|
||||
long len = connection.getContentLength();
|
||||
if (len >= 0) {
|
||||
setDownloadSize(len);
|
||||
}
|
||||
setStatus(Status.DOWNLOADING);
|
||||
|
||||
synchronized (this) {
|
||||
stream = connection.getInputStream();
|
||||
}
|
||||
byte buffer[] = new byte[10240];
|
||||
while (status == Status.DOWNLOADING) {
|
||||
int read = stream.read(buffer);
|
||||
if (read == -1)
|
||||
break;
|
||||
|
||||
file.write(buffer, 0, read);
|
||||
setDownloaded(getDownloaded() + read);
|
||||
|
||||
if (Thread.interrupted()) {
|
||||
file.close();
|
||||
throw new InterruptedException();
|
||||
}
|
||||
}
|
||||
|
||||
if (getDownloadSize() != null) {
|
||||
if (getDownloaded() < getDownloadSize())
|
||||
throw new Exception("Incomplete download");
|
||||
} else {
|
||||
openConnectionAndFillTheFile(noResume);
|
||||
}
|
||||
setStatus(Status.COMPLETE);
|
||||
|
||||
} catch (InterruptedException e) {
|
||||
setStatus(Status.CANCELLED);
|
||||
// lets InterruptedException go up to the caller
|
||||
@ -243,17 +190,99 @@ public class FileDownloader extends Observable {
|
||||
} catch (SocketTimeoutException e) {
|
||||
setStatus(Status.CONNECTION_TIMEOUT_ERROR);
|
||||
setError(e);
|
||||
log.error("The request went in socket timeout", e);
|
||||
|
||||
} catch (Exception e) {
|
||||
setStatus(Status.ERROR);
|
||||
setError(e);
|
||||
log.error("The request stop", e);
|
||||
}
|
||||
|
||||
} finally {
|
||||
IOUtils.closeQuietly(file);
|
||||
}
|
||||
|
||||
synchronized (this) {
|
||||
IOUtils.closeQuietly(stream);
|
||||
private Optional<File> getFileCached(FileDownloaderCache.FileCached fileCached) {
|
||||
|
||||
try {
|
||||
final Optional<File> fileFromCache =
|
||||
fileCached.getFileFromCache();
|
||||
if (fileFromCache.isPresent()) {
|
||||
log.info("No need to download using cached file: {}", fileCached);
|
||||
return fileFromCache;
|
||||
} else {
|
||||
log.info(
|
||||
"The file in the cache is not in the path or the md5 validation failed: path={}, file exist={}, md5 validation={}",
|
||||
fileCached.getLocalPath(), fileCached.exists(), fileCached.md5Check());
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.warn(
|
||||
"Cannot get the file from the cache, will be downloaded a new one ", e);
|
||||
}
|
||||
log.info("The file is change {}", fileCached);
|
||||
return Optional.empty();
|
||||
}
|
||||
|
||||
private void openConnectionAndFillTheFile(boolean noResume) throws Exception {
|
||||
initialSize = outputFile.length();
|
||||
if (noResume && initialSize > 0) {
|
||||
// delete file and restart downloading
|
||||
Files.deleteIfExists(outputFile.toPath());
|
||||
initialSize = 0;
|
||||
}
|
||||
|
||||
final HttpURLConnection connection = new HttpConnectionManager(downloadUrl)
|
||||
.makeConnection((c) -> setDownloaded(0));
|
||||
final int resp = connection.getResponseCode();
|
||||
|
||||
if (resp < 200 || resp >= 300) {
|
||||
Files.deleteIfExists(outputFile.toPath());
|
||||
throw new IOException("Received invalid http status code from server: " + resp);
|
||||
}
|
||||
|
||||
RandomAccessFile randomAccessOutputFile = null;
|
||||
try {
|
||||
// Open file and seek to the end of it
|
||||
randomAccessOutputFile = new RandomAccessFile(outputFile, "rw");
|
||||
randomAccessOutputFile.seek(initialSize);
|
||||
readStreamCopyTo(randomAccessOutputFile, connection);
|
||||
} finally {
|
||||
IOUtils.closeQuietly(randomAccessOutputFile);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private void readStreamCopyTo(RandomAccessFile randomAccessOutputFile, HttpURLConnection connection) throws Exception {
|
||||
InputStream stream = null;
|
||||
try {
|
||||
// Check for valid content length.
|
||||
long len = connection.getContentLength();
|
||||
if (len >= 0) {
|
||||
setDownloadSize(len);
|
||||
}
|
||||
setStatus(Status.DOWNLOADING);
|
||||
|
||||
stream = connection.getInputStream();
|
||||
|
||||
byte[] buffer = new byte[10240];
|
||||
while (status == Status.DOWNLOADING) {
|
||||
int read = stream.read(buffer);
|
||||
if (read == -1)
|
||||
break;
|
||||
|
||||
randomAccessOutputFile.write(buffer, 0, read);
|
||||
setDownloaded(getDownloaded() + read);
|
||||
|
||||
if (Thread.interrupted()) {
|
||||
randomAccessOutputFile.close();
|
||||
throw new InterruptedException();
|
||||
}
|
||||
}
|
||||
|
||||
if (getDownloadSize() != null) {
|
||||
if (getDownloaded() < getDownloadSize())
|
||||
throw new Exception("Incomplete download");
|
||||
}
|
||||
} finally {
|
||||
IOUtils.closeQuietly(stream);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -0,0 +1,425 @@
|
||||
/*
|
||||
* This file is part of Arduino.
|
||||
*
|
||||
* Copyright 2019 Arduino LLC (http://www.arduino.cc/)
|
||||
*
|
||||
* Arduino is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation; either version 2 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
*
|
||||
* As a special exception, you may use this file as part of a free software
|
||||
* library without restriction. Specifically, if other files instantiate
|
||||
* templates or use macros or inline functions from this file, or you compile
|
||||
* this file and link it with other files to produce an executable, this
|
||||
* file does not by itself cause the resulting executable to be covered by
|
||||
* the GNU General Public License. This exception does not however
|
||||
* invalidate any other reasons why the executable file might be covered by
|
||||
* the GNU General Public License.
|
||||
*/
|
||||
|
||||
|
||||
package cc.arduino.utils.network;
|
||||
|
||||
import cc.arduino.utils.FileHash;
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.databind.SerializationFeature;
|
||||
import com.fasterxml.jackson.databind.node.ObjectNode;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import processing.app.BaseNoGui;
|
||||
import processing.app.PreferencesData;
|
||||
import processing.app.helpers.FileUtils;
|
||||
|
||||
import javax.script.ScriptException;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.net.HttpURLConnection;
|
||||
import java.net.ProtocolException;
|
||||
import java.net.URISyntaxException;
|
||||
import java.net.URL;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.security.NoSuchAlgorithmException;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.*;
|
||||
import java.util.function.BiFunction;
|
||||
import java.util.function.Function;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
public class FileDownloaderCache {
|
||||
private final static String CACHE_ENABLE_PREFERENCE_KEY = "cache.enable";
|
||||
private final static Logger log = LogManager
|
||||
.getLogger(FileDownloaderCache.class);
|
||||
private final static Map<String, FileCached> cachedFiles = Collections
|
||||
.synchronizedMap(new HashMap<>());
|
||||
private final static String cacheFolder;
|
||||
private static boolean enableCache;
|
||||
|
||||
static {
|
||||
enableCache = Boolean.valueOf(PreferencesData.get(CACHE_ENABLE_PREFERENCE_KEY, "true"));
|
||||
if (!enableCache) {
|
||||
log.info("The cache is disable cache.enable=false");
|
||||
}
|
||||
PreferencesData.set(CACHE_ENABLE_PREFERENCE_KEY, Boolean.toString(enableCache));
|
||||
|
||||
final File settingsFolder;
|
||||
settingsFolder = BaseNoGui.getSettingsFolder();
|
||||
if (settingsFolder != null) {
|
||||
cacheFolder = Paths.get(settingsFolder.getPath(), "cache")
|
||||
.toString();
|
||||
} else {
|
||||
enableCache = false;
|
||||
cacheFolder = null;
|
||||
log.error("The cache will disable because the setting folder is null, cannot generate the cache path");
|
||||
}
|
||||
final Path pathCacheInfo = getCachedInfoPath();
|
||||
log.info("Cache folder {}", cacheFolder);
|
||||
try {
|
||||
if (Files.exists(pathCacheInfo)) {
|
||||
ObjectMapper mapper = new ObjectMapper();
|
||||
final JsonNode jsonNode = mapper.readTree(pathCacheInfo.toFile());
|
||||
|
||||
// Read the files array
|
||||
TypeReference<List<FileCached>> typeRef = new TypeReference<List<FileCached>>() {
|
||||
};
|
||||
final List<FileCached> files = mapper
|
||||
.readValue(mapper.treeAsTokens(jsonNode.get("files")), typeRef);
|
||||
|
||||
// Update the map with the remote url as a key and the file cache info as a value
|
||||
cachedFiles.putAll(Collections
|
||||
.synchronizedMap(files
|
||||
.stream()
|
||||
.filter(FileCached::exists)
|
||||
.collect(Collectors.toMap(FileCached::getRemoteURL, Function.identity()))
|
||||
)
|
||||
);
|
||||
log.info("Number of file already in the cache {}", cachedFiles.size());
|
||||
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("Cannot initialized the cache", e);
|
||||
}
|
||||
}
|
||||
|
||||
public static Optional<FileCached> getFileCached(final URL remoteURL)
|
||||
throws URISyntaxException, NoSuchMethodException, ScriptException,
|
||||
IOException {
|
||||
return getFileCached(remoteURL, true);
|
||||
}
|
||||
|
||||
public static Optional<FileCached> getFileCached(final URL remoteURL, boolean enableCache)
|
||||
throws URISyntaxException, NoSuchMethodException, ScriptException,
|
||||
IOException {
|
||||
// Return always and empty file if the cache is not enable
|
||||
if (!(enableCache && FileDownloaderCache.enableCache)) {
|
||||
log.info("The cache is not enable.");
|
||||
return Optional.empty();
|
||||
}
|
||||
final String[] splitPath = remoteURL.getPath().split("/");
|
||||
if (splitPath.length == 0) {
|
||||
log.warn("The remote path as no file name {}", remoteURL);
|
||||
return Optional.empty();
|
||||
}
|
||||
// Create the path where the cached file should exist
|
||||
final Deque<String> addFirstRemoteURL = new LinkedList<>(Arrays.asList(splitPath));
|
||||
addFirstRemoteURL.addFirst(remoteURL.getHost());
|
||||
final Path cacheFilePath = Paths.get(cacheFolder, addFirstRemoteURL.toArray(new String[0]));
|
||||
|
||||
// Take from the cache the file info or build from scratch
|
||||
final FileCached fileCached = Optional.ofNullable(cachedFiles.get(remoteURL.toString()))
|
||||
.orElseGet(() -> new FileCached(remoteURL.toString(), cacheFilePath.toString()));
|
||||
|
||||
// If the file is change of the cache is disable run the HEAD request to check if the file is changed
|
||||
if (fileCached.isExpire() || !fileCached.exists()) {
|
||||
// Update remote etag and cache control header
|
||||
final Optional<FileCached> fileCachedInfoUpdated =
|
||||
FileDownloaderCache.updateCacheInfo(remoteURL, (remoteETagClean, cacheControl) -> {
|
||||
// Check cache control data
|
||||
if (cacheControl.isNoCache() || cacheControl.isMustRevalidate() || cacheControl.isNoStore()) {
|
||||
log.warn("The file {} must not be cache due to cache control header {}",
|
||||
remoteURL, cacheControl);
|
||||
return Optional.empty();
|
||||
}
|
||||
log.info("Update cached info of {}, createdAt {}, previous eTag {}, last eTag {}, cache control header {} ",
|
||||
remoteURL, fileCached.createdAt, fileCached.eTag, remoteETagClean, cacheControl);
|
||||
final FileCached fileCachedUpdateETag = new FileCached(
|
||||
remoteURL.toString(),
|
||||
cacheFilePath.toString(),
|
||||
fileCached.eTag,
|
||||
remoteETagClean, // Set the lastETag
|
||||
fileCached.md5,
|
||||
cacheControl // Set the new cache control
|
||||
);
|
||||
cachedFiles.put(remoteURL.toString(), fileCachedUpdateETag);
|
||||
return Optional.of(fileCachedUpdateETag);
|
||||
});
|
||||
FileDownloaderCache.updateCacheFilesInfo();
|
||||
return fileCachedInfoUpdated;
|
||||
}
|
||||
return Optional.of(fileCached);
|
||||
}
|
||||
|
||||
private static Optional<FileCached> updateCacheInfo(URL remoteURL, BiFunction<String, CacheControl, Optional<FileCached>> getNewFile)
|
||||
throws URISyntaxException, NoSuchMethodException, ScriptException,
|
||||
IOException {
|
||||
// Update the headers of the cached file
|
||||
final HttpURLConnection headRequest = new HttpConnectionManager(
|
||||
remoteURL).makeConnection((connection) -> {
|
||||
try {
|
||||
connection.setRequestMethod("HEAD");
|
||||
} catch (ProtocolException e) {
|
||||
log.error("Invalid protocol", e);
|
||||
}
|
||||
});
|
||||
final int responseCode = headRequest.getResponseCode();
|
||||
headRequest.disconnect();
|
||||
// Something bad is happening return a conservative true to try to download the file
|
||||
if (responseCode < 200 || responseCode >= 300) {
|
||||
log.warn("The head request return a bad response code " + responseCode);
|
||||
// if something bad happend
|
||||
return Optional.empty();
|
||||
}
|
||||
// Get all the useful headers
|
||||
String remoteETag = headRequest.getHeaderField("ETag");
|
||||
String cacheControlHeader = headRequest.getHeaderField("Cache-Control");
|
||||
if (remoteETag != null && cacheControlHeader != null) {
|
||||
final String remoteETagClean = remoteETag.trim().replace("\"", "");
|
||||
final CacheControl cacheControl = CacheControl.valueOf(cacheControlHeader);
|
||||
return getNewFile.apply(remoteETagClean, cacheControl);
|
||||
}
|
||||
log.warn("The head request do not return the ETag {} or the Cache-Control {}", remoteETag, cacheControlHeader);
|
||||
return Optional.empty();
|
||||
}
|
||||
|
||||
private synchronized static void updateCacheFilesInfo() throws IOException {
|
||||
ObjectMapper mapper = new ObjectMapper();
|
||||
// Generate a pretty json
|
||||
mapper.enable(SerializationFeature.INDENT_OUTPUT);
|
||||
final ObjectNode objectNode = mapper.createObjectNode();
|
||||
// Generate a json {"files":[...{files_info}...]}
|
||||
objectNode.putArray("files").addAll(
|
||||
cachedFiles.values().stream()
|
||||
.map((v) -> mapper.convertValue(v, JsonNode.class))
|
||||
.collect(Collectors.toList()));
|
||||
// Create the path Arduino15/cache
|
||||
Path cachedFileInfo = getCachedInfoPath();
|
||||
if (Files.notExists(cachedFileInfo)) {
|
||||
Files.createDirectories(cachedFileInfo.getParent());
|
||||
}
|
||||
log.info("Update cache file info in {}, number of cached files is {}", cachedFileInfo.toFile(), cachedFiles.size());
|
||||
// Write to cache.json
|
||||
mapper.writeValue(cachedFileInfo.toFile(), objectNode);
|
||||
}
|
||||
|
||||
private static Path getCachedInfoPath() {
|
||||
return Paths.get(cacheFolder, "cache.json");
|
||||
}
|
||||
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
static class FileCached {
|
||||
private final String remoteURL;
|
||||
private final String localPath;
|
||||
private final String eTag;
|
||||
private final String lastETag;
|
||||
private final String md5;
|
||||
private final String createdAt;
|
||||
private final CacheControl cacheControl;
|
||||
|
||||
FileCached() {
|
||||
this.remoteURL = null;
|
||||
this.localPath = null;
|
||||
lastETag = null;
|
||||
eTag = null;
|
||||
md5 = null;
|
||||
createdAt = null;
|
||||
cacheControl = null;
|
||||
}
|
||||
|
||||
FileCached(String remoteURL, String localPath) {
|
||||
this.remoteURL = remoteURL;
|
||||
this.localPath = localPath;
|
||||
lastETag = null;
|
||||
eTag = null;
|
||||
md5 = null;
|
||||
createdAt = LocalDateTime.now().format(DateTimeFormatter.ISO_DATE_TIME);
|
||||
cacheControl = null;
|
||||
}
|
||||
|
||||
public FileCached(String remoteURL, String localPath, String eTag, String lastETag, String md5, CacheControl cacheControl) {
|
||||
this.remoteURL = remoteURL;
|
||||
this.localPath = localPath;
|
||||
this.eTag = eTag;
|
||||
this.lastETag = lastETag;
|
||||
this.md5 = md5;
|
||||
this.createdAt = LocalDateTime.now().format(DateTimeFormatter.ISO_DATE_TIME);
|
||||
this.cacheControl = cacheControl;
|
||||
}
|
||||
|
||||
@JsonIgnore
|
||||
public boolean isExpire() {
|
||||
// Check if the file is expire
|
||||
final LocalDateTime now = LocalDateTime.now();
|
||||
return this.getExpiresTime().isBefore(now) || this.getExpiresTime().isEqual(now);
|
||||
}
|
||||
|
||||
@JsonIgnore
|
||||
public boolean isNotChange() {
|
||||
return !isChange();
|
||||
}
|
||||
|
||||
@JsonIgnore
|
||||
public boolean isChange() {
|
||||
// Check if the file is expire
|
||||
if (!isExpire()) {
|
||||
log.debug("The file \"{}\" is no expire, the eTag will not be checked. Expire time: {}", localPath,
|
||||
this.getExpiresTime().format(DateTimeFormatter.ISO_DATE_TIME));
|
||||
return false;
|
||||
}
|
||||
|
||||
if (lastETag != null) {
|
||||
// If are different means that the file is change
|
||||
return !lastETag.equals(eTag);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
@JsonIgnore
|
||||
public boolean exists() {
|
||||
return localPath != null && Files.exists(Paths.get(localPath));
|
||||
}
|
||||
|
||||
@JsonIgnore
|
||||
public Optional<File> getFileFromCache() {
|
||||
if (md5Check()) {
|
||||
return Optional.of(Paths.get(localPath).toFile());
|
||||
}
|
||||
return Optional.empty();
|
||||
|
||||
}
|
||||
|
||||
public synchronized void updateCacheFile(File fileToCache) throws Exception {
|
||||
Path cacheFilePath = Paths.get(localPath);
|
||||
|
||||
// If the cache directory does not exist create it
|
||||
if (!Files.exists(cacheFilePath.getParent())) {
|
||||
Files.createDirectories(cacheFilePath.getParent());
|
||||
}
|
||||
FileUtils.copyFile(fileToCache, cacheFilePath.toFile());
|
||||
final String md5 = this.calculateMD5();
|
||||
final String eTag;
|
||||
if (lastETag == null) {
|
||||
log.warn("The eTag was not calculate this time, is not the right behaviour fileCached={}, md5={}", this, md5);
|
||||
eTag = this.eTag;
|
||||
} else {
|
||||
eTag = this.lastETag;
|
||||
}
|
||||
FileCached newFileCached = new FileCached(
|
||||
this.remoteURL,
|
||||
this.localPath,
|
||||
eTag, // Initialize the right eTag with the last eTag because the file was updated
|
||||
eTag,
|
||||
md5,
|
||||
this.cacheControl
|
||||
);
|
||||
log.info("Update cache file: {}", newFileCached);
|
||||
cachedFiles.put(remoteURL, newFileCached);
|
||||
updateCacheFilesInfo();
|
||||
|
||||
}
|
||||
|
||||
public synchronized void invalidateCache() throws IOException {
|
||||
cachedFiles.remove(remoteURL);
|
||||
Files.deleteIfExists(Paths.get(localPath));
|
||||
}
|
||||
|
||||
private String calculateMD5() throws IOException, NoSuchAlgorithmException {
|
||||
if (exists()) {
|
||||
return FileHash.hash(Paths.get(localPath).toFile(), "MD5");
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
@JsonIgnore
|
||||
public boolean md5Check() {
|
||||
try {
|
||||
return !Objects.isNull(getMD5()) && Objects.equals(calculateMD5(), getMD5());
|
||||
} catch (Exception e) {
|
||||
log.error("Fail to calculate the MD5. file={}", this, e);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
@JsonIgnore
|
||||
public LocalDateTime getExpiresTime() {
|
||||
final int maxAge;
|
||||
if (cacheControl != null) {
|
||||
maxAge = cacheControl.getMaxAge();
|
||||
} else {
|
||||
maxAge = 0;
|
||||
}
|
||||
if (createdAt != null) {
|
||||
return LocalDateTime.parse(createdAt, DateTimeFormatter.ISO_DATE_TIME)
|
||||
.plusSeconds(maxAge);
|
||||
}
|
||||
return LocalDateTime.now();
|
||||
|
||||
}
|
||||
|
||||
public String getExpires() {
|
||||
return getExpiresTime().toString();
|
||||
}
|
||||
|
||||
public String getMD5() {
|
||||
return md5;
|
||||
}
|
||||
|
||||
public String geteTag() {
|
||||
return eTag;
|
||||
}
|
||||
|
||||
public String getRemoteURL() {
|
||||
return remoteURL;
|
||||
}
|
||||
|
||||
public String getLocalPath() {
|
||||
return localPath;
|
||||
}
|
||||
|
||||
public String getCreatedAt() {
|
||||
return createdAt;
|
||||
}
|
||||
|
||||
public CacheControl getCacheControl() {
|
||||
return cacheControl;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "FileCached{" +
|
||||
"eTag='" + eTag + '\'' +
|
||||
", lastETag='" + lastETag + '\'' +
|
||||
", remoteURL='" + remoteURL + '\'' +
|
||||
", localPath='" + localPath + '\'' +
|
||||
", md5='" + md5 + '\'' +
|
||||
", createdAt='" + createdAt + '\'' +
|
||||
", cacheControl=" + cacheControl +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,168 @@
|
||||
/*
|
||||
* This file is part of Arduino.
|
||||
*
|
||||
* Copyright 2019 Arduino LLC (http://www.arduino.cc/)
|
||||
*
|
||||
* Arduino is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation; either version 2 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
*
|
||||
* As a special exception, you may use this file as part of a free software
|
||||
* library without restriction. Specifically, if other files instantiate
|
||||
* templates or use macros or inline functions from this file, or you compile
|
||||
* this file and link it with other files to produce an executable, this
|
||||
* file does not by itself cause the resulting executable to be covered by
|
||||
* the GNU General Public License. This exception does not however
|
||||
* invalidate any other reasons why the executable file might be covered by
|
||||
* the GNU General Public License.
|
||||
*/
|
||||
|
||||
package cc.arduino.utils.network;
|
||||
|
||||
import cc.arduino.net.CustomProxySelector;
|
||||
import org.apache.commons.codec.binary.Base64;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import processing.app.BaseNoGui;
|
||||
import processing.app.PreferencesData;
|
||||
|
||||
import javax.script.ScriptException;
|
||||
import java.io.IOException;
|
||||
import java.net.HttpURLConnection;
|
||||
import java.net.Proxy;
|
||||
import java.net.URISyntaxException;
|
||||
import java.net.URL;
|
||||
import java.util.UUID;
|
||||
import java.util.function.Consumer;
|
||||
|
||||
public class HttpConnectionManager {
|
||||
private static Logger log = LogManager.getLogger(HttpConnectionManager.class);
|
||||
private static final String userAgent;
|
||||
private static final int connectTimeout;
|
||||
private static final int maxRedirectNumber;
|
||||
private final URL requestURL;
|
||||
private final String id;
|
||||
|
||||
|
||||
static {
|
||||
final String defaultUserAgent = String.format(
|
||||
"ArduinoIDE/%s (%s; %s; %s; %s) Java/%s (%s)",
|
||||
BaseNoGui.VERSION_NAME,
|
||||
System.getProperty("os.name"),
|
||||
System.getProperty("os.version"),
|
||||
System.getProperty("os.arch"),
|
||||
System.getProperty("user.language"),
|
||||
System.getProperty("java.version"),
|
||||
System.getProperty("java.vendor")
|
||||
);
|
||||
userAgent = PreferencesData.get("http.user_agent", defaultUserAgent);
|
||||
int connectTimeoutFromConfig = 5000;
|
||||
try {
|
||||
connectTimeoutFromConfig =
|
||||
Integer.parseInt(
|
||||
PreferencesData.get("http.connection_timeout_ms", "5000"));
|
||||
} catch (NumberFormatException e) {
|
||||
log.warn(
|
||||
"Cannot parse the http.connection_timeout configuration switch to default {} milliseconds", connectTimeoutFromConfig, e.getCause());
|
||||
}
|
||||
connectTimeout = connectTimeoutFromConfig;
|
||||
// Set by default 20 max redirect to follow
|
||||
int maxRedirectNumberConfig = 20;
|
||||
try {
|
||||
maxRedirectNumberConfig =
|
||||
Integer.parseInt(
|
||||
PreferencesData.get("http.max_redirect_number", "20"));
|
||||
} catch (NumberFormatException e) {
|
||||
log.warn(
|
||||
"Cannot parse the http.max_redirect_number configuration switch to default {}", maxRedirectNumberConfig, e.getCause());
|
||||
}
|
||||
maxRedirectNumber = maxRedirectNumberConfig;
|
||||
}
|
||||
|
||||
public HttpConnectionManager(URL requestURL) {
|
||||
this.requestURL = requestURL;
|
||||
if (requestURL.getHost().endsWith("arduino.cc")) {
|
||||
final String idString = PreferencesData.get("update.id", "0");
|
||||
id = Long.toString(Long.parseLong(idString));
|
||||
} else {
|
||||
id = null;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public HttpURLConnection makeConnection(Consumer<HttpURLConnection> beforeConnection)
|
||||
throws IOException, NoSuchMethodException, ScriptException, URISyntaxException {
|
||||
return makeConnection(this.requestURL, 0, beforeConnection);
|
||||
}
|
||||
|
||||
|
||||
public HttpURLConnection makeConnection()
|
||||
throws IOException, NoSuchMethodException, ScriptException, URISyntaxException {
|
||||
return makeConnection(this.requestURL, 0, (c) -> {
|
||||
});
|
||||
}
|
||||
|
||||
private HttpURLConnection makeConnection(URL requestURL, int movedTimes,
|
||||
Consumer<HttpURLConnection> beforeConnection) throws IOException, URISyntaxException, ScriptException, NoSuchMethodException {
|
||||
if (movedTimes > maxRedirectNumber) {
|
||||
log.warn("Too many redirect " + requestURL);
|
||||
throw new IOException("Too many redirect " + requestURL);
|
||||
}
|
||||
|
||||
Proxy proxy = new CustomProxySelector(PreferencesData.getMap())
|
||||
.getProxyFor(requestURL.toURI());
|
||||
log.debug("Using proxy {}", proxy);
|
||||
|
||||
final String requestId = UUID.randomUUID().toString()
|
||||
.toUpperCase().replace("-", "").substring(0, 16);
|
||||
HttpURLConnection connection = (HttpURLConnection) requestURL
|
||||
.openConnection(proxy);
|
||||
connection.setRequestProperty("User-agent", userAgent);
|
||||
connection.setRequestProperty("X-Request-ID", requestId);
|
||||
if (id != null) {
|
||||
connection.setRequestProperty("X-ID", id);
|
||||
}
|
||||
if (requestURL.getUserInfo() != null) {
|
||||
String auth = "Basic " + new String(
|
||||
new Base64().encode(requestURL.getUserInfo().getBytes()));
|
||||
connection.setRequestProperty("Authorization", auth);
|
||||
}
|
||||
|
||||
int initialSize = 0;
|
||||
connection.setRequestProperty("Range", "bytes=" + initialSize + "-");
|
||||
connection.setConnectTimeout(connectTimeout);
|
||||
beforeConnection.accept(connection);
|
||||
|
||||
// Connect
|
||||
log.info("Connect to {}, method={}, request id={}", requestURL, connection.getRequestMethod(), requestId);
|
||||
|
||||
connection.connect();
|
||||
int resp = connection.getResponseCode();
|
||||
log.info("Request complete URL=\"{}\", method={}, response code={}, request id={}, headers={}",
|
||||
requestURL, connection.getRequestMethod(), resp, requestId, StringUtils.join(connection.getHeaderFields()));
|
||||
|
||||
if (resp == HttpURLConnection.HTTP_MOVED_PERM
|
||||
|| resp == HttpURLConnection.HTTP_MOVED_TEMP) {
|
||||
|
||||
URL newUrl = new URL(connection.getHeaderField("Location"));
|
||||
log.info("The response code was a 301,302 so try again with the new URL " + newUrl);
|
||||
|
||||
return this.makeConnection(newUrl, movedTimes + 1, beforeConnection);
|
||||
}
|
||||
|
||||
return connection;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -23,6 +23,11 @@
|
||||
package processing.app;
|
||||
|
||||
import cc.arduino.packages.BoardPort;
|
||||
import cc.arduino.utils.network.HttpConnectionManager;
|
||||
import com.fasterxml.jackson.databind.DeserializationFeature;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import processing.app.debug.TargetBoard;
|
||||
import processing.app.debug.TargetPackage;
|
||||
import processing.app.debug.TargetPlatform;
|
||||
@ -31,19 +36,10 @@ import processing.app.legacy.PConstants;
|
||||
import javax.swing.*;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
|
||||
import java.net.URL;
|
||||
import java.net.URLConnection;
|
||||
import java.net.HttpURLConnection;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.databind.DeserializationFeature;
|
||||
import java.io.InputStream;
|
||||
import java.net.HttpURLConnection;
|
||||
import java.net.URL;
|
||||
import java.util.*;
|
||||
|
||||
import static processing.app.I18n.tr;
|
||||
|
||||
@ -64,7 +60,7 @@ import static processing.app.I18n.tr;
|
||||
* know if name is proper Java package syntax.)
|
||||
*/
|
||||
public class Platform {
|
||||
|
||||
private static Logger log = LogManager.getLogger(Platform.class);
|
||||
|
||||
/**
|
||||
* Set the default L & F. While I enjoy the bounty of the sixteen possible
|
||||
@ -170,6 +166,7 @@ public class Platform {
|
||||
}
|
||||
|
||||
private native String resolveDeviceAttachedToNative(String serial);
|
||||
|
||||
private native String[] listSerialsNative();
|
||||
|
||||
public String preListAllCandidateDevices() {
|
||||
@ -180,7 +177,7 @@ public class Platform {
|
||||
return new ArrayList<>(Arrays.asList(listSerialsNative()));
|
||||
}
|
||||
|
||||
public List<String> listSerialsNames(){
|
||||
public List<String> listSerialsNames() {
|
||||
List<String> list = new LinkedList<>();
|
||||
for (String port : listSerialsNative()) {
|
||||
list.add(port.split("_")[0]);
|
||||
@ -188,46 +185,33 @@ public class Platform {
|
||||
return list;
|
||||
}
|
||||
|
||||
public static class BoardCloudAPIid {
|
||||
public BoardCloudAPIid() { }
|
||||
private String name;
|
||||
private String architecture;
|
||||
private String id;
|
||||
public String getName() { return name; }
|
||||
public String getArchitecture() { return architecture; }
|
||||
public String getId() { return id; }
|
||||
public void setName(String tmp) { name = tmp; }
|
||||
public void setArchitecture(String tmp) { architecture = tmp; }
|
||||
public void setId(String tmp) { id = tmp; }
|
||||
}
|
||||
|
||||
public synchronized void getBoardWithMatchingVidPidFromCloud(String vid, String pid) {
|
||||
// this method is less useful in Windows < WIN10 since you need drivers to be already installed
|
||||
ObjectMapper mapper = new ObjectMapper();
|
||||
mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
|
||||
try {
|
||||
URL jsonUrl = new URL("http", "api-builder.arduino.cc", 80, "/builder/v1/boards/0x"+vid+"/0x"+pid);
|
||||
URLConnection connection = jsonUrl.openConnection();
|
||||
String userAgent = "ArduinoIDE/" + BaseNoGui.VERSION_NAME + " Java/"
|
||||
+ System.getProperty("java.version");
|
||||
connection.setRequestProperty("User-agent", userAgent);
|
||||
connection.connect();
|
||||
HttpURLConnection httpConnection = (HttpURLConnection) connection;
|
||||
URL jsonUrl = new URL(String.format("https://builder.arduino.cc/builder/v1/boards/0x%s/0x%s", vid, pid));
|
||||
|
||||
final HttpURLConnection httpConnection = new HttpConnectionManager(jsonUrl)
|
||||
.makeConnection();
|
||||
int code = httpConnection.getResponseCode();
|
||||
if (code == 404) {
|
||||
log.warn("Fail to get the Vid Pid information from the builder response code={}", code);
|
||||
return;
|
||||
}
|
||||
InputStream is = httpConnection.getInputStream();
|
||||
BoardCloudAPIid board = mapper.readValue(is, BoardCloudAPIid.class);
|
||||
log.info("Board info from the cloud {}", board);
|
||||
// Launch a popup with a link to boardmanager#board.getName()
|
||||
// replace spaces with &
|
||||
String realBoardName = board.getName().replaceAll("\\(.*?\\)", "").trim();
|
||||
String boardNameReplaced = realBoardName.replaceAll(" ", "&");
|
||||
String message = I18n.format(tr("{0}Install this package{1} to use your {2} board"), "<a href=\"http://boardsmanager/all#"+boardNameReplaced+"\">", "</a>", realBoardName);
|
||||
String message = I18n.format(tr("{0}Install this package{1} to use your {2} board"), "<a href=\"http://boardsmanager/all#" + boardNameReplaced + "\">", "</a>", realBoardName);
|
||||
BaseNoGui.setBoardManagerLink(message);
|
||||
} catch (Exception e) {
|
||||
// No connection no problem, fail silently
|
||||
//e.printStackTrace();
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@ -259,7 +243,7 @@ public class Platform {
|
||||
boardData.put("board", board);
|
||||
boardData.put("vid", vids.get(i));
|
||||
boardData.put("pid", pids.get(i));
|
||||
String extrafields = vid_pid_iSerial.substring(vidPid.length()+1);
|
||||
String extrafields = vid_pid_iSerial.substring(vidPid.length() + 1);
|
||||
String[] parts = extrafields.split("_");
|
||||
boardData.put("iserial", parts[0]);
|
||||
return boardData;
|
||||
@ -272,6 +256,56 @@ public class Platform {
|
||||
return null;
|
||||
}
|
||||
|
||||
public static class BoardCloudAPIid {
|
||||
|
||||
private String fqbn;
|
||||
private String name;
|
||||
private String architecture;
|
||||
private String id;
|
||||
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
public String getFqbn() {
|
||||
return fqbn;
|
||||
}
|
||||
|
||||
public String getArchitecture() {
|
||||
return architecture;
|
||||
}
|
||||
|
||||
public String getId() {
|
||||
return id;
|
||||
}
|
||||
|
||||
public void setName(String tmp) {
|
||||
name = tmp;
|
||||
}
|
||||
|
||||
public void setFqbn(String fqbn) {
|
||||
this.fqbn = fqbn;
|
||||
}
|
||||
|
||||
public void setArchitecture(String tmp) {
|
||||
architecture = tmp;
|
||||
}
|
||||
|
||||
public void setId(String tmp) {
|
||||
id = tmp;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "BoardCloudAPIid{" +
|
||||
"name='" + name + '\'' +
|
||||
", fqbn='" + fqbn + '\'' +
|
||||
", architecture='" + architecture + '\'' +
|
||||
", id='" + id + '\'' +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
|
||||
public String resolveDeviceByBoardID(Map<String, TargetPackage> packages, String boardId) {
|
||||
assert packages != null;
|
||||
assert boardId != null;
|
||||
|
||||
@ -265,7 +265,10 @@ public class PreferencesData {
|
||||
}
|
||||
|
||||
public static Collection<String> getCollection(String key) {
|
||||
return Arrays.asList(get(key, "").split(","));
|
||||
return Arrays.stream(get(key, "").split(","))
|
||||
// Remove empty strings from the collection
|
||||
.filter((v) -> !v.trim().isEmpty())
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
public static void setCollection(String key, Collection<String> values) {
|
||||
|
||||
@ -438,6 +438,7 @@
|
||||
<option value="-Dcom.apple.smallTabs=true"/>
|
||||
<option value="-DAPP_DIR=$APP_ROOT/Contents/Java"/>
|
||||
<option value="-Djava.ext.dirs=$JVM_RUNTIME/Contents/Home/lib/ext/:$JVM_RUNTIME/Contents/Home/jre/lib/ext/"/>
|
||||
<option value="-Djava.net.preferIPv4Stack=true"/>
|
||||
|
||||
<!--
|
||||
<option value="-Dapple.awt.showGrowBox=false"/>
|
||||
|
||||
@ -1,3 +1,4 @@
|
||||
-Xms128M
|
||||
-Xmx512M
|
||||
-Dfile.encoding=UTF8
|
||||
-Xms128M
|
||||
-Xmx512M
|
||||
-Dfile.encoding=UTF8
|
||||
-Djava.net.preferIPv4Stack=true
|
||||
|
||||
@ -36,6 +36,7 @@
|
||||
<cp>%EXEDIR%/lib/commons-lang3-3.8.1.jar</cp>
|
||||
<cp>%EXEDIR%/lib/commons-logging-1.0.4.jar</cp>
|
||||
<cp>%EXEDIR%/lib/commons-net-3.3.jar</cp>
|
||||
<cp>%EXEDIR%/lib/commons-io-2.6.jar</cp>
|
||||
<cp>%EXEDIR%/lib/jackson-annotations-2.9.5.jar</cp>
|
||||
<cp>%EXEDIR%/lib/jackson-core-2.9.5.jar</cp>
|
||||
<cp>%EXEDIR%/lib/jackson-databind-2.9.5.jar</cp>
|
||||
@ -43,6 +44,8 @@
|
||||
<cp>%EXEDIR%/lib/java-semver-0.8.0.jar</cp>
|
||||
<cp>%EXEDIR%/lib/jmdns-3.5.3.jar</cp>
|
||||
<cp>%EXEDIR%/lib/jtouchbar-1.0.0.jar</cp>
|
||||
<cp>%EXEDIR%/lib/log4j-api-2.12.0.jar</cp>
|
||||
<cp>%EXEDIR%/lib/log4j-core-2.12.0.jar</cp>
|
||||
<cp>%EXEDIR%/lib/slf4j-simple-1.7.22.jar</cp>
|
||||
<cp>%EXEDIR%/lib/slf4j-api-1.7.22.jar</cp>
|
||||
<cp>%EXEDIR%/lib/jna-4.2.2.jar</cp>
|
||||
|
||||
@ -36,6 +36,7 @@
|
||||
<cp>%EXEDIR%/lib/commons-lang3-3.8.1.jar</cp>
|
||||
<cp>%EXEDIR%/lib/commons-logging-1.0.4.jar</cp>
|
||||
<cp>%EXEDIR%/lib/commons-net-3.3.jar</cp>
|
||||
<cp>%EXEDIR%/lib/commons-io-2.6.jar</cp>
|
||||
<cp>%EXEDIR%/lib/jackson-annotations-2.9.5.jar</cp>
|
||||
<cp>%EXEDIR%/lib/jackson-core-2.9.5.jar</cp>
|
||||
<cp>%EXEDIR%/lib/jackson-databind-2.9.5.jar</cp>
|
||||
@ -43,6 +44,8 @@
|
||||
<cp>%EXEDIR%/lib/java-semver-0.8.0.jar</cp>
|
||||
<cp>%EXEDIR%/lib/jmdns-3.5.3.jar</cp>
|
||||
<cp>%EXEDIR%/lib/jtouchbar-1.0.0.jar</cp>
|
||||
<cp>%EXEDIR%/lib/log4j-api-2.12.0.jar</cp>
|
||||
<cp>%EXEDIR%/lib/log4j-core-2.12.0.jar</cp>
|
||||
<cp>%EXEDIR%/lib/slf4j-simple-1.7.22.jar</cp>
|
||||
<cp>%EXEDIR%/lib/slf4j-api-1.7.22.jar</cp>
|
||||
<cp>%EXEDIR%/lib/jna-4.2.2.jar</cp>
|
||||
|
||||
Loading…
Reference in New Issue
Block a user