From 8f1f4be95cda231f55ada6bca635f8f9354a4fd8 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Thu, 12 Mar 2020 17:27:12 +0100 Subject: [PATCH] compile/verify now uses arduino-cli daemon The build options are now passed through GRPC. There should be no need to pass library or hardware paths anymore. By default the daemon compiles and export the hex in the sketch folder but this is disallowed by the IDE unless the user explicitly run the "Export .hex" command. All the business logic to compose board properties and command line for the build is now embedded in the daemon and has been removed from the IDE. \o/ --- arduino-core/src/cc/arduino/Compiler.java | 384 ++++-------------- .../src/cc/arduino/cli/ArduinoCore.java | 5 +- .../cc/arduino/cli/ArduinoCoreInstance.java | 31 +- 3 files changed, 102 insertions(+), 318 deletions(-) diff --git a/arduino-core/src/cc/arduino/Compiler.java b/arduino-core/src/cc/arduino/Compiler.java index 21bd3e5ef..565c7f23f 100644 --- a/arduino-core/src/cc/arduino/Compiler.java +++ b/arduino-core/src/cc/arduino/Compiler.java @@ -29,37 +29,34 @@ package cc.arduino; -import cc.arduino.cli.ArduinoCoreInstance; -import cc.arduino.i18n.I18NAwareMessageConsumer; -import cc.arduino.packages.BoardPort; -import org.apache.commons.exec.CommandLine; -import org.apache.commons.exec.DefaultExecutor; -import org.apache.commons.exec.PumpStreamHandler; -import org.apache.commons.lang3.StringUtils; -import processing.app.*; -import processing.app.debug.*; -import processing.app.helpers.PreferencesMap; -import processing.app.helpers.PreferencesMapException; -import processing.app.helpers.ProcessUtils; -import processing.app.helpers.StringReplacer; -import processing.app.legacy.PApplet; -import processing.app.tools.DoubleQuotedArgumentsOnWindowsCommandLine; +import static processing.app.I18n.tr; -import java.io.*; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.StandardCopyOption; +import java.io.File; +import java.io.IOException; import java.util.ArrayList; -import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.regex.Pattern; import java.util.stream.Collectors; -import java.util.stream.Stream; -import static processing.app.I18n.tr; +import org.apache.commons.lang3.StringUtils; + +import cc.arduino.cli.ArduinoCoreInstance; +import cc.arduino.cli.commands.Compile.CompileReq; +import cc.arduino.cli.commands.Compile.CompileResult; +import cc.arduino.i18n.I18NAwareMessageConsumer; +import cc.arduino.packages.BoardPort; +import processing.app.BaseNoGui; +import processing.app.I18n; +import processing.app.PreferencesData; +import processing.app.Sketch; +import processing.app.SketchFile; +import processing.app.debug.MessageConsumer; +import processing.app.debug.RunnerException; +import processing.app.debug.TargetBoard; +import processing.app.debug.TargetPlatform; +import processing.app.helpers.PreferencesMapException; +import processing.app.legacy.PApplet; public class Compiler implements MessageConsumer { @@ -124,16 +121,9 @@ public class Compiler implements MessageConsumer { tr("Sketchbook path not defined"); tr("The --upload option supports only one file at a time"); tr("Verifying and uploading..."); - } - - enum BuilderAction { - COMPILE("-compile"), DUMP_PREFS("-dump-prefs"); - - final String value; - - BuilderAction(String value) { - this.value = value; - } + tr("Warning: This core does not support exporting sketches. Please consider upgrading it or contacting its author"); + tr("{0} returned {1}"); + tr("Error compiling."); } private static final Pattern ERROR_FORMAT = Pattern.compile("(.+\\.\\w+):(\\d+)(:\\d+)*:\\s*((fatal)?\\s*error:\\s*)(.*)\\s*", Pattern.MULTILINE | Pattern.DOTALL); @@ -171,28 +161,72 @@ public class Compiler implements MessageConsumer { if (!mayBoard.isPresent()) { throw new RunnerException("Board is not selected"); } - TargetBoard board = mayBoard.get(); + + CompileReq.Builder req = CompileReq.newBuilder(); + TargetPlatform platform = board.getContainerPlatform(); - TargetPackage aPackage = platform.getContainerPackage(); + String vendor = platform.getContainerPackage().getId(); + String fqbn = vendor+":"+ platform.getId()+":"+board.getId(); + String options = boardOptions(board); + if (!options.isEmpty()) { + fqbn += ":" + options; + } + req.setFqbn(fqbn); + String vidpid = VIDPID(); + if (!vidpid.isEmpty()) { + req.setVidPid(vidpid); + } - PreferencesMap prefs = loadPreferences(board, platform, aPackage, vidpid); + req.setBuildPath(buildPath); - MessageConsumerOutputStream out = new MessageConsumerOutputStream(new ProgressAwareMessageConsumer(new I18NAwareMessageConsumer(System.out, System.err), progListeners), "\n"); - MessageConsumerOutputStream err = new MessageConsumerOutputStream(new I18NAwareMessageConsumer(System.err, Compiler.this), "\n"); + if (PreferencesData.getBoolean("compiler.cache_core") && buildCache != null) { + req.setBuildCachePath(buildCache.getAbsolutePath()); + } - callArduinoBuilder(board, platform, aPackage, vidpid, BuilderAction.COMPILE, out, err); + PreferencesData.getMap() + .subTree("runtime.build_properties_custom") + .entrySet() + .stream() + .forEach(kv -> req.addBuildProperties(kv.getKey() + "=" + kv.getValue())); - out.flush(); - err.flush(); + req.addBuildProperties("build.warn_data_percentage=" + + PreferencesData.get("build.warn_data_percentage")); - if (exportHex) { - runActions("hooks.savehex.presavehex", prefs); + req.setWarnings(PreferencesData.get("compiler.warning_level")); - saveHex(prefs); + req.setSketchPath(pathToSketch.getAbsolutePath()); - runActions("hooks.savehex.postsavehex", prefs); + req.setDryRun(!exportHex); + + if (verbose) { + req.setVerbose(true); + System.out.println("Build options"); + System.out.println("-------------"); + System.out.println(req); + } + + CompileResult result; + try { + MessageConsumerOutputStream out = new MessageConsumerOutputStream(new ProgressAwareMessageConsumer(new I18NAwareMessageConsumer(System.out, System.err), progListeners), "\n"); + MessageConsumerOutputStream err = new MessageConsumerOutputStream(new I18NAwareMessageConsumer(System.err, this), "\n"); + + result = core.compile(req.build(), out, err); + + out.flush(); + err.flush(); + } catch (Exception e) { + throw new RunnerException(e); + } + + if (exception != null) + throw exception; + + if (result == CompileResult.error) { + RunnerException re = new RunnerException(I18n.format(tr("Error compiling for board {0}."), board.getName())); + re.hideStackTrace(); + throw re; } return sketch.getPrimaryFile().getFileName(); @@ -213,264 +247,6 @@ public class Compiler implements MessageConsumer { return vid.toUpperCase() + "_" + pid.toUpperCase(); } - private PreferencesMap loadPreferences(TargetBoard board, TargetPlatform platform, TargetPackage aPackage, String vidpid) throws RunnerException, IOException { - return new PreferencesMap(); - } - - private void addPathFlagIfPathExists(List cmd, String flag, File folder) { - if (folder.exists()) { - cmd.add(flag); - cmd.add(folder.getAbsolutePath()); - } - } - - private void callArduinoBuilder(TargetBoard board, TargetPlatform platform, TargetPackage aPackage, String vidpid, BuilderAction action, OutputStream outStream, OutputStream errStream) throws RunnerException { - List cmd = new ArrayList<>(); - cmd.add(BaseNoGui.getContentFile("arduino-builder").getAbsolutePath()); - cmd.add(action.value); - cmd.add("-logger=machine"); - - File installedPackagesFolder = new File(BaseNoGui.getSettingsFolder(), "packages"); - - addPathFlagIfPathExists(cmd, "-hardware", BaseNoGui.getHardwareFolder()); - addPathFlagIfPathExists(cmd, "-hardware", installedPackagesFolder); - addPathFlagIfPathExists(cmd, "-hardware", BaseNoGui.getSketchbookHardwareFolder()); - - addPathFlagIfPathExists(cmd, "-tools", BaseNoGui.getContentFile("tools-builder")); - addPathFlagIfPathExists(cmd, "-tools", Paths.get(BaseNoGui.getHardwarePath(), "tools", "avr").toFile()); - addPathFlagIfPathExists(cmd, "-tools", installedPackagesFolder); - - addPathFlagIfPathExists(cmd, "-built-in-libraries", BaseNoGui.getContentFile("libraries")); - addPathFlagIfPathExists(cmd, "-libraries", BaseNoGui.getSketchbookLibrariesFolder()); - - String fqbn = Stream.of(aPackage.getId(), platform.getId(), board.getId(), boardOptions(board)).filter(s -> !s.isEmpty()).collect(Collectors.joining(":")); - cmd.add("-fqbn=" + fqbn); - - if (!"".equals(vidpid)) { - cmd.add("-vid-pid=" + vidpid); - } - - cmd.add("-ide-version=" + BaseNoGui.REVISION); - cmd.add("-build-path"); - cmd.add(buildPath); - cmd.add("-warnings=" + PreferencesData.get("compiler.warning_level")); - - if (PreferencesData.getBoolean("compiler.cache_core") == true && buildCache != null) { - cmd.add("-build-cache"); - cmd.add(buildCache.getAbsolutePath()); - } - - PreferencesData.getMap() - .subTree("runtime.build_properties_custom") - .entrySet() - .stream() - .forEach(kv -> cmd.add("-prefs=" + kv.getKey() + "=" + kv.getValue())); - - cmd.add("-prefs=build.warn_data_percentage=" + PreferencesData.get("build.warn_data_percentage")); - - for (Map.Entry entry : BaseNoGui.getBoardPreferences().entrySet()) { - if (entry.getKey().startsWith("runtime.tools")) { - cmd.add("-prefs=" + entry.getKey() + "=" + entry.getValue()); - } - } - - //commandLine.addArgument("-debug-level=10", false); - - if (verbose) { - cmd.add("-verbose"); - } - - cmd.add(pathToSketch.getAbsolutePath()); - - if (verbose) { - System.out.println(StringUtils.join(cmd, ' ')); - } - - int result; - try { - core.compile(fqbn, pathToSketch.getAbsolutePath()); - - /* - MessageSiphon in = new MessageSiphon(proc.getInputStream(), (msg) -> { - try { - outStream.write(msg.getBytes()); - } catch (Exception e) { - exception = new RunnerException(e); - } - }); - MessageSiphon err = new MessageSiphon(proc.getErrorStream(), (msg) -> { - try { - errStream.write(msg.getBytes()); - } catch (Exception e) { - exception = new RunnerException(e); - } - }); - - in.join(); - err.join(); - result = proc.waitFor(); - */ - } catch (Exception e) { - throw new RunnerException(e); - } - - if (exception != null) { - RunnerException re = new RunnerException(I18n.format(tr("Error compiling for board {0}."), board.getName())); - re.hideStackTrace(); - throw re; - } - } - - private void saveHex(PreferencesMap prefs) throws RunnerException { - List compiledSketches = new ArrayList<>(prefs.subTree("recipe.output.tmp_file", 1).values()); - List copyOfCompiledSketches = new ArrayList<>(prefs.subTree("recipe.output.save_file", 1).values()); - - if (isExportCompiledSketchSupported(compiledSketches, copyOfCompiledSketches, prefs)) { - System.err.println(tr("Warning: This core does not support exporting sketches. Please consider upgrading it or contacting its author")); - return; - } - - PreferencesMap dict = new PreferencesMap(prefs); - dict.put("ide_version", "" + BaseNoGui.REVISION); - PreferencesMap withBootloaderDict = new PreferencesMap(dict); - dict.put("build.project_name", dict.get("build.project_name") + ".with_bootloader"); - - if (!compiledSketches.isEmpty()) { - for (int i = 0; i < compiledSketches.size(); i++) { - saveHex(compiledSketches.get(i), copyOfCompiledSketches.get(i), dict); - saveHex(compiledSketches.get(i), copyOfCompiledSketches.get(i), withBootloaderDict); - } - } else { - try { - saveHex(prefs.getOrExcept("recipe.output.tmp_file"), prefs.getOrExcept("recipe.output.save_file"), dict); - saveHex(prefs.getOrExcept("recipe.output.tmp_file"), prefs.getOrExcept("recipe.output.save_file"), withBootloaderDict); - } catch (PreferencesMapException e) { - throw new RunnerException(e); - } - } - } - - private void saveHex(String compiledSketch, String copyOfCompiledSketch, PreferencesMap prefs) throws RunnerException { - try { - compiledSketch = StringReplacer.replaceFromMapping(compiledSketch, prefs); - copyOfCompiledSketch = StringReplacer.replaceFromMapping(copyOfCompiledSketch, prefs); - copyOfCompiledSketch = copyOfCompiledSketch.replaceAll(":", "_"); - - Path compiledSketchPath; - Path compiledSketchPathInSubfolder = Paths.get(prefs.get("build.path"), "sketch", compiledSketch); - Path compiledSketchPathInBuildPath = Paths.get(prefs.get("build.path"), compiledSketch); - if (Files.exists(compiledSketchPathInSubfolder)) { - compiledSketchPath = compiledSketchPathInSubfolder; - } else if (Files.exists(compiledSketchPathInBuildPath)) { - compiledSketchPath = compiledSketchPathInBuildPath; - } else { - return; - } - - Path copyOfCompiledSketchFilePath = Paths.get(this.sketch.getFolder().getAbsolutePath(), copyOfCompiledSketch); - - Files.copy(compiledSketchPath, copyOfCompiledSketchFilePath, StandardCopyOption.REPLACE_EXISTING); - } catch (IOException e) { - throw new RunnerException(e); - } - } - - private boolean isExportCompiledSketchSupported(List compiledSketches, List copyOfCompiledSketches, PreferencesMap prefs) { - return (compiledSketches.isEmpty() || copyOfCompiledSketches.isEmpty() || copyOfCompiledSketches.size() < compiledSketches.size()) && (!prefs.containsKey("recipe.output.tmp_file") || !prefs.containsKey("recipe.output.save_file")); - } - - private void runActions(String recipeClass, PreferencesMap prefs) throws RunnerException, PreferencesMapException { - List patterns = prefs.keySet().stream().filter(key -> key.startsWith("recipe." + recipeClass) && key.endsWith(".pattern")).collect(Collectors.toList()); - Collections.sort(patterns); - for (String recipe : patterns) { - runRecipe(recipe, prefs); - } - } - - private void runRecipe(String recipe, PreferencesMap prefs) throws RunnerException, PreferencesMapException { - PreferencesMap dict = new PreferencesMap(prefs); - dict.put("ide_version", "" + BaseNoGui.REVISION); - dict.put("sketch_path", sketch.getFolder().getAbsolutePath()); - - String[] cmdArray; - String cmd = prefs.getOrExcept(recipe); - try { - cmdArray = StringReplacer.formatAndSplit(cmd, dict); - } catch (Exception e) { - throw new RunnerException(e); - } - exec(cmdArray); - } - - private void exec(String[] command) throws RunnerException { - // eliminate any empty array entries - List stringList = new ArrayList<>(); - for (String string : command) { - string = string.trim(); - if (string.length() != 0) - stringList.add(string); - } - command = stringList.toArray(new String[stringList.size()]); - if (command.length == 0) - return; - - if (verbose) { - for (String c : command) - System.out.print(c + " "); - System.out.println(); - } - - DefaultExecutor executor = new DefaultExecutor(); - executor.setStreamHandler(new PumpStreamHandler() { - - @Override - protected Thread createPump(InputStream is, OutputStream os, boolean closeWhenExhausted) { - final Thread result = new Thread(new MyStreamPumper(is, Compiler.this)); - result.setName("MyStreamPumper Thread"); - result.setDaemon(true); - return result; - - } - }); - - CommandLine commandLine = new DoubleQuotedArgumentsOnWindowsCommandLine(command[0]); - for (int i = 1; i < command.length; i++) { - commandLine.addArgument(command[i], false); - } - - int result; - executor.setExitValues(null); - try { - result = executor.execute(commandLine); - } catch (IOException e) { - RunnerException re = new RunnerException(e.getMessage()); - re.hideStackTrace(); - throw re; - } - executor.setExitValues(new int[0]); - - // an error was queued up by message(), barf this back to compile(), - // which will barf it back to Editor. if you're having trouble - // discerning the imagery, consider how cows regurgitate their food - // to digest it, and the fact that they have five stomaches. - // - //System.out.println("throwing up " + exception); - if (exception != null) - throw exception; - - if (result > 1) { - // a failure in the tool (e.g. unable to locate a sub-executable) - System.err - .println(I18n.format(tr("{0} returned {1}"), command[0], result)); - } - - if (result != 0) { - RunnerException re = new RunnerException(tr("Error compiling.")); - re.hideStackTrace(); - throw re; - } - } - private String boardOptions(TargetBoard board) { return board.getMenuIds().stream() .filter(board::hasMenu) diff --git a/arduino-core/src/cc/arduino/cli/ArduinoCore.java b/arduino-core/src/cc/arduino/cli/ArduinoCore.java index 20b8daaeb..1c9bc94a0 100644 --- a/arduino-core/src/cc/arduino/cli/ArduinoCore.java +++ b/arduino-core/src/cc/arduino/cli/ArduinoCore.java @@ -39,11 +39,8 @@ import cc.arduino.cli.commands.Commands.InitReq; import cc.arduino.cli.commands.Commands.InitResp; import cc.arduino.cli.commands.Common.Instance; import cc.arduino.cli.settings.SettingsGrpc; -import cc.arduino.cli.settings.SettingsOuterClass; -import cc.arduino.cli.settings.SettingsOuterClass.GetAllRequest; -import cc.arduino.cli.settings.SettingsOuterClass.RawData; -import cc.arduino.cli.settings.SettingsOuterClass.SetValueResponse; import cc.arduino.cli.settings.SettingsGrpc.SettingsBlockingStub; +import cc.arduino.cli.settings.SettingsOuterClass; import io.grpc.ManagedChannel; import io.grpc.ManagedChannelBuilder; import processing.app.BaseNoGui; diff --git a/arduino-core/src/cc/arduino/cli/ArduinoCoreInstance.java b/arduino-core/src/cc/arduino/cli/ArduinoCoreInstance.java index 7fe0c1399..1ae28e3f1 100644 --- a/arduino-core/src/cc/arduino/cli/ArduinoCoreInstance.java +++ b/arduino-core/src/cc/arduino/cli/ArduinoCoreInstance.java @@ -31,6 +31,8 @@ package cc.arduino.cli; import static processing.app.I18n.tr; +import java.io.IOException; +import java.io.OutputStream; import java.util.ArrayList; import java.util.Iterator; import java.util.List; @@ -49,6 +51,7 @@ import cc.arduino.cli.commands.Common.Instance; import cc.arduino.cli.commands.Common.TaskProgress; import cc.arduino.cli.commands.Compile.CompileReq; import cc.arduino.cli.commands.Compile.CompileResp; +import cc.arduino.cli.commands.Compile.CompileResult; import cc.arduino.cli.commands.Lib.InstalledLibrary; import cc.arduino.cli.commands.Lib.LibraryInstallReq; import cc.arduino.cli.commands.Lib.LibraryInstallResp; @@ -88,21 +91,29 @@ public class ArduinoCoreInstance { } } - public void compile(String fqbn, String sketchPath) throws StatusException { + public CompileResult compile(CompileReq req, OutputStream out, + OutputStream err) throws StatusException { + req = CompileReq.newBuilder(req) // + .setInstance(instance) // + .build(); try { - CompileReq req = CompileReq.newBuilder() // - .setInstance(instance) // - .setFqbn(fqbn) // - .setSketchPath(sketchPath) // - .setVerbose(true) // - .build(); Iterator stream = stub.compile(req); + CompileResult result = CompileResult.error; while (stream.hasNext()) { CompileResp resp = stream.next(); - ByteString out = resp.getOutStream(); - if (out != null) - System.out.print(out.toStringUtf8()); + try { + ByteString outdata = resp.getOutStream(); + if (outdata != null) + out.write(outdata.toByteArray()); + ByteString errdata = resp.getErrStream(); + if (errdata != null) + err.write(errdata.toByteArray()); + } catch (IOException e) { + e.printStackTrace(); + } + result = resp.getResult(); } + return result; } catch (StatusRuntimeException e) { throw e.getStatus().asException(); }