Improving detection of functions that need to be prototyped; no longer moving #include's to the top of the sketch. Actually putting todo items in todo.txt.

app/preproc/PdePreprocessor.java

@@ -33,6 +33,8 @@ import processing.app.*;
 import processing.core.*;
 
 import java.io.*;
+import java.util.ArrayList;
+import java.util.List;
 
 import antlr.*;
 import antlr.collections.*;
@@ -47,7 +49,7 @@ public class PdePreprocessor {
   static final int JDK13 = 1;
   static final int JDK14 = 2;
 
-  static String defaultImports[][] = new String[3][];
+  //static String defaultImports[][] = new String[3][];
 
   // these ones have the .* at the end, since a class name
   // might be at the end instead of .* whcih would make trouble
@@ -55,10 +57,6 @@ public class PdePreprocessor {
   // it to produce a package name consistently.
   public String extraImports[];
 
-  // imports just from the code folder, treated differently
-  // than the others, since the imports are auto-generated.
-  public String codeFolderImports[];
-
   static public final int STATIC = 0;  // formerly BEGINNER
   static public final int ACTIVE = 1;  // formerly INTERMEDIATE
   static public final int JAVA   = 2;  // formerly ADVANCED
@@ -68,9 +66,6 @@ public class PdePreprocessor {
   Reader programReader;
   String buildPath;
 
-  // used for calling the ASTFactory to get the root node
-  private static final int ROOT_ID = 0;
-  
   // stores number of built user-defined function prototypes
   public int prototypeCount = 0;
 
@@ -81,21 +76,97 @@ public class PdePreprocessor {
    * These may change in-between (if the prefs panel adds this option)
    * so grab them here on construction.
    */
-  public PdePreprocessor() {
+  public PdePreprocessor() {}
-/*    defaultImports[JDK11] =
-      Base.split(Preferences.get("preproc.imports.jdk11"), ',');
-    defaultImports[JDK13] =
-      Base.split(Preferences.get("preproc.imports.jdk13"), ',');
-    defaultImports[JDK14] =
-      Base.split(Preferences.get("preproc.imports.jdk14"), ',');
-*/  }
-
 
   /**
    * Used by PdeEmitter.dumpHiddenTokens()
    */
   //public static TokenStreamCopyingHiddenTokenFilter filter;
   
+  /**
+   * Strips comments, pre-processor directives, single- and double-quoted
+   * strings from a string.
+   * @param in the String to strip
+   * @return the stripped String
+   */
+  public String strip(String in) throws MalformedPatternException {
+    PatternCompiler compiler = new Perl5Compiler();
+    PatternMatcher matcher = new Perl5Matcher();
+    Pattern pattern = compiler.compile(
+      // XXX: doesn't properly handle special single-quoted characters
+      // single-quoted character
+      "('.')" + "|" +
+      // double-quoted string
+      "(\"(?:[^\"\\\\]|\\\\.)*\")" + "|" +
+      // multi-line comment
+      "(/\\*(?:.|\\n)*?\\*/)" + "|" + 
+      // single-line comment
+      "(//.*?$)" + "|" +
+      // pre-processor directive
+      "(^\\s*#.*?$)",
+      Perl5Compiler.MULTILINE_MASK);
+      
+    while (matcher.contains(in, pattern)) {
+      MatchResult result = matcher.getMatch();
+      // XXX: should preserve newlines in the result so that line numbers of
+      // the stripped string correspond to those in the original source.
+      in = in.substring(0, result.beginOffset(0)) + " " + in.substring(result.endOffset(0));
+    }
+    
+    return in;
+  }
+  
+  /**
+   * Removes the contents of all top-level curly brace pairs {}.
+   * @param in the String to collapse
+   * @return the collapsed String
+   */
+  private String collapseBraces(String in) {
+    StringBuffer buffer = new StringBuffer();
+    int nesting = 0;
+    int start = 0;
+    
+    // XXX: need to keep newlines inside braces so we can determine the line
+    // number of a prototype
+    for (int i = 0; i < in.length(); i++) {
+      if (in.charAt(i) == '{') {
+        if (nesting == 0) {
+          buffer.append(in.substring(start, i + 1));  // include the '{'
+        }
+        nesting++;
+      }
+      if (in.charAt(i) == '}') {
+        nesting--;
+        if (nesting == 0) {
+          start = i; // include the '}'
+        }
+      }
+    }
+    
+    buffer.append(in.substring(start));
+    
+    return buffer.toString();
+  }
+  
+  private List prototypes(String in) throws MalformedPatternException {
+    in = collapseBraces(strip(in));
+    
+    PatternMatcherInput input = new PatternMatcherInput(in);
+    PatternCompiler compiler = new Perl5Compiler();
+    PatternMatcher matcher = new Perl5Matcher();
+    // XXX: doesn't handle ... varargs
+    // XXX: doesn't handle function pointers
+    Pattern pattern = compiler.compile(
+      "[\\w\\[\\]\\*]+\\s+[\\[\\]\\*\\w\\s]+\\([,\\[\\]\\*\\w\\s]*\\)(?=\\s*\\{)");
+    List matches = new ArrayList();
+    
+    while (matcher.contains(input, pattern)) {
+      matches.add(matcher.getMatch().group(0) + ";");
+    }
+    
+    return matches;
+  }
+
 
   /**
    * preprocesses a pde file and write out a java file
@@ -167,7 +238,7 @@ public class PdePreprocessor {
     PatternCompiler compiler = new Perl5Compiler();
     //String mess = "^\\s*(import\\s+\\S+\\s*;)";
     //String mess = "^\\s*(import\\s+)(\\S+)(\\s*;)";
-    String mess = "^\\s*(#include\\s+[<\"])(\\S+)([\">]\\s*)";
+    String mess = "^\\s*#include\\s+[<\"](\\S+)[\">]";
     java.util.Vector imports = new java.util.Vector();
 
     Pattern pattern = null;
@@ -178,184 +249,35 @@ public class PdePreprocessor {
       return null;
     }
 
-    do {
     PatternMatcherInput input = new PatternMatcherInput(program);    
-      if (!matcher.contains(input, pattern)) break;
+    while (matcher.contains(input, pattern)) {
-
+      imports.add(matcher.getMatch().group(1));
-      MatchResult result = matcher.getMatch();
+    }    
-      String piece1 = result.group(1).toString();
-      String piece2 = result.group(2).toString();  // the header file name w/o quotes
-      String piece3 = result.group(3).toString();
-      String piece = piece1 + piece2 + piece3;
-      int len = piece.length();
-
-      //imports.add(piece);
-      imports.add(piece2);
-      int idx = program.indexOf(piece);
-      // just remove altogether?
-      program = program.substring(0, idx) + program.substring(idx + len);
-
-      //System.out.println("removing " + piece);
-
-    } while (true);
-    
 
     extraImports = new String[imports.size()];
     imports.copyInto(extraImports);
 
-    if (codeFolderPackages != null) {
-      codeFolderImports = new String[codeFolderPackages.length];
-      for (int i = 0; i < codeFolderPackages.length; i++) {
-        codeFolderImports[i] = codeFolderPackages[i] + ".*";
-      }
-    } else {
-      codeFolderImports = null;
-    }
-
     // do this after the program gets re-combobulated
     this.programReader = new StringReader(program);
     this.buildPath = buildPath;
     
-    // create function prototypes
+    List prototypes = prototypes(program);
-    mess = "^(\\w+)\\s+(\\w+)\\s*\\(([^)]*)\\)\\s*{";
+    
-    pattern = null;
-    try {
-      pattern = compiler.compile(mess);
-    } catch (MalformedPatternException e) {
-      e.printStackTrace();
-      return null;
-    }
-    PatternMatcherInput input = new PatternMatcherInput(program);
-    MatchResult result;
-    String returntype, functioname, parameterlist, prototype;
-    java.util.LinkedList prototypes = new java.util.LinkedList();
-    //System.out.println("prototypes:");
-    while(matcher.contains(input, pattern)){
-      result = matcher.getMatch();
-      //System.out.println(result);
-      returntype = result.group(1).toString();
-      functioname = result.group(2).toString();
-      parameterlist = result.group(3).toString().replace('\n', ' ');
-      prototype = returntype + " " + functioname + "(" + parameterlist + ");";
-      if(0 == functioname.compareTo("setup")){
-        continue;
-      }
-      if(0 == functioname.compareTo("loop")){
-        continue;
-      }
-      prototypes.add(prototype);
-      //System.out.println(prototype);
-    }
     // store # of prototypes so that line number reporting can be adjusted
     prototypeCount = prototypes.size();
   
-
-    // create a lexer with the stream reader, and tell it to handle
-    // hidden tokens (eg whitespace, comments) since we want to pass these
-    // through so that the line numbers when the compiler reports errors
-    // match those that will be highlighted in the PDE IDE
-    //
-    //System.out.println(program);
-    // DAM:WTF? WLexer lexer  = new WLexer(programReader);
-    //lexer.setTokenObjectClass("antlr.CommonHiddenStreamToken");
-    // DAM:WTF? lexer.setTokenObjectClass("processing.app.preproc.CToken");
-    // DAM:WTF? lexer.initialize();
-
-    // create the filter for hidden tokens and specify which tokens to
-    // hide and which to copy to the hidden text
-    //
-    /*filter = new TokenStreamCopyingHiddenTokenFilter(lexer);
-    filter.hide(WParser.CPPComment);
-    filter.hide(WParser.Comment);
-    filter.hide(WParser.Whitespace);
-    filter.copy(WParser.SEMI);
-    filter.copy(WParser.LPAREN);
-    filter.copy(WParser.RPAREN);
-    filter.copy(WParser.LCURLY);
-    filter.copy(WParser.RCURLY);
-    filter.copy(WParser.COMMA);
-    filter.copy(WParser.RBRACK);
-    filter.copy(WParser.LBRACK);
-    filter.copy(WParser.COLON);
-    */
-    // create a parser and set what sort of AST should be generated
-    //
-    //PdeRecognizer parser = new PdeRecognizer(filter);
-    // DAM:WTF? WParser parser = new WParser(lexer);
-
-    // use our extended AST class
-    //
-    //parser.setASTNodeClass("antlr.ExtendedCommonASTWithHiddenTokens");
-    // DAM:WTF? parser.setASTNodeType(TNode.class.getName());
-    // DAM:WTF? TNode.setTokenVocabulary("processing.app.preproc.WTokenTypes");
-
-    // start parsing at the compilationUnit non-terminal
-    //
-    //parser.pdeProgram();
-    //parser.translationUnit();
-
-    // set up the AST for traversal by PdeEmitter
-    //
-    // DAM:WTF? ASTFactory factory = new ASTFactory();
-    // DAM:WTF? AST parserAST = parser.getAST();
-    // DAM:WTF? AST rootNode = factory.create(ROOT_ID, "AST ROOT");
-    // DAM:WTF? rootNode.setFirstChild(parserAST);
-
-    // unclear if this actually works, but it's worth a shot
-    //
-    //((CommonAST)parserAST).setVerboseStringConversion(
-    //  true, parser.getTokenNames());
-    // (made to use the static version because of jikes 1.22 warning)
-    // DAM:WTF? CommonAST.setVerboseStringConversion(true, parser.getTokenNames());
-
-    // if this is an advanced program, the classname is already defined.
-    //
-    // DAM:WTF? if (programType == JAVA) {
-    // DAM:WTF?   name = getFirstClassName(parserAST);
-    // DAM:WTF? }
-
-    // if 'null' was passed in for the name, but this isn't
-    // a 'java' mode class, then there's a problem, so punt.
-    //
     if (name == null) return null;
 
     // output the code
-    //
-    // DAM:WTF? WEmitter emitter = new WEmitter(lexer.getPreprocessorInfoChannel());
     File streamFile = new File(buildPath, name + ".cpp");
     PrintStream stream = new PrintStream(new FileOutputStream(streamFile));
 
-    //writeHeader(stream, extraImports, name);
     writeHeader(stream, name, prototypes);
     //added to write the pde code to the cpp file
     writeProgram(stream, name, program);
-    // DAM:WTF? emitter.setASTNodeType(TNode.class.getName());
-    // DAM:WTF? emitter.setOut(stream);
-    //emitter.printDeclarations(rootNode);
-    //emitter.print(rootNode);
-    //emitter.translationUnit(parser.getAST());
-
     writeFooter(stream, target);
     stream.close();
 
-    // if desired, serialize the parse tree to an XML file.  can
-    // be viewed usefully with Mozilla or IE
-    /* DAM:WTF?
-    if (Preferences.getBoolean("preproc.output_parse_tree")) {
-
-      stream = new PrintStream(new FileOutputStream("parseTree.xml"));
-      stream.println("<?xml version=\"1.0\"?>");
-      stream.println("<document>");
-      OutputStreamWriter writer = new OutputStreamWriter(stream);
-      if (parserAST != null) {
-        ((CommonAST)parserAST).xmlSerialize(writer);
-      }
-      writer.flush();
-      stream.println("</document>");
-      writer.close();
-    }
-    */
-
     return name;
   }
 
@@ -372,77 +294,22 @@ public class PdePreprocessor {
    * @param exporting           Is this being exported from PDE?
    * @param name                Name of the class being created.
    */
-  void writeHeader(PrintStream out, String className, java.util.LinkedList prototypes)
+  void writeHeader(PrintStream out, String className, List prototypes)
     throws IOException {
     out.print("#include \"WProgram.h\"\n");
-    /* Arduino doesn't automatically use all available libraries.
-    // print library headers
-    LibraryManager libraryManager = new LibraryManager();
-    String[] headerFiles = libraryManager.getHeaderFiles();
-    for(int i = 0; i < headerFiles.length; ++i){
-      out.print("#include \"" + headerFiles[i] + "\"\n");
-    }
     
-    // record number of header lines written for error line adjustment
-    headerCount = headerFiles.length;
-    */
     // print user defined prototypes
-    while(0 < prototypes.size()){
+    for (int i = 0; i < prototypes.size(); i++) {
-      out.print(prototypes.removeFirst() + "\n");
+      out.print(prototypes.get(i) + "\n");
     }
 
-    // emit emports that are needed for classes from the code folder
+//    // emit emports that are needed for classes from the code folder
-    if (extraImports != null) {
+//    if (extraImports != null) {
-      for (int i = 0; i < extraImports.length; i++) {
+//      for (int i = 0; i < extraImports.length; i++) {
-        out.print("#include \"" + extraImports[i] + "\"\n");
+//        out.print("#include \"" + extraImports[i] + "\"\n");
+//      }
+//    }
   }
-    }
-
-    if (codeFolderImports != null) {
-      for (int i = 0; i < codeFolderImports.length; i++) {
-        out.print("import " + codeFolderImports[i] + "; ");
-      }
-    }
-
-    // emit standard imports (read from pde.properties)
-    // for each language level that's being used.
-/*    String jdkVersionStr = Preferences.get("preproc.jdk_version");
-
-    int jdkVersion = JDK11;  // default
-    if (jdkVersionStr.equals("1.3")) { jdkVersion = JDK13; };
-    if (jdkVersionStr.equals("1.4")) { jdkVersion = JDK14; };
-
-    for (int i = 0; i <= jdkVersion; i++) {
-      for (int j = 0; j < defaultImports[i].length; j++) {
-        out.print("import " + defaultImports[i][j] + ".*; ");
-      }
-    }
-
-    //boolean opengl = Preferences.get("renderer").equals("opengl");
-    //if (opengl) {
-    //out.println("import processing.opengl.*; ");
-    //}
-*/
-/*    if (programType < JAVA) {
-      // open the class definition
-      out.print("public class " + className + " extends ");
-      //if (opengl) {
-      //out.print("PAppletGL");
-      //} else {
-      out.print("PApplet");
-      //}
-      out.print(" {");
-
-      if (programType == STATIC) {
-        // now that size() and background() can go inside of draw()
-        // actually, use setup(), because when running externally
-        // the applet size needs to be set before the window is drawn,
-        // meaning that after setup() things need to be ducky.
-        //out.print("public void draw() {");
-        out.print("public void setup() {");
-      }
-    }
-*/  }
 
   /**
    * Write any necessary closing text.

readme.txt

@@ -54,6 +54,7 @@ UPDATES
 * Added interrupts() and noInterrupts() functions.
 * Added degrees() and radians() functions.
 * Support for uploading sketch using a programmer (upload.using preference).
+* Including a working version of the Firmata firmware.
 * New script for downloading the reference from Tom Pollard.  Thanks Tom!
 * Miscellaneous Mac OS X and other patches from Wim Lewis.  Thanks Wim!
 

todo.txt

@@ -1 +1,81 @@
 0011 arduino
+
+0011
+
+Improve preprocessing of sketches:
+ - Don't move #include statements.
+ - Insert prototypes as a better spot in the code (after pre-processor directives).
+ - Better determine which header files are included (not commented out).
+ - Remember the original locations of function prototypes to highlight the correct line on error.
+ - [done] Better determine which functions need prototypes
+Update version of the FTDI drivers.
+Floating point support in the map() function.
+Modify parallel port programmer burning (add -F, lower or remove delay).
+Incorporate ladyada's new SoftwareSerial library.
+Add timeout parameter to pulseIn().
+[done] Allow uploading using a hardware programmer.
+[done] Add analogReference() function.
+[done] Add miscellaneous #defines (interrupts(), int(), etc.)
+[done] Add map() function.
+[done] Fix find in reference.
+[done] Include stable version of Firmata in hardware/firmwares.
+
+AVR
+
+Add support for ATmega1280 (e.g. timers, external interrupts, multiple serial ports).
+Good way to receive multiple values / complex messages over the serial port.
+Add pulseOut(), etc. functions from Wiring.
+Add String library.
+Create Encoder library (but don't include in the distribution).
+Create Ping library (but don't include in the distribution).
+Include Arduino as AVR-ISP sketch in hardware/firmwares.
+
+COMPUTER
+
+Easier library discovery and installation.
+Easier board installation.
+Clean up Library and LibraryManager.
+Byte-based serial monitor.
+Investigate method for auto-detecting serial port on Windows.
+Guess serial port on the Mac and Linux.
+Automatic detection of baud rate for serial monitor (based on the call to Serial.begin() in the current sketch).
+Improve, generally, the upload experience (e.g. faster program start after upload, keep-alive messages to bootloader from IDE, shorter bootloader timeout if possible, progress bar)
+Enable drag-and-drop of .pde files to Arduino dock icon.
+Allow uploading of .hex files.
+Allow for arbitrary compilation command line arguments.
+Find in reference should give same message for missing page as for missing page association.
+Test find in reference on libraries.
+
+DEVELOPMENT
+
+Include newer version of avr-gcc (one that supports the ATmega1280).
+Revise fetch.sh to look for version specific pages (names suffixed with, e.g., "-0007")
+If possible, install bug tracking and svn on arduino.cc.
+Move to ant for build process (limitations: can't build bootloader, can't build Windows arduino.exe, etc.)
+
+DOCUMENTATION
+
+Electronics reference a la Wiring.
+Documentation for moving from Arduino to custom PCBs.
+Examples should demonstrate use of functions.
+Add examples using specific hardware.
+Programming tutorial for Arduino.
+Write advanced library tutorial.
+Work on opening up website to public editing.
+Document optocoupler.
+Replace rainbow diagram on the guide to the board page.
+Upload photo of single sided Arduino board.
+Update pictures to use Arduino Diecimila: Guide/Board, tutorials, 
+Create an example for using a simple analog sensor like an LDR, PWM.
+Create diagrams and schematics for the examples.
+Create community section of site.
+Create form for submitting workshops.
+Create form for submitting projects.
+Create community to do list.
+
+Getting Started:
+Consider deleting many of the pictures in the howto's as they just make it harder to see the instructions without adding much (e.g. the pictures of files in folders).
+Tell people not to put the board on a Powerbook.
+People don't know what a jumper is.
+Add picture of the RX/TX LEDs flashing.
+Show a picture of the LED flashing.