From ba302ee9c8504d9865219bcd92c48a02af69e930 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Andreas=20Kr=C3=BCger?= <dj3ei@famsik.de>
Date: Fri, 10 Feb 2017 01:39:33 +0100
Subject: [PATCH] Properly decode UTF-8 characters comming in from serial one
 byte at a time.

This fixes #2430.
---
 arduino-core/src/processing/app/Serial.java | 49 ++++++++++++++++++---
 1 file changed, 44 insertions(+), 5 deletions(-)

diff --git a/arduino-core/src/processing/app/Serial.java b/arduino-core/src/processing/app/Serial.java
index b6b91fdc7..6a984d2f7 100644
--- a/arduino-core/src/processing/app/Serial.java
+++ b/arduino-core/src/processing/app/Serial.java
@@ -28,11 +28,17 @@ import jssc.SerialPortEventListener;
 import jssc.SerialPortException;
 
 import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.CharBuffer;
+import java.nio.charset.Charset;
+import java.nio.charset.CharsetDecoder;
+import java.nio.charset.CodingErrorAction;
+import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
 import java.util.List;
 
-import static processing.app.I18n.tr;
 import static processing.app.I18n.format;
+import static processing.app.I18n.tr;
 
 public class Serial implements SerialPortEventListener {
 
@@ -47,6 +53,12 @@ public class Serial implements SerialPortEventListener {
 
   private SerialPort port;
 
+  private CharsetDecoder bytesToStrings;
+  private static final int IN_BUFFER_CAPACITY = 128;
+  private static final int OUT_BUFFER_CAPACITY = 128;
+  private ByteBuffer inFromSerial = ByteBuffer.allocate(IN_BUFFER_CAPACITY);
+  private CharBuffer outToMessage = CharBuffer.allocate(OUT_BUFFER_CAPACITY);
+
   public Serial() throws SerialException {
     this(PreferencesData.get("serial.port"),
       PreferencesData.getInteger("serial.debug_rate", 9600),
@@ -101,6 +113,8 @@ public class Serial implements SerialPortEventListener {
     //this.parent = parent;
     //parent.attach(this);
 
+    resetDecoding(StandardCharsets.UTF_8);
+
     int parity = SerialPort.PARITY_NONE;
     if (iparity == 'E') parity = SerialPort.PARITY_EVEN;
     if (iparity == 'O') parity = SerialPort.PARITY_ODD;
@@ -153,10 +167,24 @@ public class Serial implements SerialPortEventListener {
     if (serialEvent.isRXCHAR()) {
       try {
         byte[] buf = port.readBytes(serialEvent.getEventValue());
-        if (buf.length > 0) {
-          String msg = new String(buf);
-          char[] chars = msg.toCharArray();
-          message(chars, chars.length);
+        int next = 0;
+        while(next < buf.length) {
+          while(next < buf.length && outToMessage.hasRemaining()) {
+            int spaceInIn = inFromSerial.remaining();
+            int copyNow = buf.length - next < spaceInIn ? buf.length - next : spaceInIn;
+            inFromSerial.put(buf, next, copyNow);
+            next += copyNow;
+            inFromSerial.flip();
+            bytesToStrings.decode(inFromSerial, outToMessage, false);
+            inFromSerial.compact();
+          }
+          outToMessage.flip();
+          if(outToMessage.hasRemaining()) {
+            char[] chars = new char[outToMessage.remaining()];
+            outToMessage.get(chars);
+            message(chars, chars.length);
+          }
+          outToMessage.clear();
         }
       } catch (SerialPortException e) {
         errorMessage("serialEvent", e);
@@ -226,6 +254,17 @@ public class Serial implements SerialPortEventListener {
     }
   }
 
+  /**
+   * Reset the encoding used to convert the bytes coming in
+   * before they are handed as Strings to {@Link #message(char[], int)}.
+   */
+  public synchronized void resetDecoding(Charset charset) {
+    bytesToStrings = charset.newDecoder()
+                      .onMalformedInput(CodingErrorAction.REPLACE)
+                      .onUnmappableCharacter(CodingErrorAction.REPLACE)
+                      .replaceWith("\u2e2e");
+  }
+
   static public List<String> list() {
     return Arrays.asList(SerialPortList.getPortNames());
   }