Use a specific hardware/package_index_bundled.json. AVR core version to 1.6.13

Previously, during the build, the full package_index.json was downloaded
and distributed with the Arduino IDE.
This lead to a situation where it was difficult to test new AVR cores
before publishing them to the public package_index.json.

Now the bundled AVR core is specificed in the file:
`hardware/package_index_bundled.json`
this index is loaded from the IDE at startup and the package_index.json
is overlayed on it.

This should also solve part of #5143 (Repeatable builds and snapshots of
package/library indexes)

arduino-core/src/cc/arduino/Constants.java

@@ -39,6 +39,7 @@ public class Constants {
   public static final String PREF_CONTRIBUTIONS_TRUST_ALL = "contributions.trust.all";
 
   public static final String DEFAULT_INDEX_FILE_NAME = "package_index.json";
+  public static final String BUNDLED_INDEX_FILE_NAME = "package_index_bundled.json";
   public static final List<String> PROTECTED_PACKAGE_NAMES = Arrays.asList("arduino", "Intel");
 
   public static final String LIBRARY_DEVELOPMENT_FLAG_FILE = ".development";

arduino-core/src/cc/arduino/contributions/packages/ContributionsIndex.java

@@ -141,8 +141,4 @@ public abstract class ContributionsIndex {
       res += pack + "\n";
     return res;
   }
-
-  public void setTrusted() {
-    getPackages().stream().forEach(pack -> pack.setTrusted(true));
-  }
 }

arduino-core/src/cc/arduino/contributions/packages/ContributionsIndexer.java

@@ -82,12 +82,21 @@ public class ContributionsIndexer {
 
   public void parseIndex() throws Exception {
     File defaultIndexFile = getIndexFile(Constants.DEFAULT_INDEX_FILE_NAME);
+    File bundledIndexFile = new File(builtInHardwareFolder, Constants.BUNDLED_INDEX_FILE_NAME);
+
+    // Check main index signature
     if (!PreferencesData.getBoolean("allow_insecure_packages") && !signatureVerifier.isSigned(defaultIndexFile)) {
       throw new SignatureVerificationFailedException(Constants.DEFAULT_INDEX_FILE_NAME);
     }
-    index = parseIndex(defaultIndexFile);
-    index.setTrusted();
 
+    // Read bundled index and overlay the default index
+    index = parseIndex(bundledIndexFile);
+    mergeContributions(parseIndex(defaultIndexFile), defaultIndexFile);
+
+    // Set main and bundled indexes as trusted
+    index.getPackages().forEach(pack -> pack.setTrusted(true));
+
+    // Overlay 3rd party indexes
     File[] indexFiles = preferencesFolder.listFiles(new TestPackageIndexFilenameFilter(new PackageIndexFilenameFilter(Constants.DEFAULT_INDEX_FILE_NAME)));
 
     for (File indexFile : indexFiles) {
@@ -100,6 +109,7 @@ public class ContributionsIndexer {
       }
     }
 
+    // Fill tools and toolsDependency cross references
     List<ContributedPackage> packages = index.getPackages();
     Collection<ContributedPackage> packagesWithTools = packages.stream()
       .filter(input -> input.getTools() != null && !input.getTools().isEmpty())

build/build.xml

@@ -10,7 +10,6 @@
 
   <!-- Main url for boards and libraries repository -->
   <property name="library_index_url" value="http://downloads.arduino.cc/libraries/library_index.json" />
-  <property name="package_index_url" value="http://downloads.arduino.cc/packages/package_index.json" />
 
   <!-- Sets properties for macosx/windows/linux depending on current system -->
   <condition property="platform" value="macosx-old">
@@ -274,7 +273,7 @@
     </loadfile>
 
     <!-- figure out the latest AVR core version number in package_index.json -->
-    <loadfile srcfile="${staging_folder}/work/${staging_hardware_folder}/../dist/package_index.json" property="revision.avr.index">
+    <loadfile srcfile="${staging_folder}/work/${staging_hardware_folder}/package_index_bundled.json" property="revision.avr.index">
       <filterchain>
         <tokenfilter>
           <linetokenizer />
@@ -286,7 +285,7 @@
     </loadfile>
 
     <echo message="AVR Arduino core version in platform.txt is: ${revision.avr.platform}" />
-    <echo message="                    latest in index.json is: ${revision.avr.index}" />
+    <echo message="    latest in package_index_bundled.json is: ${revision.avr.index}" />
     <condition property="revision.avr.ok">
       <or>
         <!-- fail build if -Drevision.check=enforce is set -->
@@ -295,8 +294,8 @@
       </or>
     </condition>
     <fail unless="revision.avr.ok">
-      Mismatching versions for bundled AVR core and package_index.json.
+      Mismatching versions for bundled AVR core and package_index_bundled.json.
-      Please check your platform.txt and package_index.json.
+      Please check your platform.txt and package_index_bundled.json.
     </fail>
 
     <!-- figure out the IDE version number -->
@@ -1168,19 +1167,10 @@
   </target>
 
   <target name="package-library-index-json-bundle">
+    <mkdir dir="${staging_folder}/work/${staging_hardware_folder}"/>
+    <copy file="../hardware/package_index_bundled.json" tofile="${staging_folder}/work/${staging_hardware_folder}/package_index_bundled.json"/>
+
     <mkdir dir="${staging_folder}/work/${staging_hardware_folder}/../dist/"/>
-
-    <get src="${package_index_url}.gz"
-         dest="${staging_folder}/work/${staging_hardware_folder}/../dist/package_index.json.gz"
-         verbose="true" skipexisting="false" />
-    <gunzip src ="${staging_folder}/work/${staging_hardware_folder}/../dist/package_index.json.gz"
-            dest="${staging_folder}/work/${staging_hardware_folder}/../dist/"/>
-    <delete file="${staging_folder}/work/${staging_hardware_folder}/../dist/package_index.json.gz"/>
-
-    <get src="${package_index_url}.sig"
-         dest="${staging_folder}/work/${staging_hardware_folder}/../dist/package_index.json.sig"
-         verbose="true" skipexisting="false" />
-
     <get src="${library_index_url}.gz"
          dest="${staging_folder}/work/${staging_hardware_folder}/../dist/library_index.json.gz"
          verbose="true" skipexisting="false" />

hardware/arduino/avr/platform.txt

@@ -6,7 +6,7 @@
 # https://github.com/arduino/Arduino/wiki/Arduino-IDE-1.5-3rd-party-Hardware-specification
 
 name=Arduino AVR Boards
-version=1.6.12
+version=1.6.13
 
 # AVR compile variables
 # ---------------------

hardware/package_index_bundled.json

@@ -0,0 +1,146 @@
+{
+  "packages": [
+    {
+      "name": "arduino",
+      "maintainer": "Arduino",
+      "websiteURL": "http://www.arduino.cc/",
+      "email": "packages@arduino.cc",
+      "help": {
+        "online": "http://www.arduino.cc/en/Reference/HomePage"
+      },
+      "platforms": [
+        {
+          "name": "Arduino AVR Boards",
+          "architecture": "avr",
+          "version": "1.6.13",
+          "category": "Arduino",
+          "help": {
+            "online": "http://www.arduino.cc/en/Reference/HomePage"
+          },
+          "url": "http://downloads.arduino.cc/cores/avr-1.6.13.tar.bz2",
+          "archiveFileName": "avr-1.6.13.tar.bz2",
+          "checksum": "SHA-256:617f458dd3507072b9a6f9fdc78888c66aa420a2fb081c4c1556598a2d69d643",
+          "size": "4993644",
+          "boards": [
+            {"name": "Arduino Yún"},
+            {"name": "Arduino/Genuino Uno"},
+            {"name": "Arduino Diecimila"},
+            {"name": "Arduino Nano"},
+            {"name": "Arduino/Genuino Mega"},
+            {"name": "Arduino MegaADK"},
+            {"name": "Arduino Leonardo"},
+            {"name": "Arduino/Genuino Micro"},
+            {"name": "Arduino Esplora"},
+            {"name": "Arduino Mini"},
+            {"name": "Arduino Ethernet"},
+            {"name": "Arduino Fio"},
+            {"name": "Arduino BT"},
+            {"name": "Arduino LilyPadUSB"},
+            {"name": "Arduino Lilypad"},
+            {"name": "Arduino Pro"},
+            {"name": "Arduino ATMegaNG"},
+            {"name": "Arduino Robot Control"},
+            {"name": "Arduino Robot Motor"},
+            {"name": "Arduino Gemma"}
+          ],
+          "toolsDependencies": [
+            {
+              "packager": "arduino",
+              "name": "avr-gcc",
+              "version": "4.9.2-atmel3.5.3-arduino2"
+            },
+            {
+              "packager": "arduino",
+              "name": "avrdude",
+              "version": "6.0.1-arduino5"
+            }
+          ]
+        }
+      ],
+      "tools": [
+        {
+          "name": "avrdude",
+          "version": "6.0.1-arduino5",
+          "systems": [
+            {
+              "size": "267095",
+              "checksum": "SHA-256:23ea1341dbc117ec067f2eb1a498ad2bdd7d11fff0143c00b2e018c39804f6b4",
+              "host": "arm-linux-gnueabihf",
+              "archiveFileName": "avrdude-6.0.1-arduino5-armhf-pc-linux-gnu.tar.bz2",
+              "url": "http://downloads.arduino.cc/tools/avrdude-6.0.1-arduino5-armhf-pc-linux-gnu.tar.bz2"
+            },
+            {
+              "size": "264894",
+              "checksum": "SHA-256:41af8d3b0a586853c8317b4fb5163ca0db594a1870ddf680fd988c42166fc3e5",
+              "host": "i386-apple-darwin11",
+              "archiveFileName": "avrdude-6.0.1-arduino5-i386-apple-darwin11.tar.bz2",
+              "url": "http://downloads.arduino.cc/tools/avrdude-6.0.1-arduino5-i386-apple-darwin11.tar.bz2"
+            },
+            {
+              "size": "292629",
+              "checksum": "SHA-256:d826cca7383461f7e8adde686372cf900e9cb3afd639555cf2d6c645b283a476",
+              "host": "x86_64-linux-gnu",
+              "archiveFileName": "avrdude-6.0.1-arduino5-x86_64-pc-linux-gnu.tar.bz2",
+              "url": "http://downloads.arduino.cc/tools/avrdude-6.0.1-arduino5-x86_64-pc-linux-gnu.tar.bz2"
+            },
+            {
+              "size": "283121",
+              "checksum": "SHA-256:5933d66927bce46ababa9b68a8b7f1d53f68c4f3ff7a5ce4b85d7cf4e6c6bfee",
+              "host": "i686-linux-gnu",
+              "archiveFileName": "avrdude-6.0.1-arduino5-i686-pc-linux-gnu.tar.bz2",
+              "url": "http://downloads.arduino.cc/tools/avrdude-6.0.1-arduino5-i686-pc-linux-gnu.tar.bz2"
+            },
+            {
+              "size": "241634",
+              "checksum": "SHA-256:41f667f1f6a0ab8df46b4ffacd023176dcdef331d6db3b74bddd37d18cca0a44",
+              "host": "i686-mingw32",
+              "archiveFileName": "avrdude-6.0.1-arduino5-i686-mingw32.zip",
+              "url": "http://downloads.arduino.cc/tools/avrdude-6.0.1-arduino5-i686-mingw32.zip"
+            }
+          ]
+        },
+        {
+          "name": "avr-gcc",
+          "version": "4.9.2-atmel3.5.3-arduino2",
+          "systems": [
+            {
+              "size": "27400889",
+              "checksum": "SHA-256:77f300d519bc6b9a25df17b36cb303218e9a258c059b2f6bff8f71a0d8f96821",
+              "host": "arm-linux-gnueabihf",
+              "archiveFileName": "avr-gcc-4.9.2-atmel3.5.3-arduino2-armhf-pc-linux-gnu.tar.bz2",
+              "url": "http://downloads.arduino.cc/tools/avr-gcc-4.9.2-atmel3.5.3-arduino2-armhf-pc-linux-gnu.tar.bz2"
+            },
+            {
+              "size": "27048070",
+              "checksum": "SHA-256:311258af188defe24a4b341e4e1f4dc93ca6c80516d3e3b55a2fc07a7050248b",
+              "host": "i386-apple-darwin11",
+              "archiveFileName": "avr-gcc-4.9.2-atmel3.5.3-arduino2-i386-apple-darwin11.tar.bz2",
+              "url": "http://downloads.arduino.cc/tools/avr-gcc-4.9.2-atmel3.5.3-arduino2-i386-apple-darwin11.tar.bz2"
+            },
+            {
+              "size": "43847945",
+              "checksum": "SHA-256:f8e6ede8746c70be01ec79a30803277cd94360cc5b2e104762da0fbcf536fcc6",
+              "host": "i686-mingw32",
+              "archiveFileName": "avr-gcc-4.9.2-atmel3.5.3-arduino2-i686-mingw32.zip",
+              "url": "http://downloads.arduino.cc/tools/avr-gcc-4.9.2-atmel3.5.3-arduino2-i686-mingw32.zip"
+            },
+            {
+              "size": "29292729",
+              "checksum": "SHA-256:f108951e7c4dc90926d1fc76cc27549f6ea63c702a2bb7ff39647a19ae86ec68",
+              "host": "i686-linux-gnu",
+              "archiveFileName": "avr-gcc-4.9.2-atmel3.5.3-arduino2-i686-pc-linux-gnu.tar.bz2",
+              "url": "http://downloads.arduino.cc/tools/avr-gcc-4.9.2-atmel3.5.3-arduino2-i686-pc-linux-gnu.tar.bz2"
+            },
+            {
+              "size": "29882960",
+              "checksum": "SHA-256:3903a6d1bb9fdd91727e504b5993d5501f119bcb7f99f7aee98a2101e5629188",
+              "host": "x86_64-linux-gnu",
+              "archiveFileName": "avr-gcc-4.9.2-atmel3.5.3-arduino2-x86_64-pc-linux-gnu.tar.bz2",
+              "url": "http://downloads.arduino.cc/tools/avr-gcc-4.9.2-atmel3.5.3-arduino2-x86_64-pc-linux-gnu.tar.bz2"
+            }
+          ]
+        }
+      ]
+    }
+  ]
+}