Merged in james-duley/librepilot/LP-113_tool_installer (pull request #49)

LP-113_tool_installer

make/scripts/win_sdk_install.sh

@@ -36,12 +36,10 @@ fi
 # Tools URLs to fetch
 WGET_URL="http://librepilot.github.io/tools/wget.exe"
 MAKE_URL="http://librepilot.github.io/tools/make.exe"
-SEVENZIP_URL="http://librepilot.github.io/tools/7za.exe"
 
 # Expected tools paths
 WGET="$TOOLS_DIR/bin/`basename \"$WGET_URL\"`"
 MAKE="$TOOLS_DIR/bin/`basename \"$MAKE_URL\"`"
-SEVENZIP="$TOOLS_DIR/bin/`basename \"$SEVENZIP_URL\"`"
 
 # wget is necessary to fetch other files
 WGET_NAME="`basename \"$WGET\"`"
@@ -97,20 +95,9 @@ if [ ! -x "$MAKE" ]; then
     fi
 fi
 
-# 7-Zip is necessary to install some SDKs
-if [ ! -x "$SEVENZIP" ]; then
-    echo "$SCRIPT_NAME: $SEVENZIP_NAME not found, fetching from $SEVENZIP_URL"
-    SEVENZIP_DIR="`dirname \"$SEVENZIP\"`"
-    mkdir -p "$SEVENZIP_DIR"
-    $WGET --no-check-certificate -N --content-disposition -P "$SEVENZIP_DIR" "$SEVENZIP_URL"
-    if [ $? -ne 0 ]; then
-        echo "$SCRIPT_NAME: $SEVENZIP_NAME fetch error, hope it's in the path..."
-        SEVENZIP_NAME="`basename \"$SEVENZIP\"`"
-        SEVENZIP="$SEVENZIP_NAME"
-    fi
-fi
 
 # Finally we can fetch all SDKs using top level Makefile
 cd "$ROOT_DIR"
+./tool_install.sh 7z
 echo "Run 'tools/bin/make all_sdk_install' to install the other tools"
 echo " or 'tools/bin/make help' for more info on make targets"

make/tool_install/7z.sh

@@ -0,0 +1,31 @@
+# tool_url="http://www.7-zip.org/a/7z920.exe"
+#tool_url="http://www.7-zip.org/a/7z920.msi"
+tool_url="http://librepilot.github.io/tools/7za.exe"
+
+tool_install_name="bin/7za.exe"
+tool_extract_dir="$tools_dir/bin"
+
+bin_dir=$tool_extract_dir
+
+no_extract=true
+
+module_file=bin
+
+#Override
+function validate_target { [ "$uname" = Windows ]; }
+
+# Override
+function dddownload_and_extract
+{
+	local full_tool_install_name="$tools_dir/bin/$tool_install_name"
+	if ! [ -e  "$full_tool_install_name" ] || $force
+	then
+		download_and_verify && \
+		rm -rf "$full_tool_install_name" && \
+		mkdir -p "$(dirname "$full_tool_install_name")" && \
+		mv "$downloaded_file" "$full_tool_install_name"
+		#msiexec //i "$downloaded_file" //q INSTALLDIR="$tools_dir"
+		#cmd //C "$downloaded_file" /S /D="${tools_dir//\//\\}"
+	fi
+}
+

make/tool_install/gcc-arm-none-eabi.sh

@@ -0,0 +1,29 @@
+if [ "$uname" = Linux ]
+then
+	url_ext="linux.tar.bz2"
+elif [ "$uname" = Darwin ]
+then
+	url_ext="mac.tar.bz2"
+elif [ "$uname" = Windows ]
+then
+	url_ext="win32.zip"
+	depends=(7z)
+fi
+
+pkgver=4.9_2015_q2_update
+pkgdate=20150609
+_pkgver=${pkgver//_/-}
+_pkgvershort=${_pkgver%-*}
+_pkgvershort=${_pkgvershort/-q/q}
+
+tool_url="https://launchpad.net/gcc-arm-embedded/${pkgver%%_*}/${_pkgver}/+download/${tool}-${_pkgvershort/./_}-${pkgdate}-${url_ext}"
+tool_md5_url="${tool_url}/+md5"
+tool_install_name="${tool}-${_pkgvershort/./_}"
+if [ "$uname" = Windows ]
+then
+	tool_extract_dir=$tools_dir/$tool_install_name
+fi
+
+bin_subdir=bin
+
+function validate_target { true; }

make/tools.mk

@@ -51,6 +51,35 @@ ifndef TOP_LEVEL_MAKEFILE
     $(error $(notdir $(lastword $(MAKEFILE_LIST))) should be included by the top level Makefile)
 endif
 
+##############################
+#
+# Already installed tools modules
+#
+##############################
+-include $(wildcard $(TOOLS_DIR)/*.mk)
+
+TOOL_INSTALL := $(ROOT_DIR)/tool_install.sh
+TOOL_TARGETS := gcc-arm-none-eabi
+
+TOOL_INSTALL_TARGETS       := $(addsuffix _install,$(TOOL_TARGETS))
+TOOL_FORCE_INSTALL_TARGETS := $(addsuffix _force_install,$(TOOL_TARGETS))
+TOOL_REMOVE_TARGETS        := $(addsuffix _remove,$(TOOL_TARGETS))
+
+.PHONY: $(TOOL_INSTALL_TARGETS)
+$(TOOL_INSTALL_TARGETS):
+	@$(ECHO) $(MSG_INSTALLING) $(@:_install=)
+	$(V1) $(TOOL_INSTALL) -n $(@:_install=)
+
+.PHONY: $(TOOL_FORCE_INSTALL_TARGETS)
+$(TOOL_FORCE_INSTALL_TARGETS):
+	@$(ECHO) $(MSG_INSTALLING) $(@:_install=)
+	$(V1) $(TOOL_INSTALL) -n -f $(@:_force_install=)
+
+.PHONY: $(TOOL_REMOVE_TARGETS)
+$(TOOL_REMOVE_TARGETS):
+	@$(ECHO) $(MSG_CLEANING) $(@:_install=)
+	$(V1) $(TOOL_INSTALL) -n -r $(@:_remove=)
+
 ##############################
 #
 # Toolchain URLs and directories
@@ -59,15 +88,11 @@ endif
 
 ifeq ($(UNAME), Linux)
     ifeq ($(ARCH), x86_64)
-        ARM_SDK_URL := https://launchpad.net/gcc-arm-embedded/4.9/4.9-2014-q4-major/+download/gcc-arm-none-eabi-4_9-2014q4-20141203-linux.tar.bz2
-        ARM_SDK_MD5_URL:= https://launchpad.net/gcc-arm-embedded/4.9/4.9-2014-q4-major/+download/gcc-arm-none-eabi-4_9-2014q4-20141203-linux.tar.bz2/+md5
         QT_SDK_URL  := http://download.qt.io/official_releases/qt/5.4/5.4.1/qt-opensource-linux-x64-5.4.1.run
         QT_SDK_MD5_URL := http://download.qt.io/official_releases/qt/5.4/5.4.1/qt-opensource-linux-x64-5.4.1.run.md5
         QT_SDK_ARCH := gcc_64
         OSG_URL        := http://librepilot.github.io/tools/osg-3.4-linux-x64-qt-5.4.1.tar.gz
     else
-        ARM_SDK_URL := https://launchpad.net/gcc-arm-embedded/4.9/4.9-2014-q4-major/+download/gcc-arm-none-eabi-4_9-2014q4-20141203-linux.tar.bz2
-        ARM_SDK_MD5_URL := https://launchpad.net/gcc-arm-embedded/4.9/4.9-2014-q4-major/+download/gcc-arm-none-eabi-4_9-2014q4-20141203-linux.tar.bz2/+md5
         QT_SDK_URL  := http://download.qt.io/official_releases/qt/5.4/5.4.1/qt-opensource-linux-x86-5.4.1.run
         QT_SDK_MD5_URL := http://download.qt.io/official_releases/qt/5.4/5.4.1/qt-opensource-linux-x86-5.4.1.run.md5
         QT_SDK_ARCH := gcc
@@ -76,8 +101,6 @@ ifeq ($(UNAME), Linux)
     UNCRUSTIFY_URL := http://librepilot.github.io/tools/uncrustify-0.60.tar.gz
     DOXYGEN_URL    := http://librepilot.github.io/tools/doxygen-1.8.3.1.src.tar.gz
 else ifeq ($(UNAME), Darwin)
-    ARM_SDK_URL    := https://launchpad.net/gcc-arm-embedded/4.9/4.9-2014-q4-major/+download/gcc-arm-none-eabi-4_9-2014q4-20141203-mac.tar.bz2
-    ARM_SDK_MD5_URL:= https://launchpad.net/gcc-arm-embedded/4.9/4.9-2014-q4-major/+download/gcc-arm-none-eabi-4_9-2014q4-20141203-mac.tar.bz2/+md5
     QT_SDK_URL  := http://download.qt.io/official_releases/qt/5.4/5.4.1/qt-opensource-mac-x64-clang-5.4.1.dmg
     QT_SDK_MD5_URL := http://download.qt.io/official_releases/qt/5.4/5.4.1/qt-opensource-mac-x64-clang-5.4.1.dmg.md5
     QT_SDK_ARCH := clang_64
@@ -88,8 +111,6 @@ else ifeq ($(UNAME), Darwin)
     DOXYGEN_URL    := http://librepilot.github.io/tools/doxygen-1.8.3.1.src.tar.gz
     OSG_URL        :=
 else ifeq ($(UNAME), Windows)
-    ARM_SDK_URL    := https://launchpad.net/gcc-arm-embedded/4.9/4.9-2014-q4-major/+download/gcc-arm-none-eabi-4_9-2014q4-20141203-win32.zip
-    ARM_SDK_MD5_URL:= https://launchpad.net/gcc-arm-embedded/4.9/4.9-2014-q4-major/+download/gcc-arm-none-eabi-4_9-2014q4-20141203-win32.zip/+md5
     QT_SDK_URL     := http://download.qt.io/official_releases/qt/5.4/5.4.1/qt-opensource-windows-x86-mingw491_opengl-5.4.1.exe
     QT_SDK_MD5_URL := http://download.qt.io/official_releases/qt/5.4/5.4.1/qt-opensource-windows-x86-mingw491_opengl-5.4.1.exe.md5
     QT_SDK_ARCH    := mingw491_32
@@ -109,7 +130,6 @@ GTEST_URL := http://librepilot.github.io/tools/gtest-1.6.0.zip
 CCACHE_URL     := http://samba.org/ftp/ccache/ccache-3.2.2.tar.bz2
 CCACHE_MD5_URL := http://librepilot.github.io/tools/ccache-3.2.2.tar.bz2.md5
 
-ARM_SDK_DIR    := $(TOOLS_DIR)/gcc-arm-none-eabi-4_9-2014q4
 QT_SDK_DIR     := $(TOOLS_DIR)/qt-5.4.1
 UNCRUSTIFY_DIR := $(TOOLS_DIR)/uncrustify-0.60
 DOXYGEN_DIR    := $(TOOLS_DIR)/doxygen-1.8.3.1
@@ -581,26 +601,14 @@ endef
 # ARM SDK
 #
 ##############################
+export ARM_SDK_PREFIX := arm-none-eabi-
+ARM_SDK_TOOL := gcc-arm-none-eabi
 
-ifeq ($(UNAME), Windows)
+.PHONY: arm_sdk_install
+arm_sdk_install: $(ARM_SDK_TOOL)_install
 
-# unfortunately zip package for this release is missing root directory, so adding / at the end of the path
+.PHONY: arm_sdk_clean
-# so that template interpret last part as directory and use the full path
+arm_sdk_clean: $(ARM_SDK_TOOL)_remove
-$(eval $(call TOOL_INSTALL_TEMPLATE,arm_sdk,$(ARM_SDK_DIR)/,$(ARM_SDK_URL),$(ARM_SDK_MD5_URL),$(notdir $(ARM_SDK_URL))))
-
-else
-
-$(eval $(call TOOL_INSTALL_TEMPLATE,arm_sdk,$(ARM_SDK_DIR),$(ARM_SDK_URL),$(ARM_SDK_MD5_URL),$(notdir $(ARM_SDK_URL))))
-
-endif
-
-ifeq ($(shell [ -d "$(ARM_SDK_DIR)" ] && $(ECHO) "exists"), exists)
-    export ARM_SDK_PREFIX := $(ARM_SDK_DIR)/bin/arm-none-eabi-
-else
-    # not installed, hope it's in the path...
-    # $(info $(EMPTY) WARNING     $(call toprel, $(ARM_SDK_DIR)) not found (make arm_sdk_install), using system PATH)
-    export ARM_SDK_PREFIX ?= arm-none-eabi-
-endif
 
 .PHONY: arm_sdk_version
 arm_sdk_version:

tool_install.sh

@@ -0,0 +1,272 @@
+#!/bin/bash
+
+# Exit if an error or an unset variable
+set -e -u
+
+# make sure unmatched glob gives empty
+shopt -s nullglob
+
+root_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+tools_dir=${TOOLS_DIR:-${root_dir}/tools}
+downloads_dir=${DL_DIR:-${root_dir}/downloads}
+tool_overrides_dir=$root_dir/make/tool_install
+
+batch=${BATCH:-false}
+force=false
+remove=false
+includes=true
+
+for arg in "${@:1}"
+do
+	[ "$arg" = "-r" ] && remove=true
+	[ "$arg" = "-f" ] && force=true
+	[ "$arg" = "-n" ] && includes=false
+
+done
+
+tool=${@: -1}
+
+uname=$(uname)
+if [[ "$uname" != [LD]* ]]
+then
+	uname=Windows
+fi
+
+
+# Batch mode
+if $batch
+then
+	CURL_OPTIONS=(--silent -L)
+else
+	CURL_OPTIONS=(-L)
+fi
+
+################################################################################
+# Helper functions
+################################################################################
+
+function exit_error
+{
+	error=$?
+	echo "${@}"
+	exit $error	
+}
+
+##  Downloads a file if it doesn't exist
+#1  URL
+#2  Output filename (optional)
+#3+ Additional options to pass to curl
+## Sets:
+#out_file: path of the downloaded file 
+function download_file
+{
+	out_file="$downloads_dir/${2:-$(basename "$1")}"
+
+	if ! [ -f "$out_file" ]
+	then
+		mkdir -p "$downloads_dir" && \
+		cd "$downloads_dir" && \
+		echo "Downloading $1" && \
+		curl "${CURL_OPTIONS[@]}" "${@:3}" -o "$out_file" "$1"
+	fi
+}
+
+## Unzips a file
+#1 The file to unzip
+#2 The output directory
+function zip_extract
+{
+	if [ "$uname" = Windows ]
+	then
+		7za.exe x -o"$2" "$1"
+	else
+		unzip "$1" -d "$2"
+	fi
+}
+
+## Extracts a 7zip file
+#1 The file to extract
+#2 The output directory
+function sevenzip_extract
+{
+	if [ "$uname" = Windows ]
+	then
+		7za.exe x -o"$2" "$1"
+	else
+		7zr x -o"$2" "$1"
+	fi
+}
+
+## Extracts a tar file
+#1 The file to extract
+#2 The output directory
+function tar_extract
+{
+	tar -xf "$1" -C "$2"
+}
+
+##  Extracts a file
+#1  File to extract
+#2  Extract directory (optional)
+no_extract=false # Optional
+## Sets:
+#out_dir: directory the file was extracted into 
+function extract_file
+{
+	out_dir="${2:-.}"
+
+	echo "Extracting $1"
+
+	mkdir -p "$out_dir" && \
+	case "$1" in
+		*.zip)
+			zip_extract "$1" "$out_dir"
+			;;
+		*.7z)
+			sevenzip_extract "$1" "$out_dir"
+			;;
+		*.tar*)
+			tar_extract "$1" "$out_dir"
+			;;
+		*)
+			if $no_extract
+			then
+				cp "$1" "$out_dir"
+			else
+				return 1
+			fi
+	esac
+}
+
+################################################################################
+# Default functions
+################################################################################
+
+function validate_target { false; }
+
+function remove
+{
+	rm -rf "$tools_dir/$tool_install_name"
+	rm -f "$tools_dir/$tool".{sh,mk}
+}
+
+declare -a depends=()
+function install_deps
+{
+	# Workaround for set -u and empty array
+	for dep in "${depends[@]:+${depends}}"
+	do
+		BATCH="$batch" "${BASH_SOURCE[0]}" "$dep"
+	done && \
+	source_includes
+}
+
+## Downloads and verifies the tool
+## Required:
+#tool_url: the url to download the tool from
+## Optional:
+#tool_md5
+#tool_md5_url
+function download_and_verify
+{
+	verified=true
+	download_file "$tool_url" && \
+	downloaded_file=$out_file && \
+	if [ -n "${tool_md5_url:-}" ]
+	then
+		download_file "$tool_md5_url" "$(basename "$downloaded_file").md5" --silent && \
+		if ! ( cd "$downloads_dir" && md5sum -c "$out_file" )
+		then
+			mv -f "$downloaded_file"{,.rej} && \
+			mv -f "$downloaded_file".md5{,.rej} && \
+			verified=false
+		fi
+	elif [ -n "${tool_md5:-}" ]
+	then
+		if [[ "$tool_md5"* != "$(cd "$downloads_dir" && md5sum "$downloaded_file")" ]]
+		then
+			mv -f "$downloaded_file"{,.rej} && \
+			verified=false
+		fi
+	fi && \
+	$verified
+}
+
+function tool_is_installed { [ -e  "$full_tool_install_name" ] || which "$tool" &>/dev/null; }
+
+## Downloads and extracts the tool
+## Required:
+#tool_url: the url to download the tool from
+#tool_install_name: the directory or file the tool will be installed as
+## Optional:
+#tool_extract_dir: Directory to extract into (useful if build required)
+function download_and_extract
+{
+	local full_tool_install_name="$tools_dir/$tool_install_name"
+	if ! tool_is_installed || $force
+	then
+		download_and_verify || exit_error "Failed to verify $downloaded_file"
+		rm -rf "$full_tool_install_name" && \
+		extract_file "$downloaded_file" "${tool_extract_dir:-$tools_dir}"
+	fi
+}
+
+function build_and_install { true; } # Most tools don't need this step
+
+## Write modules that are included by this script and make
+## Optional:
+bin_dir=""
+bin_subdir="" #
+module_file=$tool
+function write_modules
+{
+	if [ -n "$bin_subdir" ]
+	then
+		bin_dir="$tools_dir/$tool_install_name/$bin_subdir"
+	fi
+
+	if [ -n "$bin_dir" ]
+	then
+		local new_path="$bin_dir"':${PATH}'
+		# Write shell module file
+		echo 'if [[ ":$PATH:" != *":'"$bin_dir"':"* ]]; then export PATH='"$new_path"'; fi' > "$tools_dir/$module_file".sh
+		# Write make module file
+		echo 'ifeq ($(findstring :'"$bin_dir"':,:$(PATH):),)' > "$tools_dir/$module_file".mk
+		echo "export PATH := $new_path" >> "$tools_dir/$module_file".mk
+		echo "endif" >> "$tools_dir/$module_file".mk
+	fi
+}
+
+
+function source_includes
+{
+	if $includes
+	then
+		for module in "$tools_dir"/*.sh
+		do
+			source "$module"
+		done
+	fi
+}
+
+################################################################################
+# Peform tool install
+################################################################################
+source_includes || exit_error "failed to source includes"
+
+source "$tool_overrides_dir/${tool}.sh"
+
+if $remove
+then
+	remove || exit_error "Failed to remove ${tool}"
+else
+	validate_target || exit_error "${tool} is not a valid target"
+
+	install_deps || exit_error "Failed to install dependencies for ${tool}"
+
+	download_and_extract || exit_error "Failed to download and extract ${tool}"
+
+	build_and_install || exit_error "Failed to build and install ${tool}"
+
+	write_modules || exit_error "Failed to write modules for ${tool}"
+fi