diff -r 4c248180e576 -r 840132a4a9b3 MozillaFirefox/create-tar.sh
--- a/MozillaFirefox/create-tar.sh	Mon Jul 08 12:56:52 2019 +0200
+++ b/MozillaFirefox/create-tar.sh	Wed Jul 10 08:14:34 2019 +0200
@@ -1,20 +1,124 @@
 #!/bin/bash
 
-# TODO
-# http://ftp.mozilla.org/pub/firefox/candidates/48.0-candidates/build2/linux-x86_64/en-US/firefox-48.0.json
-# "moz_source_stamp": "c1de04f39fa956cfce83f6065b0e709369215ed5"
-# http://ftp.mozilla.org/pub/firefox/candidates/48.0-candidates/build2/l10n_changesets.txt
+function print_usage_and_exit() {
+  echo "Usage: create-tar.sh tar_stamp"
+  echo ""
+  echo "Where tar_stamp should look like this:"
+  echo ""
+  cat << EOF
+# Node ID: 64ee63facd4ff96b3e8590cff559d7e97ac6b061
+PRODUCT="firefox" # "firefox" or "thunderbird"
+CHANNEL="esr60"
+VERSION="60.7.0"
+VERSION_SUFFIX="esr"
+FF_RELEASE_TAG="" # Needs only to be set if no tar-ball can be downloaded
+TB_RELEASE_TAG="" # Only relevant for Thunderbird
+PREV_VERSION="60.6.3" # Prev. version only needed for locales (leave empty to force l10n-generation)
+PREV_VERSION_SUFFIX="esr"
+#SKIP_LOCALES="" # Uncomment to skip l10n and compare-locales-generation
+EOF
+
+exit 1
+}
+
+if [ $# -ne 1 ]; then
+  print_usage_and_exit
+fi
+
+# Sourcing the given tar_stamp-file to have the variables available
+source "$1" || print_usage_and_exit
+
+# Internal variables
+BRANCH="releases/mozilla-$CHANNEL"
+if [ "$PRODUCT" = "firefox" ]; then
+  LOCALE_FILE="firefox-$VERSION/browser/locales/l10n-changesets.json"
+else
+  LOCALE_FILE="thunderbird-$VERSION/comm/mail/locales/l10n-changesets.json"
+fi
+
+SOURCE_TARBALL="$PRODUCT-$VERSION$VERSION_SUFFIX.source.tar.xz"
+FTP_URL="https://ftp.mozilla.org/pub/$PRODUCT/releases/$VERSION$VERSION_SUFFIX/source"
+# Make first letter of PRODCUT upper case
+PRODUCT_CAP="${PRODUCT^}"
+LOCALES_URL="https://product-details.mozilla.org/1.0/l10n/$PRODUCT_CAP"
+# Exit script on CTRL+C
+trap "exit" INT
+
+function check_tarball_source () {
+  TARBALL=$1
+  # Print out what is going to be done:
+  if [ -e $TARBALL ]; then
+      echo "Reuse existing file"
+  elif wget --spider $FTP_URL/$TARBALL 2> /dev/null; then
+      echo "Download file"
+  else
+      echo "Mercurial checkout"
+  fi
+}
 
-CHANNEL="release"
-BRANCH="releases/mozilla-$CHANNEL"
-RELEASE_TAG="ea5154beddff08b919697e3bed6f38cfe3a3d82f"
-VERSION="67.0.4"
-VERSION_SUFFIX=""
-LOCALE_FILE="firefox-$VERSION/browser/locales/l10n-changesets.json"
+function ask_cont_abort_question() {
+  while true; do
+    read -p "$1 [(c)ontinue/(a)bort] " ca
+    case $ca in
+        [Cc]* ) return 0 ;;
+        [Aa]* ) return 1 ;;
+        * ) echo "Please answer c or a.";;
+    esac
+  done
+}
+
+function check_for_binary() {
+  if ! test -x $1; then
+    echo "$1 is missing: execute zypper in $2"
+    exit 5
+  fi
+}
+
+function locales_get() {
+  TMP_VERSION="$1"
+  URL_TO_CHECK="${LOCALES_URL}-${TMP_VERSION}"
+
+  LAST_FOUND=""
+  # Unfortunately, locales-files are not associated to releases, but to builds.
+  # And since we don't know which build was the final build, we go from 1 to
+  # the last we find and try to find the latest one that exists.
+  # Error only if not even the first one exists
+  for BUILD_ID in $(seq 1 9); do
+    FINAL_URL="${URL_TO_CHECK}-build${BUILD_ID}.json"
+    if wget --quiet --spider "$FINAL_URL"; then
+      LAST_FOUND="$FINAL_URL"
+    elif [ $BUILD_ID -gt 1 ]; then
+      echo "$LAST_FOUND"
+      return 0
+    else
+      echo "Error: Could not find locales-file (json) for Firefox $TMP_VERSION !"  1>&2
+      return 1
+    fi
+  done
+}
+
+function locales_parse() {
+  URL="$1"
+  curl -s "$URL" | python -c "import json; import sys; \
+             print('\n'.join(['{} {}'.format(key, value['changeset']) \
+                for key, value in sorted(json.load(sys.stdin)['locales'].items())]));"
+}
+
+function locales_unchanged() {
+  # If no json-file for one of the versions can be found, we say "they changed"
+  prev_url=$(locales_get "$PREV_VERSION$PREV_VERSION_SUFFIX") || return 1
+  curr_url=$(locales_get "$VERSION$VERSION_SUFFIX")      || return 1
+
+  prev_content=$(locales_parse "$prev_url") || exit 1
+  curr_content=$(locales_parse "$curr_url") || exit 1
+
+  diff -y --suppress-common-lines -d <(echo "$prev_content") <(echo "$curr_content")
+}
 
 # check required tools
-test -x /usr/bin/hg || ( echo "hg missing: execute zypper in mercurial"; exit 5 )
-test -x /usr/bin/jq || ( echo "jq missing: execute zypper in jq"; exit 5 )
+check_for_binary /usr/bin/hg "mercurial"
+check_for_binary /usr/bin/jq "jq"
+which python > /dev/null || exit 1
 
 # use parallel compression, if available
 compression='-J'
@@ -23,69 +127,139 @@
   compression='-Ipixz'
 fi
 
+if [ -z ${SKIP_LOCALES+x} ]; then
+  # TODO: Thunderbird has usually "default" as locale entry. 
+  # There we probably need to double-check Firefox-locals
+  # For now, just download every time for Thunderbird
+  if [ "$PRODUCT" = "firefox" ] && [ "$PREV_VERSION" != "" ] && locales_unchanged; then
+    printf "%-40s: Did not change. Skipping.\n" "locales"
+    LOCALES_CHANGED=0
+  else
+    printf "%-40s: Need to download.\n" "locales"
+    LOCALES_CHANGED=1
+  fi
+else 
+  printf "%-40s: User forced skip (SKIP_LOCALES set)\n" "locales"
+fi
+
+# Check what is going to be done and ask for consent
+for ff in $SOURCE_TARBALL $SOURCE_TARBALL.asc; do
+  printf "%-40s: %s\n" $ff "$(check_tarball_source $ff)"
+done
+
+$(ask_cont_abort_question "Is this ok?") || exit 0
+
+# Try to download tar-ball from officiall mozilla-mirror
+if [ ! -e $SOURCE_TARBALL ]; then
+  wget https://ftp.mozilla.org/pub/$PRODUCT/releases/$VERSION$VERSION_SUFFIX/source/$SOURCE_TARBALL
+fi
+# including signature
+if [ ! -e $SOURCE_TARBALL.asc ]; then
+  wget https://ftp.mozilla.org/pub/$PRODUCT/releases/$VERSION$VERSION_SUFFIX/source/$SOURCE_TARBALL.asc
+fi
+
 # we might have an upstream archive already and can skip the checkout
-if [ -e firefox-$VERSION$VERSION_SUFFIX.source.tar.xz ]; then
-  echo "skip firefox checkout and use available archive"
-  # still need to extract the locale information from the archive
-  echo "extract locale changesets"
-  tar -xf firefox-$VERSION$VERSION_SUFFIX.source.tar.xz $LOCALE_FILE
+if [ -e $SOURCE_TARBALL ]; then
+  if [ -z ${SKIP_LOCALES+x} ]; then
+    # still need to extract the locale information from the archive
+    echo "extract locale changesets"
+    tar -xf $SOURCE_TARBALL $LOCALE_FILE
+  fi
 else
+  # We are working on a version that is not yet published on the mozilla mirror
+  # so we have to actually check out the repo
+
   # mozilla
-  if [ -d firefox-$VERSION ]; then
-    pushd firefox-$VERSION
+  if [ -d $PRODUCT-$VERSION ]; then
+    pushd $PRODUCT-$VERSION || exit 1
     _repourl=$(hg paths)
     case "$_repourl" in
       *$BRANCH*)
         echo "updating previous tree"
         hg pull
-        popd
+        popd || exit 1
         ;;
       * )
         echo "removing obsolete tree"
-        popd
-        rm -rf firefox-$VERSION
+        popd || exit 1
+        rm -rf $PRODUCT-$VERSION
         ;;
     esac
   fi
-  if [ ! -d firefox-$VERSION ]; then
+  if [ ! -d $PRODUCT-$VERSION ]; then
     echo "cloning new $BRANCH..."
-    hg clone http://hg.mozilla.org/$BRANCH firefox-$VERSION
+    hg clone http://hg.mozilla.org/$BRANCH $PRODUCT-$VERSION
+    if [ "$PRODUCT" = "thunderbird" ]; then
+      hg clone http://hg.mozilla.org/releases/comm-$CHANNEL thunderbird-$VERSION/comm
+    fi
   fi
-  pushd firefox-$VERSION
-  hg update --check
-  [ "$RELEASE_TAG" == "default" ] || hg update -r $RELEASE_TAG
+  pushd $PRODUCT-$VERSION || exit 1
+  hg update --check $FF_RELEASE_TAG
+  [ "$FF_RELEASE_TAG" == "default" ] || hg update -r $FF_RELEASE_TAG
   # get repo and source stamp
   echo -n "REV=" > ../source-stamp.txt
   hg -R . parent --template="{node|short}\n" >> ../source-stamp.txt
   echo -n "REPO=" >> ../source-stamp.txt
   hg showconfig paths.default 2>/dev/null | head -n1 | sed -e "s/^ssh:/http:/" >> ../source-stamp.txt
-  popd
+
+  if [ "$PRODUCT" = "thunderbird" ]; then
+    pushd comm || exit 1
+    hg update --check $TB_RELEASE_TAG
+    popd || exit 1
+    rm -rf thunderbird-${VERSION}/{,comm/}other-licenses/7zstub
+  fi
+  popd || exit 1
 
   echo "creating archive..."
-  tar $compression -cf firefox-$VERSION$VERSION_SUFFIX.source.tar.xz --exclude=.hgtags --exclude=.hgignore --exclude=.hg --exclude=CVS firefox-$VERSION
+  tar $compression -cf $PRODUCT-$VERSION$VERSION_SUFFIX.source.tar.xz --exclude=.hgtags --exclude=.hgignore --exclude=.hg --exclude=CVS $PRODUCT-$VERSION
+fi
+
+if [ ! -z ${SKIP_LOCALES+x} ]; then
+  echo "Skipping locales-creation."
+  exit 0
 fi
 
-# l10n
-echo "fetching locales..."
-test ! -d l10n && mkdir l10n
-jq -r 'to_entries[]| "\(.key) \(.value|.revision)"' $LOCALE_FILE | \
-  while read locale changeset ; do
-    case $locale in
-      ja-JP-mac|en-US)
-        ;;
-      *)
-        echo "reading changeset information for $locale"
-        echo "fetching $locale changeset $changeset ..."
-        hg clone http://hg.mozilla.org/l10n-central/$locale l10n/$locale
-        [ "$RELEASE_TAG" == "default" ] || hg -R l10n/$locale up -C -r $changeset
-        ;;
-    esac
-  done
-echo "creating l10n archive..."
-tar $compression -cf l10n-$VERSION$VERSION_SUFFIX.tar.xz --exclude=.hgtags --exclude=.hgignore --exclude=.hg l10n
+if [ $LOCALES_CHANGED -ne 0 ]; then
+  # l10n
+  echo "fetching locales..."
+  test ! -d l10n && mkdir l10n
+  jq -r 'to_entries[]| "\(.key) \(.value|.revision)"' $LOCALE_FILE | \
+    while read locale changeset ; do
+      case $locale in
+        ja-JP-mac|en-US)
+          ;;
+        *)
+          echo "reading changeset information for $locale"
+          echo "fetching $locale changeset $changeset ..."
+          if [ -d "l10n/$locale/.hg" ]; then
+            pushd "l10n/$locale" || exit 1
+            hg pull
+            popd || exit 1
+          else
+            hg clone "http://hg.mozilla.org/l10n-central/$locale" "l10n/$locale"
+          fi
+          [ "$FF_RELEASE_TAG" == "default" ] || hg -R "l10n/$locale" up -C -r "$changeset"
+          ;;
+      esac
+    done
+  echo "creating l10n archive..."
+if [ "$PRODUCT" = "thunderbird" ]; then
+    TB_TAR_FLAGS="--exclude=browser --exclude=suite"
+fi
+  tar $compression -cf l10n-$VERSION$VERSION_SUFFIX.tar.xz \
+  --exclude=.hgtags --exclude=.hgignore --exclude=.hg \
+  $TB_TAR_FLAGS \
+  l10n
+fi
 
 # compare-locales
 echo "creating compare-locales"
-hg clone http://hg.mozilla.org/build/compare-locales
+if [ -d compare-locales/.hg ]; then
+  pushd compare-locales || exit 1
+  hg pull
+  popd || exit 1
+else
+  hg clone http://hg.mozilla.org/build/compare-locales
+fi
 tar $compression -cf compare-locales.tar.xz --exclude=.hgtags --exclude=.hgignore --exclude=.hg compare-locales