MozillaFirefox/create-tar.sh
changeset 1114 572ec48f3fe8
parent 1113 8e9195853a32
child 1117 d6a688186de0
--- a/MozillaFirefox/create-tar.sh	Tue Nov 19 22:54:22 2019 +0100
+++ b/MozillaFirefox/create-tar.sh	Mon Nov 25 08:26:42 2019 +0100
@@ -25,7 +25,8 @@
 fi
 
 # Sourcing the given tar_stamps-file to have the variables available
-source "$1" || print_usage_and_exit
+TAR_STAMP="$1"
+source "$TAR_STAMP" || print_usage_and_exit
 
 # Internal variables
 BRANCH="releases/mozilla-$CHANNEL"
@@ -37,12 +38,18 @@
 
 SOURCE_TARBALL="$PRODUCT-$VERSION$VERSION_SUFFIX.source.tar.xz"
 FTP_URL="https://ftp.mozilla.org/pub/$PRODUCT/releases/$VERSION$VERSION_SUFFIX/source"
+FTP_CANDIDATES_BASE_URL="https://ftp.mozilla.org/pub/$PRODUCT/candidates"
 # Make first letter of PRODCUT upper case
 PRODUCT_CAP="${PRODUCT^}"
 LOCALES_URL="https://product-details.mozilla.org/1.0/l10n/$PRODUCT_CAP"
 # Exit script on CTRL+C
 trap "exit" INT
 
+function get_ftp_candidates_url() {
+  VERSION_WITH_SUFFIX="$1"
+  echo "$FTP_CANDIDATES_BASE_URL/$VERSION_WITH_SUFFIX-candidates"
+}
+
 function check_tarball_source () {
   TARBALL=$1
   # Print out what is going to be done:
@@ -73,27 +80,56 @@
   fi
 }
 
-function locales_get() {
-  TMP_VERSION="$1"
-  URL_TO_CHECK="${LOCALES_URL}-${TMP_VERSION}"
+function get_source_stamp() {
+  BUILD_ID="$1"
+  FTP_CANDIDATES_BASE_URL=$(get_ftp_candidates_url $VERSION$VERSION_SUFFIX)
+  FTP_CANDIDATES_JSON_SUFFIX="${BUILD_ID}/linux-x86_64/en-US/firefox-$VERSION$VERSION_SUFFIX.json"
+  BUILD_JSON=$(curl --silent --fail "$FTP_CANDIDATES_BASE_URL/$FTP_CANDIDATES_JSON_SUFFIX") || return 1;
+  REV=$(echo "$BUILD_JSON" | jq .moz_source_stamp)
+  SOURCE_REPO=$(echo "$BUILD_JSON" | jq .moz_source_repo)
+  TIMESTAMP=$(echo "$BUILD_JSON" | jq .buildid)
+  echo "Extending $TAR_STAMP with:"
+  echo "RELEASE_REPO=${SOURCE_REPO}"
+  echo "RELEASE_TAG=${REV}"
+  echo "RELEASE_TIMESTAMP=${TIMESTAMP}"
+  # We "remove and add" instead of "replace" in case the entries are not there yet
+  # Removing the old RELEASE_-tags
+  sed -i "/RELEASE_\(TAG\|REPO\|TIMESTAMP\)=.*/d" "$TAR_STAMP"
+  # Appending the new 
+  echo "RELEASE_REPO=$SOURCE_REPO" >> "$TAR_STAMP"
+  echo "RELEASE_TAG=$REV" >> "$TAR_STAMP"
+  echo "RELEASE_TIMESTAMP=$TIMESTAMP" >> "$TAR_STAMP"
+}
 
+function get_build_number() {
   LAST_FOUND=""
+  VERSION_WITH_SUFFIX="$1"
+  FTP_CANDIDATES_BASE_URL=$(get_ftp_candidates_url $VERSION_WITH_SUFFIX)
   # Unfortunately, locales-files are not associated to releases, but to builds.
-  # And since we don't know which build was the final build, we go from 9 downwards
-  # try to find the latest one that exists (assuming there are no more than 9 builds).
+  # And since we don't know which build was the final build, we grep them all from
+  # the candidates-page, sort them and take the last one which should be the oldest
   # Error only if not even the first one exists
-  for BUILD_ID in $(seq 9 -1 0); do
-    FINAL_URL="${URL_TO_CHECK}-build${BUILD_ID}.json"
-    if wget --quiet --spider "$FINAL_URL"; then
-      LAST_FOUND="$FINAL_URL"
-      break
-    fi
-  done
+  LAST_FOUND=$(curl --silent --fail "$FTP_CANDIDATES_BASE_URL/" | grep -o "build[0-9]*/" | sort | uniq | tail -n 1 | cut -d "/" -f 1)
 
   if [ "$LAST_FOUND" != "" ]; then
     echo "$LAST_FOUND"
     return 0
   else
+    echo "Error: Could not find build-number for Firefox $VERSION_WITH_SUFFIX !"  1>&2
+    return 1
+  fi
+}
+
+
+function locales_get() {
+  TMP_VERSION="$1"
+  BUILD_ID="$2"
+  URL_TO_CHECK="${LOCALES_URL}-${TMP_VERSION}"
+  FINAL_URL="${URL_TO_CHECK}-${BUILD_ID}.json"
+  if wget --quiet --spider "$FINAL_URL"; then
+    echo "$FINAL_URL"
+    return 0
+  else
     echo "Error: Could not find locales-file (json) for Firefox $TMP_VERSION !"  1>&2
     return 1
   fi
@@ -107,9 +143,11 @@
 }
 
 function locales_unchanged() {
+  BUILD_ID="$1"
+  PREV_BUILD_ID=$(get_build_number "$PREV_VERSION$PREV_VERSION_SUFFIX")
   # If no json-file for one of the versions can be found, we say "they changed"
-  prev_url=$(locales_get "$PREV_VERSION$PREV_VERSION_SUFFIX") || return 1
-  curr_url=$(locales_get "$VERSION$VERSION_SUFFIX")      || return 1
+  prev_url=$(locales_get "$PREV_VERSION$PREV_VERSION_SUFFIX" "$PREV_BUILD_ID") || return 1
+  curr_url=$(locales_get "$VERSION$VERSION_SUFFIX" "$BUILD_ID")           || return 1
 
   prev_content=$(locales_parse "$prev_url") || exit 1
   curr_content=$(locales_parse "$curr_url") || exit 1
@@ -129,11 +167,14 @@
   compression='-Ipixz'
 fi
 
+# Get ID 
+BUILD_ID=$(get_build_number "$VERSION$VERSION_SUFFIX")
+
 if [ -z ${SKIP_LOCALES+x} ]; then
   # TODO: Thunderbird has usually "default" as locale entry. 
   # There we probably need to double-check Firefox-locals
   # For now, just download every time for Thunderbird
-  if [ "$PRODUCT" = "firefox" ] && [ "$PREV_VERSION" != "" ] && locales_unchanged; then
+  if [ "$PRODUCT" = "firefox" ] && [ "$PREV_VERSION" != "" ] && locales_unchanged "$BUILD_ID"; then
     printf "%-40s: Did not change. Skipping.\n" "locales"
     LOCALES_CHANGED=0
   else
@@ -167,6 +208,7 @@
     echo "extract locale changesets"
     tar -xf $SOURCE_TARBALL $LOCALE_FILE
   fi
+  get_source_stamp "$BUILD_ID"
 else
   # We are working on a version that is not yet published on the mozilla mirror
   # so we have to actually check out the repo