diff -r 8e9195853a32 -r 572ec48f3fe8 MozillaFirefox/create-tar.sh --- a/MozillaFirefox/create-tar.sh Tue Nov 19 22:54:22 2019 +0100 +++ b/MozillaFirefox/create-tar.sh Mon Nov 25 08:26:42 2019 +0100 @@ -25,7 +25,8 @@ fi # Sourcing the given tar_stamps-file to have the variables available -source "$1" || print_usage_and_exit +TAR_STAMP="$1" +source "$TAR_STAMP" || print_usage_and_exit # Internal variables BRANCH="releases/mozilla-$CHANNEL" @@ -37,12 +38,18 @@ SOURCE_TARBALL="$PRODUCT-$VERSION$VERSION_SUFFIX.source.tar.xz" FTP_URL="https://ftp.mozilla.org/pub/$PRODUCT/releases/$VERSION$VERSION_SUFFIX/source" +FTP_CANDIDATES_BASE_URL="https://ftp.mozilla.org/pub/$PRODUCT/candidates" # Make first letter of PRODCUT upper case PRODUCT_CAP="${PRODUCT^}" LOCALES_URL="https://product-details.mozilla.org/1.0/l10n/$PRODUCT_CAP" # Exit script on CTRL+C trap "exit" INT +function get_ftp_candidates_url() { + VERSION_WITH_SUFFIX="$1" + echo "$FTP_CANDIDATES_BASE_URL/$VERSION_WITH_SUFFIX-candidates" +} + function check_tarball_source () { TARBALL=$1 # Print out what is going to be done: @@ -73,27 +80,56 @@ fi } -function locales_get() { - TMP_VERSION="$1" - URL_TO_CHECK="${LOCALES_URL}-${TMP_VERSION}" +function get_source_stamp() { + BUILD_ID="$1" + FTP_CANDIDATES_BASE_URL=$(get_ftp_candidates_url $VERSION$VERSION_SUFFIX) + FTP_CANDIDATES_JSON_SUFFIX="${BUILD_ID}/linux-x86_64/en-US/firefox-$VERSION$VERSION_SUFFIX.json" + BUILD_JSON=$(curl --silent --fail "$FTP_CANDIDATES_BASE_URL/$FTP_CANDIDATES_JSON_SUFFIX") || return 1; + REV=$(echo "$BUILD_JSON" | jq .moz_source_stamp) + SOURCE_REPO=$(echo "$BUILD_JSON" | jq .moz_source_repo) + TIMESTAMP=$(echo "$BUILD_JSON" | jq .buildid) + echo "Extending $TAR_STAMP with:" + echo "RELEASE_REPO=${SOURCE_REPO}" + echo "RELEASE_TAG=${REV}" + echo "RELEASE_TIMESTAMP=${TIMESTAMP}" + # We "remove and add" instead of "replace" in case the entries are not there yet + # Removing the old RELEASE_-tags + sed -i "/RELEASE_\(TAG\|REPO\|TIMESTAMP\)=.*/d" "$TAR_STAMP" + # Appending the new + echo "RELEASE_REPO=$SOURCE_REPO" >> "$TAR_STAMP" + echo "RELEASE_TAG=$REV" >> "$TAR_STAMP" + echo "RELEASE_TIMESTAMP=$TIMESTAMP" >> "$TAR_STAMP" +} +function get_build_number() { LAST_FOUND="" + VERSION_WITH_SUFFIX="$1" + FTP_CANDIDATES_BASE_URL=$(get_ftp_candidates_url $VERSION_WITH_SUFFIX) # Unfortunately, locales-files are not associated to releases, but to builds. - # And since we don't know which build was the final build, we go from 9 downwards - # try to find the latest one that exists (assuming there are no more than 9 builds). + # And since we don't know which build was the final build, we grep them all from + # the candidates-page, sort them and take the last one which should be the oldest # Error only if not even the first one exists - for BUILD_ID in $(seq 9 -1 0); do - FINAL_URL="${URL_TO_CHECK}-build${BUILD_ID}.json" - if wget --quiet --spider "$FINAL_URL"; then - LAST_FOUND="$FINAL_URL" - break - fi - done + LAST_FOUND=$(curl --silent --fail "$FTP_CANDIDATES_BASE_URL/" | grep -o "build[0-9]*/" | sort | uniq | tail -n 1 | cut -d "/" -f 1) if [ "$LAST_FOUND" != "" ]; then echo "$LAST_FOUND" return 0 else + echo "Error: Could not find build-number for Firefox $VERSION_WITH_SUFFIX !" 1>&2 + return 1 + fi +} + + +function locales_get() { + TMP_VERSION="$1" + BUILD_ID="$2" + URL_TO_CHECK="${LOCALES_URL}-${TMP_VERSION}" + FINAL_URL="${URL_TO_CHECK}-${BUILD_ID}.json" + if wget --quiet --spider "$FINAL_URL"; then + echo "$FINAL_URL" + return 0 + else echo "Error: Could not find locales-file (json) for Firefox $TMP_VERSION !" 1>&2 return 1 fi @@ -107,9 +143,11 @@ } function locales_unchanged() { + BUILD_ID="$1" + PREV_BUILD_ID=$(get_build_number "$PREV_VERSION$PREV_VERSION_SUFFIX") # If no json-file for one of the versions can be found, we say "they changed" - prev_url=$(locales_get "$PREV_VERSION$PREV_VERSION_SUFFIX") || return 1 - curr_url=$(locales_get "$VERSION$VERSION_SUFFIX") || return 1 + prev_url=$(locales_get "$PREV_VERSION$PREV_VERSION_SUFFIX" "$PREV_BUILD_ID") || return 1 + curr_url=$(locales_get "$VERSION$VERSION_SUFFIX" "$BUILD_ID") || return 1 prev_content=$(locales_parse "$prev_url") || exit 1 curr_content=$(locales_parse "$curr_url") || exit 1 @@ -129,11 +167,14 @@ compression='-Ipixz' fi +# Get ID +BUILD_ID=$(get_build_number "$VERSION$VERSION_SUFFIX") + if [ -z ${SKIP_LOCALES+x} ]; then # TODO: Thunderbird has usually "default" as locale entry. # There we probably need to double-check Firefox-locals # For now, just download every time for Thunderbird - if [ "$PRODUCT" = "firefox" ] && [ "$PREV_VERSION" != "" ] && locales_unchanged; then + if [ "$PRODUCT" = "firefox" ] && [ "$PREV_VERSION" != "" ] && locales_unchanged "$BUILD_ID"; then printf "%-40s: Did not change. Skipping.\n" "locales" LOCALES_CHANGED=0 else @@ -167,6 +208,7 @@ echo "extract locale changesets" tar -xf $SOURCE_TARBALL $LOCALE_FILE fi + get_source_stamp "$BUILD_ID" else # We are working on a version that is not yet published on the mozilla mirror # so we have to actually check out the repo