23 if [ $# -ne 1 ]; then |
23 if [ $# -ne 1 ]; then |
24 print_usage_and_exit |
24 print_usage_and_exit |
25 fi |
25 fi |
26 |
26 |
27 # Sourcing the given tar_stamps-file to have the variables available |
27 # Sourcing the given tar_stamps-file to have the variables available |
28 source "$1" || print_usage_and_exit |
28 TAR_STAMP="$1" |
|
29 source "$TAR_STAMP" || print_usage_and_exit |
29 |
30 |
30 # Internal variables |
31 # Internal variables |
31 BRANCH="releases/mozilla-$CHANNEL" |
32 BRANCH="releases/mozilla-$CHANNEL" |
32 if [ "$PRODUCT" = "firefox" ]; then |
33 if [ "$PRODUCT" = "firefox" ]; then |
33 LOCALE_FILE="firefox-$VERSION/browser/locales/l10n-changesets.json" |
34 LOCALE_FILE="firefox-$VERSION/browser/locales/l10n-changesets.json" |
35 LOCALE_FILE="thunderbird-$VERSION/comm/mail/locales/l10n-changesets.json" |
36 LOCALE_FILE="thunderbird-$VERSION/comm/mail/locales/l10n-changesets.json" |
36 fi |
37 fi |
37 |
38 |
38 SOURCE_TARBALL="$PRODUCT-$VERSION$VERSION_SUFFIX.source.tar.xz" |
39 SOURCE_TARBALL="$PRODUCT-$VERSION$VERSION_SUFFIX.source.tar.xz" |
39 FTP_URL="https://ftp.mozilla.org/pub/$PRODUCT/releases/$VERSION$VERSION_SUFFIX/source" |
40 FTP_URL="https://ftp.mozilla.org/pub/$PRODUCT/releases/$VERSION$VERSION_SUFFIX/source" |
|
41 FTP_CANDIDATES_BASE_URL="https://ftp.mozilla.org/pub/$PRODUCT/candidates" |
40 # Make first letter of PRODCUT upper case |
42 # Make first letter of PRODCUT upper case |
41 PRODUCT_CAP="${PRODUCT^}" |
43 PRODUCT_CAP="${PRODUCT^}" |
42 LOCALES_URL="https://product-details.mozilla.org/1.0/l10n/$PRODUCT_CAP" |
44 LOCALES_URL="https://product-details.mozilla.org/1.0/l10n/$PRODUCT_CAP" |
43 # Exit script on CTRL+C |
45 # Exit script on CTRL+C |
44 trap "exit" INT |
46 trap "exit" INT |
|
47 |
|
48 function get_ftp_candidates_url() { |
|
49 VERSION_WITH_SUFFIX="$1" |
|
50 echo "$FTP_CANDIDATES_BASE_URL/$VERSION_WITH_SUFFIX-candidates" |
|
51 } |
45 |
52 |
46 function check_tarball_source () { |
53 function check_tarball_source () { |
47 TARBALL=$1 |
54 TARBALL=$1 |
48 # Print out what is going to be done: |
55 # Print out what is going to be done: |
49 if [ -e $TARBALL ]; then |
56 if [ -e $TARBALL ]; then |
71 echo "$1 is missing: execute zypper in $2" |
78 echo "$1 is missing: execute zypper in $2" |
72 exit 5 |
79 exit 5 |
73 fi |
80 fi |
74 } |
81 } |
75 |
82 |
|
83 function get_source_stamp() { |
|
84 BUILD_ID="$1" |
|
85 FTP_CANDIDATES_BASE_URL=$(get_ftp_candidates_url $VERSION$VERSION_SUFFIX) |
|
86 FTP_CANDIDATES_JSON_SUFFIX="${BUILD_ID}/linux-x86_64/en-US/firefox-$VERSION$VERSION_SUFFIX.json" |
|
87 BUILD_JSON=$(curl --silent --fail "$FTP_CANDIDATES_BASE_URL/$FTP_CANDIDATES_JSON_SUFFIX") || return 1; |
|
88 REV=$(echo "$BUILD_JSON" | jq .moz_source_stamp) |
|
89 SOURCE_REPO=$(echo "$BUILD_JSON" | jq .moz_source_repo) |
|
90 TIMESTAMP=$(echo "$BUILD_JSON" | jq .buildid) |
|
91 echo "Extending $TAR_STAMP with:" |
|
92 echo "RELEASE_REPO=${SOURCE_REPO}" |
|
93 echo "RELEASE_TAG=${REV}" |
|
94 echo "RELEASE_TIMESTAMP=${TIMESTAMP}" |
|
95 # We "remove and add" instead of "replace" in case the entries are not there yet |
|
96 # Removing the old RELEASE_-tags |
|
97 sed -i "/RELEASE_\(TAG\|REPO\|TIMESTAMP\)=.*/d" "$TAR_STAMP" |
|
98 # Appending the new |
|
99 echo "RELEASE_REPO=$SOURCE_REPO" >> "$TAR_STAMP" |
|
100 echo "RELEASE_TAG=$REV" >> "$TAR_STAMP" |
|
101 echo "RELEASE_TIMESTAMP=$TIMESTAMP" >> "$TAR_STAMP" |
|
102 } |
|
103 |
|
104 function get_build_number() { |
|
105 LAST_FOUND="" |
|
106 VERSION_WITH_SUFFIX="$1" |
|
107 FTP_CANDIDATES_BASE_URL=$(get_ftp_candidates_url $VERSION_WITH_SUFFIX) |
|
108 # Unfortunately, locales-files are not associated to releases, but to builds. |
|
109 # And since we don't know which build was the final build, we grep them all from |
|
110 # the candidates-page, sort them and take the last one which should be the oldest |
|
111 # Error only if not even the first one exists |
|
112 LAST_FOUND=$(curl --silent --fail "$FTP_CANDIDATES_BASE_URL/" | grep -o "build[0-9]*/" | sort | uniq | tail -n 1 | cut -d "/" -f 1) |
|
113 |
|
114 if [ "$LAST_FOUND" != "" ]; then |
|
115 echo "$LAST_FOUND" |
|
116 return 0 |
|
117 else |
|
118 echo "Error: Could not find build-number for Firefox $VERSION_WITH_SUFFIX !" 1>&2 |
|
119 return 1 |
|
120 fi |
|
121 } |
|
122 |
|
123 |
76 function locales_get() { |
124 function locales_get() { |
77 TMP_VERSION="$1" |
125 TMP_VERSION="$1" |
|
126 BUILD_ID="$2" |
78 URL_TO_CHECK="${LOCALES_URL}-${TMP_VERSION}" |
127 URL_TO_CHECK="${LOCALES_URL}-${TMP_VERSION}" |
79 |
128 FINAL_URL="${URL_TO_CHECK}-${BUILD_ID}.json" |
80 LAST_FOUND="" |
129 if wget --quiet --spider "$FINAL_URL"; then |
81 # Unfortunately, locales-files are not associated to releases, but to builds. |
130 echo "$FINAL_URL" |
82 # And since we don't know which build was the final build, we go from 9 downwards |
|
83 # try to find the latest one that exists (assuming there are no more than 9 builds). |
|
84 # Error only if not even the first one exists |
|
85 for BUILD_ID in $(seq 9 -1 0); do |
|
86 FINAL_URL="${URL_TO_CHECK}-build${BUILD_ID}.json" |
|
87 if wget --quiet --spider "$FINAL_URL"; then |
|
88 LAST_FOUND="$FINAL_URL" |
|
89 break |
|
90 fi |
|
91 done |
|
92 |
|
93 if [ "$LAST_FOUND" != "" ]; then |
|
94 echo "$LAST_FOUND" |
|
95 return 0 |
131 return 0 |
96 else |
132 else |
97 echo "Error: Could not find locales-file (json) for Firefox $TMP_VERSION !" 1>&2 |
133 echo "Error: Could not find locales-file (json) for Firefox $TMP_VERSION !" 1>&2 |
98 return 1 |
134 return 1 |
99 fi |
135 fi |
105 print('\n'.join(['{} {}'.format(key, value['changeset']) \ |
141 print('\n'.join(['{} {}'.format(key, value['changeset']) \ |
106 for key, value in sorted(json.load(sys.stdin)['locales'].items())]));" |
142 for key, value in sorted(json.load(sys.stdin)['locales'].items())]));" |
107 } |
143 } |
108 |
144 |
109 function locales_unchanged() { |
145 function locales_unchanged() { |
|
146 BUILD_ID="$1" |
|
147 PREV_BUILD_ID=$(get_build_number "$PREV_VERSION$PREV_VERSION_SUFFIX") |
110 # If no json-file for one of the versions can be found, we say "they changed" |
148 # If no json-file for one of the versions can be found, we say "they changed" |
111 prev_url=$(locales_get "$PREV_VERSION$PREV_VERSION_SUFFIX") || return 1 |
149 prev_url=$(locales_get "$PREV_VERSION$PREV_VERSION_SUFFIX" "$PREV_BUILD_ID") || return 1 |
112 curr_url=$(locales_get "$VERSION$VERSION_SUFFIX") || return 1 |
150 curr_url=$(locales_get "$VERSION$VERSION_SUFFIX" "$BUILD_ID") || return 1 |
113 |
151 |
114 prev_content=$(locales_parse "$prev_url") || exit 1 |
152 prev_content=$(locales_parse "$prev_url") || exit 1 |
115 curr_content=$(locales_parse "$curr_url") || exit 1 |
153 curr_content=$(locales_parse "$curr_url") || exit 1 |
116 |
154 |
117 diff -y --suppress-common-lines -d <(echo "$prev_content") <(echo "$curr_content") |
155 diff -y --suppress-common-lines -d <(echo "$prev_content") <(echo "$curr_content") |
127 pixz -h > /dev/null 2>&1 |
165 pixz -h > /dev/null 2>&1 |
128 if (($? != 127)); then |
166 if (($? != 127)); then |
129 compression='-Ipixz' |
167 compression='-Ipixz' |
130 fi |
168 fi |
131 |
169 |
|
170 # Get ID |
|
171 BUILD_ID=$(get_build_number "$VERSION$VERSION_SUFFIX") |
|
172 |
132 if [ -z ${SKIP_LOCALES+x} ]; then |
173 if [ -z ${SKIP_LOCALES+x} ]; then |
133 # TODO: Thunderbird has usually "default" as locale entry. |
174 # TODO: Thunderbird has usually "default" as locale entry. |
134 # There we probably need to double-check Firefox-locals |
175 # There we probably need to double-check Firefox-locals |
135 # For now, just download every time for Thunderbird |
176 # For now, just download every time for Thunderbird |
136 if [ "$PRODUCT" = "firefox" ] && [ "$PREV_VERSION" != "" ] && locales_unchanged; then |
177 if [ "$PRODUCT" = "firefox" ] && [ "$PREV_VERSION" != "" ] && locales_unchanged "$BUILD_ID"; then |
137 printf "%-40s: Did not change. Skipping.\n" "locales" |
178 printf "%-40s: Did not change. Skipping.\n" "locales" |
138 LOCALES_CHANGED=0 |
179 LOCALES_CHANGED=0 |
139 else |
180 else |
140 printf "%-40s: Need to download.\n" "locales" |
181 printf "%-40s: Need to download.\n" "locales" |
141 LOCALES_CHANGED=1 |
182 LOCALES_CHANGED=1 |
165 if [ -z ${SKIP_LOCALES+x} ] && [ $LOCALES_CHANGED -ne 0 ]; then |
206 if [ -z ${SKIP_LOCALES+x} ] && [ $LOCALES_CHANGED -ne 0 ]; then |
166 # still need to extract the locale information from the archive |
207 # still need to extract the locale information from the archive |
167 echo "extract locale changesets" |
208 echo "extract locale changesets" |
168 tar -xf $SOURCE_TARBALL $LOCALE_FILE |
209 tar -xf $SOURCE_TARBALL $LOCALE_FILE |
169 fi |
210 fi |
|
211 get_source_stamp "$BUILD_ID" |
170 else |
212 else |
171 # We are working on a version that is not yet published on the mozilla mirror |
213 # We are working on a version that is not yet published on the mozilla mirror |
172 # so we have to actually check out the repo |
214 # so we have to actually check out the repo |
173 |
215 |
174 # mozilla |
216 # mozilla |