15 PREV_VERSION="60.6.3" # Prev. version only needed for locales (leave empty to force l10n-generation) |
44 PREV_VERSION="60.6.3" # Prev. version only needed for locales (leave empty to force l10n-generation) |
16 PREV_VERSION_SUFFIX="esr" |
45 PREV_VERSION_SUFFIX="esr" |
17 #SKIP_LOCALES="" # Uncomment to skip l10n-generation |
46 #SKIP_LOCALES="" # Uncomment to skip l10n-generation |
18 EOF |
47 EOF |
19 |
48 |
20 exit 1 |
49 exit 1 |
21 } |
50 } |
22 |
51 |
23 if [ $# -ne 1 ]; then |
52 function check_required_tools() { |
24 print_usage_and_exit |
53 # check required tools |
25 fi |
54 check_for_binary /usr/bin/hg "mercurial" |
26 |
55 check_for_binary /usr/bin/jq "jq" |
27 # Sourcing the given tar_stamps-file to have the variables available |
56 which python3 > /dev/null || exit 1 |
28 TAR_STAMP="$1" |
57 |
29 source "$TAR_STAMP" || print_usage_and_exit |
58 # use parallel compression, if available |
30 |
59 compression='-J' |
31 # Internal variables |
60 pixz -h > /dev/null 2>&1 |
32 BRANCH="releases/mozilla-$CHANNEL" |
61 if (($? != 127)); then |
33 if [ "$PRODUCT" = "firefox" ]; then |
62 compression='-Ipixz' |
34 LOCALE_FILE="firefox-$VERSION/browser/locales/l10n-changesets.json" |
63 fi |
35 else |
64 } |
36 LOCALE_FILE="thunderbird-$VERSION/comm/mail/locales/l10n-changesets.json" |
65 |
37 fi |
66 function set_internal_variables() { |
38 |
67 # Internal variables |
39 SOURCE_TARBALL="$PRODUCT-$VERSION$VERSION_SUFFIX.source.tar.xz" |
68 BRANCH="releases/mozilla-$CHANNEL" |
40 FTP_URL="https://ftp.mozilla.org/pub/$PRODUCT/releases/$VERSION$VERSION_SUFFIX/source" |
69 if [ "$PRODUCT" = "firefox" ]; then |
41 FTP_CANDIDATES_BASE_URL="https://ftp.mozilla.org/pub/$PRODUCT/candidates" |
70 FF_LOCALE_FILE="firefox-$VERSION/browser/locales/l10n-changesets.json" |
42 # Make first letter of PRODCUT upper case |
71 else |
43 PRODUCT_CAP="${PRODUCT^}" |
72 FF_LOCALE_FILE="thunderbird-$VERSION/browser/locales/l10n-changesets.json" |
44 LOCALES_URL="https://product-details.mozilla.org/1.0/l10n/$PRODUCT_CAP" |
73 TB_LOCALE_FILE="thunderbird-$VERSION/comm/mail/locales/l10n-changesets.json" |
45 PRODUCT_URL="https://product-details.mozilla.org/1.0/$PRODUCT.json" |
74 L10N_STRING_PATTERNS="thunderbird-$VERSION/python/l10n/tbxchannel/l10n_merge.py" |
46 # Exit script on CTRL+C |
75 fi |
47 trap "exit" INT |
76 |
|
77 SOURCE_TARBALL="$PRODUCT-$VERSION$VERSION_SUFFIX.source.tar.xz" |
|
78 PREV_SOURCE_TARBALL="$PRODUCT-$PREV_VERSION$PREV_VERSION_SUFFIX.source.tar.xz" |
|
79 FTP_URL="https://ftp.mozilla.org/pub/$PRODUCT/releases/$VERSION$VERSION_SUFFIX/source" |
|
80 FTP_CANDIDATES_BASE_URL="https://ftp.mozilla.org/pub/%s/candidates" |
|
81 LOCALES_URL="https://product-details.mozilla.org/1.0/l10n" |
|
82 PRODUCT_URL="https://product-details.mozilla.org/1.0" |
|
83 ALREADY_EXTRACTED_LOCALES_FILE=0 |
|
84 } |
48 |
85 |
49 function get_ftp_candidates_url() { |
86 function get_ftp_candidates_url() { |
50 VERSION_WITH_SUFFIX="$1" |
87 local CURR_PRODUCT="$1" |
51 echo "$FTP_CANDIDATES_BASE_URL/$VERSION_WITH_SUFFIX-candidates" |
88 local VERSION_WITH_SUFFIX="$2" |
|
89 printf "$FTP_CANDIDATES_BASE_URL/$VERSION_WITH_SUFFIX-candidates" "$CURR_PRODUCT" |
52 } |
90 } |
53 |
91 |
54 function check_tarball_source () { |
92 function check_tarball_source () { |
55 TARBALL=$1 |
93 TARBALL=$1 |
56 # Print out what is going to be done: |
94 # Print out what is going to be done: |
57 if [ -e $TARBALL ]; then |
95 if [ -e "$TARBALL" ]; then |
58 echo "Reuse existing file" |
96 echo "Reuse existing file" |
59 elif wget --spider $FTP_URL/$TARBALL 2> /dev/null; then |
97 elif wget --spider "$FTP_URL/$TARBALL" 2> /dev/null; then |
60 echo "Download file" |
98 echo "Download file" |
61 else |
99 else |
62 echo "Mercurial checkout" |
100 local CANDIDATE_TARBALL_LOCATION="" |
|
101 CANDIDATE_TARBALL_LOCATION="$(printf "%s/%s/source/%s" "$(get_ftp_candidates_url "$PRODUCT" "$VERSION$VERSION_SUFFIX")" "$BUILD_ID" "$TARBALL" )" |
|
102 if wget --spider "$CANDIDATE_TARBALL_LOCATION" 2> /dev/null; then |
|
103 echo "Download UNRELEASED candidate" |
|
104 else |
|
105 echo "Mercurial checkout" |
|
106 fi |
63 fi |
107 fi |
64 } |
108 } |
65 |
109 |
66 function ask_cont_abort_question() { |
110 function ask_cont_abort_question() { |
67 while true; do |
111 while true; do |
68 read -p "$1 [(c)ontinue/(a)bort] " ca |
112 read -r -p "$1 [(c)ontinue/(a)bort] " ca |
69 case $ca in |
113 case $ca in |
70 [Cc]* ) return 0 ;; |
114 [Cc]* ) return 0 ;; |
71 [Aa]* ) return 1 ;; |
115 [Aa]* ) return 1 ;; |
72 * ) echo "Please answer c or a.";; |
116 * ) echo "Please answer c or a.";; |
73 esac |
117 esac |
74 done |
118 done |
75 } |
119 } |
76 |
120 |
77 function check_for_binary() { |
121 function check_for_binary() { |
78 if ! test -x $1; then |
122 if ! test -x "$1"; then |
79 echo "$1 is missing: execute zypper in $2" |
123 echo "$1 is missing: execute zypper in $2" |
80 exit 5 |
124 exit 5 |
81 fi |
125 fi |
82 } |
126 } |
83 |
127 |
84 function get_source_stamp() { |
128 function get_source_stamp() { |
85 BUILD_ID="$1" |
129 local CURR_BUILD_ID="$1" |
86 FTP_CANDIDATES_BASE_URL=$(get_ftp_candidates_url $VERSION$VERSION_SUFFIX) |
130 local FTP_CANDIDATES_BASE_URL=$(get_ftp_candidates_url "$PRODUCT" "$VERSION$VERSION_SUFFIX") |
87 FTP_CANDIDATES_JSON_SUFFIX="${BUILD_ID}/linux-x86_64/en-US/$PRODUCT-$VERSION$VERSION_SUFFIX.json" |
131 local FTP_CANDIDATES_JSON_SUFFIX="${CURR_BUILD_ID}/linux-x86_64/en-US/$PRODUCT-$VERSION$VERSION_SUFFIX.json" |
88 BUILD_JSON=$(curl --silent --fail "$FTP_CANDIDATES_BASE_URL/$FTP_CANDIDATES_JSON_SUFFIX") || return 1; |
132 local BUILD_JSON=$(curl --silent --fail "$FTP_CANDIDATES_BASE_URL/$FTP_CANDIDATES_JSON_SUFFIX") || return 1; |
89 REV=$(echo "$BUILD_JSON" | jq .moz_source_stamp) |
133 local REV=$(echo "$BUILD_JSON" | jq .moz_source_stamp) |
90 SOURCE_REPO=$(echo "$BUILD_JSON" | jq .moz_source_repo) |
134 local SOURCE_REPO=$(echo "$BUILD_JSON" | jq .moz_source_repo) |
91 TIMESTAMP=$(echo "$BUILD_JSON" | jq .buildid) |
135 local TIMESTAMP=$(echo "$BUILD_JSON" | jq .buildid) |
92 echo "Extending $TAR_STAMP with:" |
136 echo "Extending $TAR_STAMP with:" |
93 echo "RELEASE_REPO=${SOURCE_REPO}" |
137 echo "RELEASE_REPO=${SOURCE_REPO}" |
94 echo "RELEASE_TAG=${REV}" |
138 echo "RELEASE_TAG=${REV}" |
95 echo "RELEASE_TIMESTAMP=${TIMESTAMP}" |
139 echo "RELEASE_TIMESTAMP=${TIMESTAMP}" |
96 # We "remove and add" instead of "replace" in case the entries are not there yet |
140 # We "remove and add" instead of "replace" in case the entries are not there yet |
101 echo "RELEASE_TAG=$REV" >> "$TAR_STAMP" |
145 echo "RELEASE_TAG=$REV" >> "$TAR_STAMP" |
102 echo "RELEASE_TIMESTAMP=$TIMESTAMP" >> "$TAR_STAMP" |
146 echo "RELEASE_TIMESTAMP=$TIMESTAMP" >> "$TAR_STAMP" |
103 } |
147 } |
104 |
148 |
105 function get_build_number() { |
149 function get_build_number() { |
106 LAST_FOUND="" |
150 local LAST_FOUND="" |
107 VERSION_WITH_SUFFIX="$1" |
151 local CURR_PRODUCT="$1" |
108 |
152 local VERSION_WITH_SUFFIX="$2" |
109 BUILD_ID=$(curl --silent "$PRODUCT_URL" | jq -e '.["releases"] | .["'$PRODUCT-$VERSION_WITH_SUFFIX'"] | .["build_number"]') |
153 local CURR_BUILD_ID="" |
|
154 local CURR_FTP_BASE_URL="" |
|
155 CURR_BUILD_ID=$(curl --silent "$PRODUCT_URL/$CURR_PRODUCT.json" | jq -e '.["releases"] | .["'$CURR_PRODUCT-$VERSION_WITH_SUFFIX'"] | .["build_number"]') |
110 |
156 |
111 # Slow fall-back |
157 # Slow fall-back |
112 if [ $? -ne 0 ]; then |
158 if [ $? -ne 0 ]; then |
113 echo "Build number not found in product URL, falling back to slow FTP-parsing." 1>&2 |
159 echo "Build number not found in product URL, falling back to slow FTP-parsing." 1>&2 |
114 FTP_CANDIDATES_BASE_URL=$(get_ftp_candidates_url $VERSION_WITH_SUFFIX) |
160 CURR_FTP_BASE_URL=$(get_ftp_candidates_url "$CURR_PRODUCT" "$VERSION_WITH_SUFFIX") |
115 # Unfortunately, locales-files are not associated to releases, but to builds. |
161 # Unfortunately, locales-files are not associated to releases, but to builds. |
116 # And since we don't know which build was the final build, we grep them all from |
162 # And since we don't know which build was the final build, we grep them all from |
117 # the candidates-page, sort them and take the last one which should be the oldest |
163 # the candidates-page, sort them and take the last one which should be the oldest |
118 # Error only if not even the first one exists |
164 # Error only if not even the first one exists |
119 LAST_FOUND=$(curl --silent --fail "$FTP_CANDIDATES_BASE_URL/" | grep -o "build[0-9]*/" | sort | uniq | tail -n 1 | cut -d "/" -f 1) |
165 LAST_FOUND=$(curl --silent --fail "$CURR_FTP_BASE_URL/" | grep -o "build[0-9]*/" | sort | uniq | tail -n 1 | cut -d "/" -f 1) |
120 else |
166 else |
121 LAST_FOUND="build$BUILD_ID" |
167 LAST_FOUND="build$CURR_BUILD_ID" |
122 fi |
168 fi |
123 |
169 |
124 if [ "$LAST_FOUND" != "" ]; then |
170 if [ "$LAST_FOUND" != "" ]; then |
125 echo "$LAST_FOUND" |
171 echo "$LAST_FOUND" |
126 return 0 |
172 return 0 |
127 else |
173 else |
128 echo "Error: Could not find build-number for Firefox $VERSION_WITH_SUFFIX !" 1>&2 |
174 echo "Error: Could not find build-number for $CURR_PRODUCT $VERSION_WITH_SUFFIX !" 1>&2 |
129 return 1 |
175 return 1 |
130 fi |
176 fi |
131 } |
177 } |
132 |
178 |
133 |
|
134 function locales_get() { |
179 function locales_get() { |
135 TMP_VERSION="$1" |
180 local CURR_PRODUCT="$1" |
136 BUILD_ID="$2" |
181 local TMP_VERSION="$2" |
137 URL_TO_CHECK="${LOCALES_URL}-${TMP_VERSION}" |
182 local CURR_BUILD_ID="$3" |
138 FINAL_URL="${URL_TO_CHECK}-${BUILD_ID}.json" |
183 # Make first letter of CURR_PRODUCT upper case |
|
184 CURR_PRODUCT_CAP="${CURR_PRODUCT^}" |
|
185 URL_TO_CHECK="${LOCALES_URL}/${CURR_PRODUCT_CAP}-${TMP_VERSION}" |
|
186 FINAL_URL="${URL_TO_CHECK}-${CURR_BUILD_ID}.json" |
139 if wget --quiet --spider "$FINAL_URL"; then |
187 if wget --quiet --spider "$FINAL_URL"; then |
140 echo "$FINAL_URL" |
188 echo "$FINAL_URL" |
141 return 0 |
189 return 0 |
142 else |
190 else |
143 echo "Error: Could not find locales-file (json) for Firefox $TMP_VERSION !" 1>&2 |
191 echo "Error: Could not find locales-file (json) for Firefox $TMP_VERSION !" 1>&2 |
144 return 1 |
192 return 1 |
145 fi |
193 fi |
146 } |
194 } |
147 |
195 |
148 function locales_parse() { |
196 function locales_parse_file() { |
|
197 FILE="$1" |
|
198 python3 -c "import json; import sys; \ |
|
199 print('\n'.join(['{} {}'.format(key, value['revision']) \ |
|
200 for key, value in sorted(json.load(sys.stdin).items())]));" < "$FILE" |
|
201 } |
|
202 |
|
203 function locales_parse_url() { |
149 URL="$1" |
204 URL="$1" |
150 curl -s "$URL" | python -c "import json; import sys; \ |
205 curl -s "$URL" | python3 -c "import json; import sys; \ |
151 print('\n'.join(['{} {}'.format(key, value['changeset']) \ |
206 print('\n'.join(['{} {}'.format(key, value['changeset']) \ |
152 for key, value in sorted(json.load(sys.stdin)['locales'].items())]));" |
207 for key, value in sorted(json.load(sys.stdin)['locales'].items())]));" |
153 } |
208 } |
154 |
209 |
|
210 function extract_locales_file() { |
|
211 if [ $ALREADY_EXTRACTED_LOCALES_FILE -ne 1 ]; then |
|
212 # still need to extract the locale information from the archive |
|
213 echo "extract locale changesets" |
|
214 if [ "$PRODUCT" = "thunderbird" ]; then |
|
215 tar -xf "$SOURCE_TARBALL" "$FF_LOCALE_FILE" "$TB_LOCALE_FILE" "$L10N_STRING_PATTERNS" |
|
216 else |
|
217 tar -xf "$SOURCE_TARBALL" "$FF_LOCALE_FILE" |
|
218 fi |
|
219 ALREADY_EXTRACTED_LOCALES_FILE=1 |
|
220 else |
|
221 echo "Skipping locale changeset extraction, as it was already done." |
|
222 fi |
|
223 } |
|
224 |
155 function locales_unchanged() { |
225 function locales_unchanged() { |
156 BUILD_ID="$1" |
226 local CURR_PRODUCT="$1" |
157 PREV_BUILD_ID=$(get_build_number "$PREV_VERSION$PREV_VERSION_SUFFIX") |
227 local CURR_BUILD_ID="$2" |
|
228 local PREV_BUILD_ID=$(get_build_number "$CURR_PRODUCT" "$PREV_VERSION$PREV_VERSION_SUFFIX") |
158 # If no json-file for one of the versions can be found, we say "they changed" |
229 # If no json-file for one of the versions can be found, we say "they changed" |
159 prev_url=$(locales_get "$PREV_VERSION$PREV_VERSION_SUFFIX" "$PREV_BUILD_ID") || return 1 |
230 prev_url=$(locales_get "$CURR_PRODUCT" "$PREV_VERSION$PREV_VERSION_SUFFIX" "$PREV_BUILD_ID") || return 1 |
160 curr_url=$(locales_get "$VERSION$VERSION_SUFFIX" "$BUILD_ID") || return 1 |
231 prev_content=$(locales_parse_url "$prev_url") || exit 1 |
161 |
232 |
162 prev_content=$(locales_parse "$prev_url") || exit 1 |
233 curr_url=$(locales_get "$CURR_PRODUCT" "$VERSION$VERSION_SUFFIX" "$CURR_BUILD_ID") |
163 curr_content=$(locales_parse "$curr_url") || exit 1 |
234 if [ $? -ne 0 ]; then |
|
235 # We did not find a locales file upstream on the servers |
|
236 if [ -e "$SOURCE_TARBALL" ]; then |
|
237 # We can find out what the locales are, by extracting the json-file from the tar-ball |
|
238 # instead of getting it from the server |
|
239 extract_locales_file || return 1 |
|
240 curr_content=$(locales_parse_file "$FF_LOCALE_FILE") || exit 1 |
|
241 else |
|
242 # We can't know what the locales are in the current version |
|
243 return 1 |
|
244 fi |
|
245 else |
|
246 curr_content=$(locales_parse_url "$curr_url") || exit 1 |
|
247 fi |
164 |
248 |
165 diff -y --suppress-common-lines -d <(echo "$prev_content") <(echo "$curr_content") |
249 diff -y --suppress-common-lines -d <(echo "$prev_content") <(echo "$curr_content") |
166 } |
250 } |
167 |
251 |
168 # check required tools |
252 function get_locales_directories() { |
169 check_for_binary /usr/bin/hg "mercurial" |
253 pattern="$1" |
170 check_for_binary /usr/bin/jq "jq" |
254 |
171 which python > /dev/null || exit 1 |
255 # This file contains a list of directories, upstream uses to build locales |
172 |
256 # If it is there, use it. If not, default to all FF + 3 TB-dirs. |
173 # use parallel compression, if available |
257 if [ -e "$L10N_STRING_PATTERNS" ]; then |
174 compression='-J' |
258 python3 -c "import os; import sys; \ |
175 pixz -h > /dev/null 2>&1 |
259 sys.path.append(os.path.dirname(\"$L10N_STRING_PATTERNS\")); \ |
176 if (($? != 127)); then |
260 from l10n_merge import $pattern; \ |
177 compression='-Ipixz' |
261 print(\" \".join([p.strip('*') for p in $pattern]));" |
178 fi |
262 else |
179 |
263 if [ "$pattern" = "GECKO_STRINGS_PATTERNS" ]; then |
180 # Get ID |
264 # Default of Firefox: Take all |
181 BUILD_ID=$(get_build_number "$VERSION$VERSION_SUFFIX") |
265 echo "{lang}/" |
182 |
266 else |
183 if [ -z ${SKIP_LOCALES+x} ]; then |
267 # Default of Thunderbird: Take those 3 dirs |
184 if [ "$PREV_VERSION" != "" ] && locales_unchanged "$BUILD_ID"; then |
268 echo "{lang}/calendar/" "{lang}/chat/" "{lang}/mail/" |
|
269 fi |
|
270 fi |
|
271 } |
|
272 |
|
273 function create_and_copy_locales() { |
|
274 locale="$1" |
|
275 source_base="$2" |
|
276 source_template="$3" |
|
277 final_dest="$4" |
|
278 |
|
279 # Replace {lang} with the actual language-basedir |
|
280 for template in $source_template; do |
|
281 locale_source=$(echo "$template" | sed "s|{lang}|./$source_base/$locale|g") |
|
282 locale_dest=$(echo "$template" | sed "s|{lang}|./$final_dest/$locale|g") |
|
283 |
|
284 # Create intermediary folders |
|
285 for destdir in $locale_dest; do |
|
286 mkdir -p "$destdir" |
|
287 done |
|
288 |
|
289 # Copy over FF-files |
|
290 cp -r "$locale_source"/* "$locale_dest" |
|
291 done |
|
292 } |
|
293 |
|
294 function check_what_to_do_with_source_tarballs() { |
|
295 # Get ID |
|
296 BUILD_ID=$(get_build_number "$PRODUCT" "$VERSION$VERSION_SUFFIX") |
|
297 |
|
298 # Check what is going to be done and ask for consent |
|
299 for ff in $SOURCE_TARBALL $SOURCE_TARBALL.asc; do |
|
300 printf "%-40s: %s\n" "$ff" "$(check_tarball_source $ff)" |
|
301 done |
|
302 |
|
303 ask_cont_abort_question "Is this ok?" || exit 0 |
|
304 } |
|
305 |
|
306 function check_what_to_do_with_locales_tarballs() { |
|
307 LOCALES_CHANGED=1 |
|
308 |
|
309 extract_locales_file |
|
310 |
|
311 if [ "$PREV_VERSION" != "" ]; then |
|
312 # If we have a previous version, check either FF or (TB and FF) |
|
313 if [ "$PRODUCT" = "firefox" ]; then |
|
314 locales_unchanged "$PRODUCT" "$BUILD_ID" |
|
315 else |
|
316 FF_BUILD_ID=$(get_build_number "firefox" "$VERSION$VERSION_SUFFIX") |
|
317 locales_unchanged "$PRODUCT" "$BUILD_ID" && locales_unchanged "firefox" "$FF_BUILD_ID" |
|
318 fi |
|
319 LOCALES_CHANGED=$? |
|
320 fi |
|
321 |
|
322 # New line for better visibility |
|
323 echo "" |
|
324 if [ $LOCALES_CHANGED -eq 1 ]; then |
|
325 printf "%-40s: Need to download.\n" "locales" |
|
326 ask_cont_abort_question "Is this ok?" || exit 0 |
|
327 else |
185 printf "%-40s: Did not change. Skipping.\n" "locales" |
328 printf "%-40s: Did not change. Skipping.\n" "locales" |
186 LOCALES_CHANGED=0 |
329 fi |
187 else |
330 } |
188 printf "%-40s: Need to download.\n" "locales" |
331 |
189 LOCALES_CHANGED=1 |
332 function download_release_or_candidate_file() { |
190 fi |
333 local upstream_file="$1" |
191 else |
334 if [ -e "$upstream_file" ]; then |
192 printf "%-40s: User forced skip (SKIP_LOCALES set)\n" "locales" |
335 return; |
193 fi |
336 fi |
194 |
337 |
195 # Check what is going to be done and ask for consent |
338 if ! wget --quiet --show-progress --progress=bar "$FTP_URL/$upstream_file"; then |
196 for ff in $SOURCE_TARBALL $SOURCE_TARBALL.asc; do |
339 local CANDIDATE_TARBALL_LOCATION="" |
197 printf "%-40s: %s\n" $ff "$(check_tarball_source $ff)" |
340 CANDIDATE_TARBALL_LOCATION="$(printf "%s/%s/source/%s" "$(get_ftp_candidates_url "$PRODUCT" "$VERSION$VERSION_SUFFIX")" "$BUILD_ID" "$upstream_file" )" |
198 done |
341 wget --quiet --show-progress --progress=bar "$CANDIDATE_TARBALL_LOCATION" |
199 |
342 fi |
200 $(ask_cont_abort_question "Is this ok?") || exit 0 |
343 } |
201 |
344 |
202 # Try to download tar-ball from officiall mozilla-mirror |
345 function download_upstream_source_tarballs() { |
203 if [ ! -e $SOURCE_TARBALL ]; then |
346 # Try to download tar-ball from officiall mozilla-mirror |
204 wget https://ftp.mozilla.org/pub/$PRODUCT/releases/$VERSION$VERSION_SUFFIX/source/$SOURCE_TARBALL |
347 download_release_or_candidate_file "$SOURCE_TARBALL" |
205 fi |
348 download_release_or_candidate_file "$SOURCE_TARBALL.asc" |
206 # including signature |
349 |
207 if [ ! -e $SOURCE_TARBALL.asc ]; then |
350 # we might have an upstream archive already and can skip the checkout |
208 wget https://ftp.mozilla.org/pub/$PRODUCT/releases/$VERSION$VERSION_SUFFIX/source/$SOURCE_TARBALL.asc |
351 if [ -e "$SOURCE_TARBALL" ]; then |
209 fi |
352 get_source_stamp "$BUILD_ID" |
210 |
353 else |
211 # we might have an upstream archive already and can skip the checkout |
354 # We are working on a version that is not yet published on the mozilla mirror |
212 if [ -e $SOURCE_TARBALL ]; then |
355 # so we have to actually check out the repo |
213 if [ -z ${SKIP_LOCALES+x} ] && [ $LOCALES_CHANGED -ne 0 ]; then |
356 clone_and_repackage_sources |
214 # still need to extract the locale information from the archive |
357 fi |
215 echo "extract locale changesets" |
358 } |
216 tar -xf $SOURCE_TARBALL $LOCALE_FILE |
359 |
217 fi |
360 function clone_and_repackage_sources() { |
218 get_source_stamp "$BUILD_ID" |
361 if [ -d "$PRODUCT-$VERSION" ]; then |
219 else |
362 pushd "$PRODUCT-$VERSION" || exit 1 |
220 # We are working on a version that is not yet published on the mozilla mirror |
|
221 # so we have to actually check out the repo |
|
222 |
|
223 # mozilla |
|
224 if [ -d $PRODUCT-$VERSION ]; then |
|
225 pushd $PRODUCT-$VERSION || exit 1 |
|
226 _repourl=$(hg paths) |
363 _repourl=$(hg paths) |
227 case "$_repourl" in |
364 case "$_repourl" in |
228 *$BRANCH*) |
365 *$BRANCH*) |
229 echo "updating previous tree" |
366 echo "updating previous tree" |
230 hg pull |
367 hg pull |
231 popd || exit 1 |
368 popd || exit 1 |
232 ;; |
369 ;; |
233 * ) |
370 * ) |
234 echo "removing obsolete tree" |
371 echo "removing obsolete tree" |
235 popd || exit 1 |
372 popd || exit 1 |
236 rm -rf $PRODUCT-$VERSION |
373 rm -rf "$PRODUCT-$VERSION" |
237 ;; |
374 ;; |
238 esac |
375 esac |
239 fi |
376 fi |
240 if [ ! -d $PRODUCT-$VERSION ]; then |
377 if [ ! -d "$PRODUCT-$VERSION" ]; then |
241 echo "cloning new $BRANCH..." |
378 echo "cloning new $BRANCH..." |
242 hg clone https://hg.mozilla.org/$BRANCH $PRODUCT-$VERSION |
379 hg clone "https://hg.mozilla.org/$BRANCH $PRODUCT-$VERSION" |
243 if [ "$PRODUCT" = "thunderbird" ]; then |
380 if [ "$PRODUCT" = "thunderbird" ]; then |
244 hg clone https://hg.mozilla.org/releases/comm-$CHANNEL $PRODUCT-$VERSION/comm |
381 hg clone "https://hg.mozilla.org/releases/comm-$CHANNEL" "$PRODUCT-$VERSION/comm" |
245 fi |
382 fi |
246 fi |
383 fi |
247 pushd $PRODUCT-$VERSION || exit 1 |
384 pushd "$PRODUCT-$VERSION" || exit 1 |
248 |
385 |
249 # parse out the Firefox-release tag for this Thunderbird-checkout |
386 # parse out the Firefox-release tag for this Thunderbird-checkout |
250 if [ "$PRODUCT" = "thunderbird" ]; then |
387 if [ "$PRODUCT" = "thunderbird" ]; then |
251 FF_RELEASE_TAG=$(grep ^GECKO_HEAD_REV ./comm/.gecko_rev.yml | awk -F ' ' '{print $2}') || exit 1 |
388 FF_RELEASE_TAG=$(grep ^GECKO_HEAD_REV ./comm/.gecko_rev.yml | awk -F ' ' '{print $2}') || exit 1 |
252 echo "Parsed Firefox base ID from .gecko_rev.yml: $FF_RELEASE_TAG" |
389 echo "Parsed Firefox base ID from .gecko_rev.yml: $FF_RELEASE_TAG" |
253 else |
390 else |
254 FF_RELEASE_TAG="$RELEASE_TAG" |
391 FF_RELEASE_TAG="$RELEASE_TAG" |
255 fi |
392 fi |
256 |
393 |
257 hg update --check $FF_RELEASE_TAG |
394 hg update --check "$FF_RELEASE_TAG" |
258 [ "$FF_RELEASE_TAG" == "default" ] || hg update -r $FF_RELEASE_TAG |
395 [ "$FF_RELEASE_TAG" == "default" ] || hg update -r "$FF_RELEASE_TAG" |
259 # get repo and source stamp |
396 # get repo and source stamp |
260 REV=$(hg -R . parent --template="{node|short}\n") |
397 local REV=$(hg -R . parent --template="{node|short}\n") |
261 SOURCE_REPO=$(hg showconfig paths.default 2>/dev/null | head -n1 | sed -e "s/^ssh:/https:/") |
398 local SOURCE_REPO=$(hg showconfig paths.default 2>/dev/null | head -n1 | sed -e "s/^ssh:/https:/") |
262 TIMESTAMP=$(date +%Y%m%d%H%M%S) |
399 local TIMESTAMP=$(date +%Y%m%d%H%M%S) |
263 |
400 |
264 if [ "$PRODUCT" = "thunderbird" ]; then |
401 if [ "$PRODUCT" = "thunderbird" ]; then |
265 pushd comm || exit 1 |
402 pushd comm || exit 1 |
266 hg update --check $RELEASE_TAG |
403 hg update --check "$RELEASE_TAG" |
267 popd || exit 1 |
404 popd || exit 1 |
268 rm -rf thunderbird-${VERSION}/{,comm/}other-licenses/7zstub |
405 rm -rf thunderbird-"${VERSION}"/{,comm/}other-licenses/7zstub |
269 fi |
406 fi |
270 popd || exit 1 |
407 popd || exit 1 |
271 |
408 |
272 echo "Extending $TAR_STAMP with:" |
409 echo "Extending $TAR_STAMP with:" |
273 echo "RELEASE_REPO=${SOURCE_REPO}" |
410 echo "RELEASE_REPO=${SOURCE_REPO}" |
281 echo "RELEASE_REPO=$SOURCE_REPO" >> "$TAR_STAMP" |
418 echo "RELEASE_REPO=$SOURCE_REPO" >> "$TAR_STAMP" |
282 echo "RELEASE_TAG=$REV" >> "$TAR_STAMP" |
419 echo "RELEASE_TAG=$REV" >> "$TAR_STAMP" |
283 echo "RELEASE_TIMESTAMP=$TIMESTAMP" >> "$TAR_STAMP" |
420 echo "RELEASE_TIMESTAMP=$TIMESTAMP" >> "$TAR_STAMP" |
284 |
421 |
285 echo "creating archive..." |
422 echo "creating archive..." |
286 tar $compression -cf $PRODUCT-$VERSION$VERSION_SUFFIX.source.tar.xz --exclude=.hgtags --exclude=.hgignore --exclude=.hg --exclude=CVS $PRODUCT-$VERSION |
423 tar "$compression" -cf "$PRODUCT-$VERSION$VERSION_SUFFIX.source.tar.xz" --exclude=.hgtags --exclude=.hgignore --exclude=.hg --exclude=CVS "$PRODUCT-$VERSION" |
287 fi |
424 ALREADY_EXTRACTED_LOCALES_FILE=1 |
288 |
425 } |
289 if [ ! -z ${SKIP_LOCALES+x} ]; then |
426 |
290 echo "Skipping locales-creation." |
427 function create_locales_tarballs() { |
291 exit 0 |
428 if [ ! -z ${SKIP_LOCALES+x} ]; then |
292 fi |
429 echo "Skipping locales-creation." |
293 |
430 exit 0 |
294 if [ $LOCALES_CHANGED -ne 0 ]; then |
431 fi |
|
432 |
|
433 if [ "$LOCALES_CHANGED" -ne 0 ]; then |
|
434 clone_and_repackage_locales |
|
435 elif [ -f "l10n-$PREV_VERSION$PREV_VERSION_SUFFIX.tar.xz" ]; then |
|
436 # Locales did not change, but the old tar-ball is in this directory |
|
437 # Simply rename it: |
|
438 echo "Moving l10n-$PREV_VERSION$PREV_VERSION_SUFFIX.tar.xz to l10n-$VERSION$VERSION_SUFFIX.tar.xz" |
|
439 mv "l10n-$PREV_VERSION$PREV_VERSION_SUFFIX.tar.xz" "l10n-$VERSION$VERSION_SUFFIX.tar.xz" |
|
440 fi |
|
441 } |
|
442 |
|
443 function clone_and_repackage_locales() { |
295 # l10n |
444 # l10n |
296 echo "fetching locales..." |
445 FINAL_L10N_BASE="l10n" |
297 test ! -d l10n && mkdir l10n |
446 FF_L10N_BASE="l10n" # Only change this in TB-builds to a separate dir |
298 jq -r 'to_entries[]| "\(.key) \(.value|.revision)"' $LOCALE_FILE | \ |
447 TB_L10N_BASE="l10n_tb" |
299 while read locale changeset ; do |
448 |
|
449 # If we are doing Thunderbird, we'll have to checkout both TB and FF l10n-repos |
|
450 # Thunderbird has one single mono-repo, FF has one for each language |
|
451 if [ "$PRODUCT" = "thunderbird" ]; then |
|
452 echo "Fetching Thunderbird locales..." |
|
453 if [ -d "$TB_L10N_BASE/.hg" ]; then |
|
454 pushd "$TB_L10N_BASE/" || exit 1 |
|
455 hg pull || exit 1 |
|
456 popd || exit 1 |
|
457 else |
|
458 hg clone "https://hg.mozilla.org/projects/comm-l10n/" "$TB_L10N_BASE/" || exit 1 |
|
459 fi |
|
460 # Just using the first entry here, as all languages have the same changeset |
|
461 tb_changeset=$(jq -r 'to_entries[0]| "\(.key) \(.value|.revision)"' "$TB_LOCALE_FILE" | cut -d " " -f 2) |
|
462 [ "$RELEASE_TAG" == "default" ] || hg -R "$TB_L10N_BASE/" up -C -r "$tb_changeset" || exit 1 |
|
463 FF_L10N_BASE="l10n_ff" |
|
464 fi |
|
465 |
|
466 test ! -d $FF_L10N_BASE && mkdir $FF_L10N_BASE |
|
467 # No-op, if we are building FF: |
|
468 test ! -d $FINAL_L10N_BASE && mkdir $FINAL_L10N_BASE |
|
469 |
|
470 # This is only relevant for Thunderbird-builds |
|
471 # Here, the relevant directories we need to copy from FF and from TB |
|
472 # are specified in a python-file in the tarball |
|
473 # Is of form '{lang}/Foo/bar/ {lang}/Baz/bar/ ..' |
|
474 ff_locale_template=$(get_locales_directories "GECKO_STRINGS_PATTERNS") |
|
475 tb_locale_template=$(get_locales_directories "COMM_STRINGS_PATTERNS") |
|
476 |
|
477 echo "Fetching Browser locales..." |
|
478 jq -r 'to_entries[]| "\(.key) \(.value|.revision)"' "$FF_LOCALE_FILE" | \ |
|
479 while read -r locale changeset ; do |
300 case $locale in |
480 case $locale in |
301 ja-JP-mac|en-US) |
481 ja-JP-mac|en-US) |
302 ;; |
482 ;; |
303 *) |
483 *) |
304 echo "reading changeset information for $locale" |
484 echo "reading changeset information for $locale" |
305 echo "fetching $locale changeset $changeset ..." |
485 echo "fetching $locale changeset $changeset ..." |
306 if [ -d "l10n/$locale/.hg" ]; then |
486 if [ -d "$FF_L10N_BASE/$locale/.hg" ]; then |
307 pushd "l10n/$locale" || exit 1 |
487 pushd "$FF_L10N_BASE/$locale" || exit 1 |
308 hg pull |
488 hg pull || exit 1 |
309 popd || exit 1 |
489 popd || exit 1 |
310 else |
490 else |
311 hg clone "https://hg.mozilla.org/l10n-central/$locale" "l10n/$locale" |
491 hg clone "https://hg.mozilla.org/l10n-central/$locale" "$FF_L10N_BASE/$locale" || exit 1 |
312 fi |
492 fi |
313 [ "$RELEASE_TAG" == "default" ] || hg -R "l10n/$locale" up -C -r "$changeset" |
493 [ "$RELEASE_TAG" == "default" ] || hg -R "$FF_L10N_BASE/$locale" up -C -r "$changeset" || exit 1 |
|
494 |
|
495 # If we are doing TB, we have to merge both l10n-repos |
|
496 if [ "$PRODUCT" = "thunderbird" ] && test -d "$TB_L10N_BASE/$locale/" ; then |
|
497 create_and_copy_locales "$locale" "$FF_L10N_BASE" "$ff_locale_template" "$FINAL_L10N_BASE" |
|
498 create_and_copy_locales "$locale" "$TB_L10N_BASE" "$tb_locale_template" "$FINAL_L10N_BASE" |
|
499 fi |
314 ;; |
500 ;; |
315 esac |
501 esac |
316 done |
502 done |
317 echo "creating l10n archive..." |
503 echo "creating l10n archive..." |
318 if [ "$PRODUCT" = "thunderbird" ]; then |
504 if [ "$PRODUCT" = "thunderbird" ]; then |
319 TB_TAR_FLAGS="--exclude=browser --exclude=suite" |
505 TB_TAR_FLAGS="--exclude=suite" |
320 fi |
506 fi |
321 tar $compression -cf l10n-$VERSION$VERSION_SUFFIX.tar.xz \ |
507 tar "$compression" -cf "l10n-$VERSION$VERSION_SUFFIX.tar.xz" \ |
322 --exclude=.hgtags --exclude=.hgignore --exclude=.hg \ |
508 --exclude=.hgtags --exclude=.hgignore --exclude=.hg \ |
323 $TB_TAR_FLAGS \ |
509 "$TB_TAR_FLAGS" \ |
324 l10n |
510 "$FINAL_L10N_BASE" |
325 elif [ -f "l10n-$PREV_VERSION$PREV_VERSION_SUFFIX.tar.xz" ]; then |
511 } |
326 # Locales did not change, but the old tar-ball is in this directory |
512 |
327 # Simply rename it: |
513 function clean_up_old_tarballs() { |
328 echo "Moving l10n-$PREV_VERSION$PREV_VERSION_SUFFIX.tar.xz to l10n-$VERSION$VERSION_SUFFIX.tar.xz" |
514 if [ -e "$PREV_SOURCE_TARBALL" ]; then |
329 mv "l10n-$PREV_VERSION$PREV_VERSION_SUFFIX.tar.xz" "l10n-$VERSION$VERSION_SUFFIX.tar.xz" |
515 echo "" |
330 fi |
516 echo "Deleting old sources tarball $PREV_SOURCE_TARBALL" |
|
517 ask_cont_abort_question "Is this ok?" || exit 0 |
|
518 rm "$PREV_SOURCE_TARBALL" |
|
519 rm "$PREV_SOURCE_TARBALL.asc" |
|
520 # if old and new lang-tarball are there, delete the old one |
|
521 if [ -f "l10n-$PREV_VERSION$PREV_VERSION_SUFFIX.tar.xz" ] && [ -f "l10n-$VERSION$VERSION_SUFFIX.tar.xz" ]; then |
|
522 rm "l10n-$PREV_VERSION$PREV_VERSION_SUFFIX.tar.xz" |
|
523 fi |
|
524 fi |
|
525 } |
|
526 |
|
527 main "$@" |