1 #!/bin/bash |
1 #!/bin/bash |
2 |
2 |
3 # TODO |
3 function print_usage_and_exit() { |
4 # http://ftp.mozilla.org/pub/firefox/candidates/48.0-candidates/build2/linux-x86_64/en-US/firefox-48.0.json |
4 echo "Usage: create-tar.sh tar_stamps" |
5 # "moz_source_stamp": "c1de04f39fa956cfce83f6065b0e709369215ed5" |
5 echo "" |
6 # http://ftp.mozilla.org/pub/firefox/candidates/48.0-candidates/build2/l10n_changesets.txt |
6 echo "Where tar_stamps should look like this:" |
7 |
7 echo "" |
8 CHANNEL="beta" |
8 cat << EOF |
|
9 # Node ID: 64ee63facd4ff96b3e8590cff559d7e97ac6b061 |
|
10 PRODUCT="firefox" # "firefox" or "thunderbird" |
|
11 CHANNEL="esr60" |
|
12 VERSION="60.7.0" |
|
13 VERSION_SUFFIX="esr" |
|
14 RELEASE_TAG="" # Needs only to be set if no tar-ball can be downloaded |
|
15 PREV_VERSION="60.6.3" # Prev. version only needed for locales (leave empty to force l10n-generation) |
|
16 PREV_VERSION_SUFFIX="esr" |
|
17 #SKIP_LOCALES="" # Uncomment to skip l10n and compare-locales-generation |
|
18 EOF |
|
19 |
|
20 exit 1 |
|
21 } |
|
22 |
|
23 if [ $# -ne 1 ]; then |
|
24 print_usage_and_exit |
|
25 fi |
|
26 |
|
27 # Sourcing the given tar_stamps-file to have the variables available |
|
28 source "$1" || print_usage_and_exit |
|
29 |
|
30 # Internal variables |
9 BRANCH="releases/mozilla-$CHANNEL" |
31 BRANCH="releases/mozilla-$CHANNEL" |
10 RELEASE_TAG="1ea7b51ef5bb91bdc34fb7406fd4d35ed7961363" |
32 if [ "$PRODUCT" = "firefox" ]; then |
11 VERSION="65.0.1" |
33 LOCALE_FILE="firefox-$VERSION/browser/locales/l10n-changesets.json" |
12 VERSION_SUFFIX="" |
34 else |
13 LOCALE_FILE="firefox-$VERSION/browser/locales/l10n-changesets.json" |
35 LOCALE_FILE="thunderbird-$VERSION/comm/mail/locales/l10n-changesets.json" |
|
36 fi |
|
37 |
|
38 SOURCE_TARBALL="$PRODUCT-$VERSION$VERSION_SUFFIX.source.tar.xz" |
|
39 FTP_URL="https://ftp.mozilla.org/pub/$PRODUCT/releases/$VERSION$VERSION_SUFFIX/source" |
|
40 # Make first letter of PRODCUT upper case |
|
41 PRODUCT_CAP="${PRODUCT^}" |
|
42 LOCALES_URL="https://product-details.mozilla.org/1.0/l10n/$PRODUCT_CAP" |
|
43 # Exit script on CTRL+C |
|
44 trap "exit" INT |
|
45 |
|
46 function check_tarball_source () { |
|
47 TARBALL=$1 |
|
48 # Print out what is going to be done: |
|
49 if [ -e $TARBALL ]; then |
|
50 echo "Reuse existing file" |
|
51 elif wget --spider $FTP_URL/$TARBALL 2> /dev/null; then |
|
52 echo "Download file" |
|
53 else |
|
54 echo "Mercurial checkout" |
|
55 fi |
|
56 } |
|
57 |
|
58 function ask_cont_abort_question() { |
|
59 while true; do |
|
60 read -p "$1 [(c)ontinue/(a)bort] " ca |
|
61 case $ca in |
|
62 [Cc]* ) return 0 ;; |
|
63 [Aa]* ) return 1 ;; |
|
64 * ) echo "Please answer c or a.";; |
|
65 esac |
|
66 done |
|
67 } |
|
68 |
|
69 function check_for_binary() { |
|
70 if ! test -x $1; then |
|
71 echo "$1 is missing: execute zypper in $2" |
|
72 exit 5 |
|
73 fi |
|
74 } |
|
75 |
|
76 function locales_get() { |
|
77 TMP_VERSION="$1" |
|
78 URL_TO_CHECK="${LOCALES_URL}-${TMP_VERSION}" |
|
79 |
|
80 LAST_FOUND="" |
|
81 # Unfortunately, locales-files are not associated to releases, but to builds. |
|
82 # And since we don't know which build was the final build, we go from 9 downwards |
|
83 # try to find the latest one that exists (assuming there are no more than 9 builds). |
|
84 # Error only if not even the first one exists |
|
85 for BUILD_ID in $(seq 9 -1 0); do |
|
86 FINAL_URL="${URL_TO_CHECK}-build${BUILD_ID}.json" |
|
87 if wget --quiet --spider "$FINAL_URL"; then |
|
88 LAST_FOUND="$FINAL_URL" |
|
89 break |
|
90 fi |
|
91 done |
|
92 |
|
93 if [ "$LAST_FOUND" != "" ]; then |
|
94 echo "$LAST_FOUND" |
|
95 return 0 |
|
96 else |
|
97 echo "Error: Could not find locales-file (json) for Firefox $TMP_VERSION !" 1>&2 |
|
98 return 1 |
|
99 fi |
|
100 } |
|
101 |
|
102 function locales_parse() { |
|
103 URL="$1" |
|
104 curl -s "$URL" | python -c "import json; import sys; \ |
|
105 print('\n'.join(['{} {}'.format(key, value['changeset']) \ |
|
106 for key, value in sorted(json.load(sys.stdin)['locales'].items())]));" |
|
107 } |
|
108 |
|
109 function locales_unchanged() { |
|
110 # If no json-file for one of the versions can be found, we say "they changed" |
|
111 prev_url=$(locales_get "$PREV_VERSION$PREV_VERSION_SUFFIX") || return 1 |
|
112 curr_url=$(locales_get "$VERSION$VERSION_SUFFIX") || return 1 |
|
113 |
|
114 prev_content=$(locales_parse "$prev_url") || exit 1 |
|
115 curr_content=$(locales_parse "$curr_url") || exit 1 |
|
116 |
|
117 diff -y --suppress-common-lines -d <(echo "$prev_content") <(echo "$curr_content") |
|
118 } |
14 |
119 |
15 # check required tools |
120 # check required tools |
16 test -x /usr/bin/hg || ( echo "hg missing: execute zypper in mercurial"; exit 5 ) |
121 check_for_binary /usr/bin/hg "mercurial" |
17 test -x /usr/bin/jq || ( echo "jq missing: execute zypper in jq"; exit 5 ) |
122 check_for_binary /usr/bin/jq "jq" |
|
123 which python > /dev/null || exit 1 |
18 |
124 |
19 # use parallel compression, if available |
125 # use parallel compression, if available |
20 compression='-J' |
126 compression='-J' |
21 pixz -h > /dev/null 2>&1 |
127 pixz -h > /dev/null 2>&1 |
22 if (($? != 127)); then |
128 if (($? != 127)); then |
23 compression='-Ipixz' |
129 compression='-Ipixz' |
24 fi |
130 fi |
25 |
131 |
|
132 if [ -z ${SKIP_LOCALES+x} ]; then |
|
133 # TODO: Thunderbird has usually "default" as locale entry. |
|
134 # There we probably need to double-check Firefox-locals |
|
135 # For now, just download every time for Thunderbird |
|
136 if [ "$PRODUCT" = "firefox" ] && [ "$PREV_VERSION" != "" ] && locales_unchanged; then |
|
137 printf "%-40s: Did not change. Skipping.\n" "locales" |
|
138 LOCALES_CHANGED=0 |
|
139 else |
|
140 printf "%-40s: Need to download.\n" "locales" |
|
141 LOCALES_CHANGED=1 |
|
142 fi |
|
143 else |
|
144 printf "%-40s: User forced skip (SKIP_LOCALES set)\n" "locales" |
|
145 fi |
|
146 |
|
147 # Check what is going to be done and ask for consent |
|
148 for ff in $SOURCE_TARBALL $SOURCE_TARBALL.asc; do |
|
149 printf "%-40s: %s\n" $ff "$(check_tarball_source $ff)" |
|
150 done |
|
151 |
|
152 $(ask_cont_abort_question "Is this ok?") || exit 0 |
|
153 |
|
154 # Try to download tar-ball from officiall mozilla-mirror |
|
155 if [ ! -e $SOURCE_TARBALL ]; then |
|
156 wget https://ftp.mozilla.org/pub/$PRODUCT/releases/$VERSION$VERSION_SUFFIX/source/$SOURCE_TARBALL |
|
157 fi |
|
158 # including signature |
|
159 if [ ! -e $SOURCE_TARBALL.asc ]; then |
|
160 wget https://ftp.mozilla.org/pub/$PRODUCT/releases/$VERSION$VERSION_SUFFIX/source/$SOURCE_TARBALL.asc |
|
161 fi |
|
162 |
26 # we might have an upstream archive already and can skip the checkout |
163 # we might have an upstream archive already and can skip the checkout |
27 if [ -e firefox-$VERSION$VERSION_SUFFIX.source.tar.xz ]; then |
164 if [ -e $SOURCE_TARBALL ]; then |
28 echo "skip firefox checkout and use available archive" |
165 if [ -z ${SKIP_LOCALES+x} ] && [ $LOCALES_CHANGED -ne 0 ]; then |
29 # still need to extract the locale information from the archive |
166 # still need to extract the locale information from the archive |
30 echo "extract locale changesets" |
167 echo "extract locale changesets" |
31 tar -xf firefox-$VERSION$VERSION_SUFFIX.source.tar.xz $LOCALE_FILE |
168 tar -xf $SOURCE_TARBALL $LOCALE_FILE |
|
169 fi |
32 else |
170 else |
|
171 # We are working on a version that is not yet published on the mozilla mirror |
|
172 # so we have to actually check out the repo |
|
173 |
33 # mozilla |
174 # mozilla |
34 if [ -d firefox-$VERSION ]; then |
175 if [ -d $PRODUCT-$VERSION ]; then |
35 pushd firefox-$VERSION |
176 pushd $PRODUCT-$VERSION || exit 1 |
36 _repourl=$(hg paths) |
177 _repourl=$(hg paths) |
37 case "$_repourl" in |
178 case "$_repourl" in |
38 *$BRANCH*) |
179 *$BRANCH*) |
39 echo "updating previous tree" |
180 echo "updating previous tree" |
40 hg pull |
181 hg pull |
41 popd |
182 popd || exit 1 |
42 ;; |
183 ;; |
43 * ) |
184 * ) |
44 echo "removing obsolete tree" |
185 echo "removing obsolete tree" |
45 popd |
186 popd || exit 1 |
46 rm -rf firefox-$VERSION |
187 rm -rf $PRODUCT-$VERSION |
47 ;; |
188 ;; |
48 esac |
189 esac |
49 fi |
190 fi |
50 if [ ! -d firefox-$VERSION ]; then |
191 if [ ! -d $PRODUCT-$VERSION ]; then |
51 echo "cloning new $BRANCH..." |
192 echo "cloning new $BRANCH..." |
52 hg clone http://hg.mozilla.org/$BRANCH firefox-$VERSION |
193 hg clone http://hg.mozilla.org/$BRANCH $PRODUCT-$VERSION |
53 fi |
194 if [ "$PRODUCT" = "thunderbird" ]; then |
54 pushd firefox-$VERSION |
195 hg clone http://hg.mozilla.org/releases/comm-$CHANNEL $PRODUCT-$VERSION/comm |
55 hg update --check |
196 fi |
56 [ "$RELEASE_TAG" == "default" ] || hg update -r $RELEASE_TAG |
197 fi |
|
198 pushd $PRODUCT-$VERSION || exit 1 |
|
199 |
|
200 # parse out the Firefox-release tag for this Thunderbird-checkout |
|
201 if [ "$PRODUCT" = "thunderbird" ]; then |
|
202 FF_RELEASE_TAG=$(grep ^GECKO_HEAD_REV ./comm/.gecko_rev.yml | awk -F ' ' '{print $2}') || exit 1 |
|
203 echo "Parsed Firefox base ID from .gecko_rev.yml: $FF_RELEASE_TAG" |
|
204 else |
|
205 FF_RELEASE_TAG="$RELEASE_TAG" |
|
206 fi |
|
207 |
|
208 hg update --check $FF_RELEASE_TAG |
|
209 [ "$FF_RELEASE_TAG" == "default" ] || hg update -r $FF_RELEASE_TAG |
57 # get repo and source stamp |
210 # get repo and source stamp |
58 echo -n "REV=" > ../source-stamp.txt |
211 echo -n "REV=" > ../source-stamp.txt |
59 hg -R . parent --template="{node|short}\n" >> ../source-stamp.txt |
212 hg -R . parent --template="{node|short}\n" >> ../source-stamp.txt |
60 echo -n "REPO=" >> ../source-stamp.txt |
213 echo -n "REPO=" >> ../source-stamp.txt |
61 hg showconfig paths.default 2>/dev/null | head -n1 | sed -e "s/^ssh:/http:/" >> ../source-stamp.txt |
214 hg showconfig paths.default 2>/dev/null | head -n1 | sed -e "s/^ssh:/http:/" >> ../source-stamp.txt |
62 popd |
215 |
|
216 if [ "$PRODUCT" = "thunderbird" ]; then |
|
217 pushd comm || exit 1 |
|
218 hg update --check $RELEASE_TAG |
|
219 popd || exit 1 |
|
220 rm -rf thunderbird-${VERSION}/{,comm/}other-licenses/7zstub |
|
221 fi |
|
222 popd || exit 1 |
63 |
223 |
64 echo "creating archive..." |
224 echo "creating archive..." |
65 tar $compression -cf firefox-$VERSION$VERSION_SUFFIX.source.tar.xz --exclude=.hgtags --exclude=.hgignore --exclude=.hg --exclude=CVS firefox-$VERSION |
225 tar $compression -cf $PRODUCT-$VERSION$VERSION_SUFFIX.source.tar.xz --exclude=.hgtags --exclude=.hgignore --exclude=.hg --exclude=CVS $PRODUCT-$VERSION |
66 fi |
226 fi |
67 |
227 |
68 # l10n |
228 if [ ! -z ${SKIP_LOCALES+x} ]; then |
69 echo "fetching locales..." |
229 echo "Skipping locales-creation." |
70 test ! -d l10n && mkdir l10n |
230 exit 0 |
71 jq -r 'to_entries[]| "\(.key) \(.value|.revision)"' $LOCALE_FILE | \ |
231 fi |
72 while read locale changeset ; do |
232 |
73 case $locale in |
233 if [ $LOCALES_CHANGED -ne 0 ]; then |
74 ja-JP-mac|en-US) |
234 # l10n |
75 ;; |
235 echo "fetching locales..." |
76 *) |
236 test ! -d l10n && mkdir l10n |
77 echo "reading changeset information for $locale" |
237 jq -r 'to_entries[]| "\(.key) \(.value|.revision)"' $LOCALE_FILE | \ |
78 echo "fetching $locale changeset $changeset ..." |
238 while read locale changeset ; do |
79 hg clone http://hg.mozilla.org/l10n-central/$locale l10n/$locale |
239 case $locale in |
80 [ "$RELEASE_TAG" == "default" ] || hg -R l10n/$locale up -C -r $changeset |
240 ja-JP-mac|en-US) |
81 ;; |
241 ;; |
82 esac |
242 *) |
83 done |
243 echo "reading changeset information for $locale" |
84 echo "creating l10n archive..." |
244 echo "fetching $locale changeset $changeset ..." |
85 tar $compression -cf l10n-$VERSION$VERSION_SUFFIX.tar.xz --exclude=.hgtags --exclude=.hgignore --exclude=.hg l10n |
245 if [ -d "l10n/$locale/.hg" ]; then |
|
246 pushd "l10n/$locale" || exit 1 |
|
247 hg pull |
|
248 popd || exit 1 |
|
249 else |
|
250 hg clone "http://hg.mozilla.org/l10n-central/$locale" "l10n/$locale" |
|
251 fi |
|
252 [ "$RELEASE_TAG" == "default" ] || hg -R "l10n/$locale" up -C -r "$changeset" |
|
253 ;; |
|
254 esac |
|
255 done |
|
256 echo "creating l10n archive..." |
|
257 if [ "$PRODUCT" = "thunderbird" ]; then |
|
258 TB_TAR_FLAGS="--exclude=browser --exclude=suite" |
|
259 fi |
|
260 tar $compression -cf l10n-$VERSION$VERSION_SUFFIX.tar.xz \ |
|
261 --exclude=.hgtags --exclude=.hgignore --exclude=.hg \ |
|
262 $TB_TAR_FLAGS \ |
|
263 l10n |
|
264 elif [ -f "l10n-$PREV_VERSION$PREV_VERSION_SUFFIX.tar.xz" ]; then |
|
265 # Locales did not change, but the old tar-ball is in this directory |
|
266 # Simply rename it: |
|
267 echo "Moving l10n-$PREV_VERSION$PREV_VERSION_SUFFIX.tar.xz to l10n-$VERSION$VERSION_SUFFIX.tar.xz" |
|
268 mv "l10n-$PREV_VERSION$PREV_VERSION_SUFFIX.tar.xz" "l10n-$VERSION$VERSION_SUFFIX.tar.xz" |
|
269 fi |
86 |
270 |
87 # compare-locales |
271 # compare-locales |
88 echo "creating compare-locales" |
272 echo "creating compare-locales" |
89 hg clone http://hg.mozilla.org/build/compare-locales |
273 if [ -d compare-locales/.hg ]; then |
|
274 pushd compare-locales || exit 1 |
|
275 hg pull |
|
276 popd || exit 1 |
|
277 else |
|
278 hg clone http://hg.mozilla.org/build/compare-locales |
|
279 fi |
90 tar $compression -cf compare-locales.tar.xz --exclude=.hgtags --exclude=.hgignore --exclude=.hg compare-locales |
280 tar $compression -cf compare-locales.tar.xz --exclude=.hgtags --exclude=.hgignore --exclude=.hg compare-locales |
91 |
281 |