author | Wolfgang Rosenauer <wr@rosenauer.org> |
Tue, 19 Nov 2019 22:54:22 +0100 | |
changeset 1113 | 8e9195853a32 |
parent 1088 | 84cdfb476431 |
parent 1111 | 97a6da6d7e29 |
child 1114 | 572ec48f3fe8 |
permissions | -rwxr-xr-x |
262
b680975515e0
probably last working Firefox-on-xulrunner build
Wolfgang Rosenauer <wr@rosenauer.org>
parents:
diff
changeset
|
1 |
#!/bin/bash |
b680975515e0
probably last working Firefox-on-xulrunner build
Wolfgang Rosenauer <wr@rosenauer.org>
parents:
diff
changeset
|
2 |
|
1097 | 3 |
function print_usage_and_exit() { |
1111 | 4 |
echo "Usage: create-tar.sh tar_stamps" |
1097 | 5 |
echo "" |
1111 | 6 |
echo "Where tar_stamps should look like this:" |
1097 | 7 |
echo "" |
8 |
cat << EOF |
|
9 |
# Node ID: 64ee63facd4ff96b3e8590cff559d7e97ac6b061 |
|
10 |
PRODUCT="firefox" # "firefox" or "thunderbird" |
|
11 |
CHANNEL="esr60" |
|
12 |
VERSION="60.7.0" |
|
13 |
VERSION_SUFFIX="esr" |
|
1111 | 14 |
RELEASE_TAG="" # Needs only to be set if no tar-ball can be downloaded |
1097 | 15 |
PREV_VERSION="60.6.3" # Prev. version only needed for locales (leave empty to force l10n-generation) |
16 |
PREV_VERSION_SUFFIX="esr" |
|
17 |
#SKIP_LOCALES="" # Uncomment to skip l10n and compare-locales-generation |
|
18 |
EOF |
|
19 |
||
20 |
exit 1 |
|
21 |
} |
|
22 |
||
23 |
if [ $# -ne 1 ]; then |
|
24 |
print_usage_and_exit |
|
25 |
fi |
|
26 |
||
1111 | 27 |
# Sourcing the given tar_stamps-file to have the variables available |
1097 | 28 |
source "$1" || print_usage_and_exit |
29 |
||
30 |
# Internal variables |
|
31 |
BRANCH="releases/mozilla-$CHANNEL" |
|
32 |
if [ "$PRODUCT" = "firefox" ]; then |
|
33 |
LOCALE_FILE="firefox-$VERSION/browser/locales/l10n-changesets.json" |
|
34 |
else |
|
35 |
LOCALE_FILE="thunderbird-$VERSION/comm/mail/locales/l10n-changesets.json" |
|
36 |
fi |
|
37 |
||
38 |
SOURCE_TARBALL="$PRODUCT-$VERSION$VERSION_SUFFIX.source.tar.xz" |
|
39 |
FTP_URL="https://ftp.mozilla.org/pub/$PRODUCT/releases/$VERSION$VERSION_SUFFIX/source" |
|
40 |
# Make first letter of PRODCUT upper case |
|
41 |
PRODUCT_CAP="${PRODUCT^}" |
|
42 |
LOCALES_URL="https://product-details.mozilla.org/1.0/l10n/$PRODUCT_CAP" |
|
43 |
# Exit script on CTRL+C |
|
44 |
trap "exit" INT |
|
45 |
||
46 |
function check_tarball_source () { |
|
47 |
TARBALL=$1 |
|
48 |
# Print out what is going to be done: |
|
49 |
if [ -e $TARBALL ]; then |
|
50 |
echo "Reuse existing file" |
|
51 |
elif wget --spider $FTP_URL/$TARBALL 2> /dev/null; then |
|
52 |
echo "Download file" |
|
53 |
else |
|
54 |
echo "Mercurial checkout" |
|
55 |
fi |
|
56 |
} |
|
909 | 57 |
|
1097 | 58 |
function ask_cont_abort_question() { |
59 |
while true; do |
|
60 |
read -p "$1 [(c)ontinue/(a)bort] " ca |
|
61 |
case $ca in |
|
62 |
[Cc]* ) return 0 ;; |
|
63 |
[Aa]* ) return 1 ;; |
|
64 |
* ) echo "Please answer c or a.";; |
|
65 |
esac |
|
66 |
done |
|
67 |
} |
|
68 |
||
69 |
function check_for_binary() { |
|
70 |
if ! test -x $1; then |
|
71 |
echo "$1 is missing: execute zypper in $2" |
|
72 |
exit 5 |
|
73 |
fi |
|
74 |
} |
|
75 |
||
76 |
function locales_get() { |
|
77 |
TMP_VERSION="$1" |
|
78 |
URL_TO_CHECK="${LOCALES_URL}-${TMP_VERSION}" |
|
79 |
||
80 |
LAST_FOUND="" |
|
81 |
# Unfortunately, locales-files are not associated to releases, but to builds. |
|
1098 | 82 |
# And since we don't know which build was the final build, we go from 9 downwards |
83 |
# try to find the latest one that exists (assuming there are no more than 9 builds). |
|
1097 | 84 |
# Error only if not even the first one exists |
1098 | 85 |
for BUILD_ID in $(seq 9 -1 0); do |
1097 | 86 |
FINAL_URL="${URL_TO_CHECK}-build${BUILD_ID}.json" |
87 |
if wget --quiet --spider "$FINAL_URL"; then |
|
88 |
LAST_FOUND="$FINAL_URL" |
|
1098 | 89 |
break |
1097 | 90 |
fi |
91 |
done |
|
1098 | 92 |
|
93 |
if [ "$LAST_FOUND" != "" ]; then |
|
94 |
echo "$LAST_FOUND" |
|
95 |
return 0 |
|
96 |
else |
|
97 |
echo "Error: Could not find locales-file (json) for Firefox $TMP_VERSION !" 1>&2 |
|
98 |
return 1 |
|
99 |
fi |
|
1097 | 100 |
} |
101 |
||
102 |
function locales_parse() { |
|
103 |
URL="$1" |
|
104 |
curl -s "$URL" | python -c "import json; import sys; \ |
|
105 |
print('\n'.join(['{} {}'.format(key, value['changeset']) \ |
|
106 |
for key, value in sorted(json.load(sys.stdin)['locales'].items())]));" |
|
107 |
} |
|
108 |
||
109 |
function locales_unchanged() { |
|
110 |
# If no json-file for one of the versions can be found, we say "they changed" |
|
111 |
prev_url=$(locales_get "$PREV_VERSION$PREV_VERSION_SUFFIX") || return 1 |
|
112 |
curr_url=$(locales_get "$VERSION$VERSION_SUFFIX") || return 1 |
|
113 |
||
114 |
prev_content=$(locales_parse "$prev_url") || exit 1 |
|
115 |
curr_content=$(locales_parse "$curr_url") || exit 1 |
|
116 |
||
117 |
diff -y --suppress-common-lines -d <(echo "$prev_content") <(echo "$curr_content") |
|
118 |
} |
|
262
b680975515e0
probably last working Firefox-on-xulrunner build
Wolfgang Rosenauer <wr@rosenauer.org>
parents:
diff
changeset
|
119 |
|
1032
8220ea23b47d
remove obsolete patches
Wolfgang Rosenauer <wr@rosenauer.org>
parents:
1031
diff
changeset
|
120 |
# check required tools |
1097 | 121 |
check_for_binary /usr/bin/hg "mercurial" |
122 |
check_for_binary /usr/bin/jq "jq" |
|
123 |
which python > /dev/null || exit 1 |
|
1032
8220ea23b47d
remove obsolete patches
Wolfgang Rosenauer <wr@rosenauer.org>
parents:
1031
diff
changeset
|
124 |
|
990
0d76004e9fa4
support parallel compression if available
Wolfgang Rosenauer <wr@rosenauer.org>
parents:
986
diff
changeset
|
125 |
# use parallel compression, if available |
0d76004e9fa4
support parallel compression if available
Wolfgang Rosenauer <wr@rosenauer.org>
parents:
986
diff
changeset
|
126 |
compression='-J' |
0d76004e9fa4
support parallel compression if available
Wolfgang Rosenauer <wr@rosenauer.org>
parents:
986
diff
changeset
|
127 |
pixz -h > /dev/null 2>&1 |
0d76004e9fa4
support parallel compression if available
Wolfgang Rosenauer <wr@rosenauer.org>
parents:
986
diff
changeset
|
128 |
if (($? != 127)); then |
0d76004e9fa4
support parallel compression if available
Wolfgang Rosenauer <wr@rosenauer.org>
parents:
986
diff
changeset
|
129 |
compression='-Ipixz' |
0d76004e9fa4
support parallel compression if available
Wolfgang Rosenauer <wr@rosenauer.org>
parents:
986
diff
changeset
|
130 |
fi |
0d76004e9fa4
support parallel compression if available
Wolfgang Rosenauer <wr@rosenauer.org>
parents:
986
diff
changeset
|
131 |
|
1097 | 132 |
if [ -z ${SKIP_LOCALES+x} ]; then |
133 |
# TODO: Thunderbird has usually "default" as locale entry. |
|
134 |
# There we probably need to double-check Firefox-locals |
|
135 |
# For now, just download every time for Thunderbird |
|
136 |
if [ "$PRODUCT" = "firefox" ] && [ "$PREV_VERSION" != "" ] && locales_unchanged; then |
|
137 |
printf "%-40s: Did not change. Skipping.\n" "locales" |
|
138 |
LOCALES_CHANGED=0 |
|
139 |
else |
|
140 |
printf "%-40s: Need to download.\n" "locales" |
|
141 |
LOCALES_CHANGED=1 |
|
142 |
fi |
|
143 |
else |
|
144 |
printf "%-40s: User forced skip (SKIP_LOCALES set)\n" "locales" |
|
145 |
fi |
|
146 |
||
147 |
# Check what is going to be done and ask for consent |
|
148 |
for ff in $SOURCE_TARBALL $SOURCE_TARBALL.asc; do |
|
149 |
printf "%-40s: %s\n" $ff "$(check_tarball_source $ff)" |
|
150 |
done |
|
151 |
||
152 |
$(ask_cont_abort_question "Is this ok?") || exit 0 |
|
153 |
||
154 |
# Try to download tar-ball from officiall mozilla-mirror |
|
155 |
if [ ! -e $SOURCE_TARBALL ]; then |
|
156 |
wget https://ftp.mozilla.org/pub/$PRODUCT/releases/$VERSION$VERSION_SUFFIX/source/$SOURCE_TARBALL |
|
157 |
fi |
|
158 |
# including signature |
|
159 |
if [ ! -e $SOURCE_TARBALL.asc ]; then |
|
160 |
wget https://ftp.mozilla.org/pub/$PRODUCT/releases/$VERSION$VERSION_SUFFIX/source/$SOURCE_TARBALL.asc |
|
161 |
fi |
|
162 |
||
1056
90e1f32cf034
several changes to make upstream tarballs a good neighbour for locale fetching and HG checkouts
Wolfgang Rosenauer <wr@rosenauer.org>
parents:
1053
diff
changeset
|
163 |
# we might have an upstream archive already and can skip the checkout |
1097 | 164 |
if [ -e $SOURCE_TARBALL ]; then |
1098 | 165 |
if [ -z ${SKIP_LOCALES+x} ] && [ $LOCALES_CHANGED -ne 0 ]; then |
1097 | 166 |
# still need to extract the locale information from the archive |
167 |
echo "extract locale changesets" |
|
168 |
tar -xf $SOURCE_TARBALL $LOCALE_FILE |
|
169 |
fi |
|
1056
90e1f32cf034
several changes to make upstream tarballs a good neighbour for locale fetching and HG checkouts
Wolfgang Rosenauer <wr@rosenauer.org>
parents:
1053
diff
changeset
|
170 |
else |
1097 | 171 |
# We are working on a version that is not yet published on the mozilla mirror |
172 |
# so we have to actually check out the repo |
|
173 |
||
1056
90e1f32cf034
several changes to make upstream tarballs a good neighbour for locale fetching and HG checkouts
Wolfgang Rosenauer <wr@rosenauer.org>
parents:
1053
diff
changeset
|
174 |
# mozilla |
1097 | 175 |
if [ -d $PRODUCT-$VERSION ]; then |
176 |
pushd $PRODUCT-$VERSION || exit 1 |
|
1056
90e1f32cf034
several changes to make upstream tarballs a good neighbour for locale fetching and HG checkouts
Wolfgang Rosenauer <wr@rosenauer.org>
parents:
1053
diff
changeset
|
177 |
_repourl=$(hg paths) |
90e1f32cf034
several changes to make upstream tarballs a good neighbour for locale fetching and HG checkouts
Wolfgang Rosenauer <wr@rosenauer.org>
parents:
1053
diff
changeset
|
178 |
case "$_repourl" in |
90e1f32cf034
several changes to make upstream tarballs a good neighbour for locale fetching and HG checkouts
Wolfgang Rosenauer <wr@rosenauer.org>
parents:
1053
diff
changeset
|
179 |
*$BRANCH*) |
90e1f32cf034
several changes to make upstream tarballs a good neighbour for locale fetching and HG checkouts
Wolfgang Rosenauer <wr@rosenauer.org>
parents:
1053
diff
changeset
|
180 |
echo "updating previous tree" |
90e1f32cf034
several changes to make upstream tarballs a good neighbour for locale fetching and HG checkouts
Wolfgang Rosenauer <wr@rosenauer.org>
parents:
1053
diff
changeset
|
181 |
hg pull |
1097 | 182 |
popd || exit 1 |
1056
90e1f32cf034
several changes to make upstream tarballs a good neighbour for locale fetching and HG checkouts
Wolfgang Rosenauer <wr@rosenauer.org>
parents:
1053
diff
changeset
|
183 |
;; |
90e1f32cf034
several changes to make upstream tarballs a good neighbour for locale fetching and HG checkouts
Wolfgang Rosenauer <wr@rosenauer.org>
parents:
1053
diff
changeset
|
184 |
* ) |
90e1f32cf034
several changes to make upstream tarballs a good neighbour for locale fetching and HG checkouts
Wolfgang Rosenauer <wr@rosenauer.org>
parents:
1053
diff
changeset
|
185 |
echo "removing obsolete tree" |
1097 | 186 |
popd || exit 1 |
187 |
rm -rf $PRODUCT-$VERSION |
|
1056
90e1f32cf034
several changes to make upstream tarballs a good neighbour for locale fetching and HG checkouts
Wolfgang Rosenauer <wr@rosenauer.org>
parents:
1053
diff
changeset
|
188 |
;; |
90e1f32cf034
several changes to make upstream tarballs a good neighbour for locale fetching and HG checkouts
Wolfgang Rosenauer <wr@rosenauer.org>
parents:
1053
diff
changeset
|
189 |
esac |
90e1f32cf034
several changes to make upstream tarballs a good neighbour for locale fetching and HG checkouts
Wolfgang Rosenauer <wr@rosenauer.org>
parents:
1053
diff
changeset
|
190 |
fi |
1097 | 191 |
if [ ! -d $PRODUCT-$VERSION ]; then |
1056
90e1f32cf034
several changes to make upstream tarballs a good neighbour for locale fetching and HG checkouts
Wolfgang Rosenauer <wr@rosenauer.org>
parents:
1053
diff
changeset
|
192 |
echo "cloning new $BRANCH..." |
1097 | 193 |
hg clone http://hg.mozilla.org/$BRANCH $PRODUCT-$VERSION |
194 |
if [ "$PRODUCT" = "thunderbird" ]; then |
|
1111 | 195 |
hg clone http://hg.mozilla.org/releases/comm-$CHANNEL $PRODUCT-$VERSION/comm |
1097 | 196 |
fi |
1056
90e1f32cf034
several changes to make upstream tarballs a good neighbour for locale fetching and HG checkouts
Wolfgang Rosenauer <wr@rosenauer.org>
parents:
1053
diff
changeset
|
197 |
fi |
1097 | 198 |
pushd $PRODUCT-$VERSION || exit 1 |
1111 | 199 |
|
200 |
# parse out the Firefox-release tag for this Thunderbird-checkout |
|
201 |
if [ "$PRODUCT" = "thunderbird" ]; then |
|
202 |
FF_RELEASE_TAG=$(grep ^GECKO_HEAD_REV ./comm/.gecko_rev.yml | awk -F ' ' '{print $2}') || exit 1 |
|
203 |
echo "Parsed Firefox base ID from .gecko_rev.yml: $FF_RELEASE_TAG" |
|
204 |
else |
|
205 |
FF_RELEASE_TAG="$RELEASE_TAG" |
|
206 |
fi |
|
207 |
||
1097 | 208 |
hg update --check $FF_RELEASE_TAG |
209 |
[ "$FF_RELEASE_TAG" == "default" ] || hg update -r $FF_RELEASE_TAG |
|
1056
90e1f32cf034
several changes to make upstream tarballs a good neighbour for locale fetching and HG checkouts
Wolfgang Rosenauer <wr@rosenauer.org>
parents:
1053
diff
changeset
|
210 |
# get repo and source stamp |
90e1f32cf034
several changes to make upstream tarballs a good neighbour for locale fetching and HG checkouts
Wolfgang Rosenauer <wr@rosenauer.org>
parents:
1053
diff
changeset
|
211 |
echo -n "REV=" > ../source-stamp.txt |
90e1f32cf034
several changes to make upstream tarballs a good neighbour for locale fetching and HG checkouts
Wolfgang Rosenauer <wr@rosenauer.org>
parents:
1053
diff
changeset
|
212 |
hg -R . parent --template="{node|short}\n" >> ../source-stamp.txt |
90e1f32cf034
several changes to make upstream tarballs a good neighbour for locale fetching and HG checkouts
Wolfgang Rosenauer <wr@rosenauer.org>
parents:
1053
diff
changeset
|
213 |
echo -n "REPO=" >> ../source-stamp.txt |
90e1f32cf034
several changes to make upstream tarballs a good neighbour for locale fetching and HG checkouts
Wolfgang Rosenauer <wr@rosenauer.org>
parents:
1053
diff
changeset
|
214 |
hg showconfig paths.default 2>/dev/null | head -n1 | sed -e "s/^ssh:/http:/" >> ../source-stamp.txt |
1097 | 215 |
|
216 |
if [ "$PRODUCT" = "thunderbird" ]; then |
|
217 |
pushd comm || exit 1 |
|
1111 | 218 |
hg update --check $RELEASE_TAG |
1097 | 219 |
popd || exit 1 |
220 |
rm -rf thunderbird-${VERSION}/{,comm/}other-licenses/7zstub |
|
221 |
fi |
|
222 |
popd || exit 1 |
|
1056
90e1f32cf034
several changes to make upstream tarballs a good neighbour for locale fetching and HG checkouts
Wolfgang Rosenauer <wr@rosenauer.org>
parents:
1053
diff
changeset
|
223 |
|
90e1f32cf034
several changes to make upstream tarballs a good neighbour for locale fetching and HG checkouts
Wolfgang Rosenauer <wr@rosenauer.org>
parents:
1053
diff
changeset
|
224 |
echo "creating archive..." |
1097 | 225 |
tar $compression -cf $PRODUCT-$VERSION$VERSION_SUFFIX.source.tar.xz --exclude=.hgtags --exclude=.hgignore --exclude=.hg --exclude=CVS $PRODUCT-$VERSION |
226 |
fi |
|
227 |
||
228 |
if [ ! -z ${SKIP_LOCALES+x} ]; then |
|
229 |
echo "Skipping locales-creation." |
|
230 |
exit 0 |
|
1056
90e1f32cf034
several changes to make upstream tarballs a good neighbour for locale fetching and HG checkouts
Wolfgang Rosenauer <wr@rosenauer.org>
parents:
1053
diff
changeset
|
231 |
fi |
262
b680975515e0
probably last working Firefox-on-xulrunner build
Wolfgang Rosenauer <wr@rosenauer.org>
parents:
diff
changeset
|
232 |
|
1097 | 233 |
if [ $LOCALES_CHANGED -ne 0 ]; then |
234 |
# l10n |
|
235 |
echo "fetching locales..." |
|
236 |
test ! -d l10n && mkdir l10n |
|
237 |
jq -r 'to_entries[]| "\(.key) \(.value|.revision)"' $LOCALE_FILE | \ |
|
238 |
while read locale changeset ; do |
|
239 |
case $locale in |
|
240 |
ja-JP-mac|en-US) |
|
241 |
;; |
|
242 |
*) |
|
243 |
echo "reading changeset information for $locale" |
|
244 |
echo "fetching $locale changeset $changeset ..." |
|
245 |
if [ -d "l10n/$locale/.hg" ]; then |
|
246 |
pushd "l10n/$locale" || exit 1 |
|
247 |
hg pull |
|
248 |
popd || exit 1 |
|
249 |
else |
|
250 |
hg clone "http://hg.mozilla.org/l10n-central/$locale" "l10n/$locale" |
|
251 |
fi |
|
1111 | 252 |
[ "$RELEASE_TAG" == "default" ] || hg -R "l10n/$locale" up -C -r "$changeset" |
1097 | 253 |
;; |
254 |
esac |
|
255 |
done |
|
256 |
echo "creating l10n archive..." |
|
1098 | 257 |
if [ "$PRODUCT" = "thunderbird" ]; then |
1097 | 258 |
TB_TAR_FLAGS="--exclude=browser --exclude=suite" |
1098 | 259 |
fi |
1097 | 260 |
tar $compression -cf l10n-$VERSION$VERSION_SUFFIX.tar.xz \ |
261 |
--exclude=.hgtags --exclude=.hgignore --exclude=.hg \ |
|
262 |
$TB_TAR_FLAGS \ |
|
263 |
l10n |
|
1098 | 264 |
elif [ -f "l10n-$PREV_VERSION$PREV_VERSION_SUFFIX.tar.xz" ]; then |
265 |
# Locales did not change, but the old tar-ball is in this directory |
|
266 |
# Simply rename it: |
|
267 |
echo "Moving l10n-$PREV_VERSION$PREV_VERSION_SUFFIX.tar.xz to l10n-$VERSION$VERSION_SUFFIX.tar.xz" |
|
268 |
mv "l10n-$PREV_VERSION$PREV_VERSION_SUFFIX.tar.xz" "l10n-$VERSION$VERSION_SUFFIX.tar.xz" |
|
1097 | 269 |
fi |
262
b680975515e0
probably last working Firefox-on-xulrunner build
Wolfgang Rosenauer <wr@rosenauer.org>
parents:
diff
changeset
|
270 |
|
b680975515e0
probably last working Firefox-on-xulrunner build
Wolfgang Rosenauer <wr@rosenauer.org>
parents:
diff
changeset
|
271 |
# compare-locales |
409 | 272 |
echo "creating compare-locales" |
1097 | 273 |
if [ -d compare-locales/.hg ]; then |
274 |
pushd compare-locales || exit 1 |
|
275 |
hg pull |
|
276 |
popd || exit 1 |
|
277 |
else |
|
278 |
hg clone http://hg.mozilla.org/build/compare-locales |
|
279 |
fi |
|
990
0d76004e9fa4
support parallel compression if available
Wolfgang Rosenauer <wr@rosenauer.org>
parents:
986
diff
changeset
|
280 |
tar $compression -cf compare-locales.tar.xz --exclude=.hgtags --exclude=.hgignore --exclude=.hg compare-locales |
262
b680975515e0
probably last working Firefox-on-xulrunner build
Wolfgang Rosenauer <wr@rosenauer.org>
parents:
diff
changeset
|
281 |