#!/usr/bin/python
import urllib2
import sgmllib
import string
import pickle
import os
class MyParser(sgmllib.SGMLParser):
"A simple parser class."
def parse(self, s):
"Parse the given string 's'."
self.feed(s)
self.close()
def __init__(self, verbose=0):
"Initialise an object, passing 'verbose' to the superclass."
sgmllib.SGMLParser.__init__(self, verbose)
self.packages = {}
def start_a(self, attributes):
"Process a hyperlink and its 'attributes' and only add buildsymbols rpms to the list."
for name, value in attributes:
if name == "href":
fn = value.rsplit('.', 1)
if len(fn) > 1:
if fn[1] == "rpm":
if fn[0].find("buildsymbols") != -1:
tmpstr = fn[0].rsplit(".", 1)[0]
tmpstr = tmpstr.rsplit("-", 2)
name = tmpstr[0].rsplit("-", 1)[0]
version = tmpstr[1] + "-" + tmpstr[2]
if version.find("_") == -1:
self.packages[name] = version
def get_buildsymbols(self):
"Return the list of packages."
return self.packages
def submit_package(baseuri, filename):
uri = baseuri + filename
# download
print "fetching " + uri
try:
f = urllib2.urlopen(uri)
lf = open(filename, "wb")
lf.write(f.read())
lf.close()
except HTTPError, e:
print "HTTP Error:",e.code , uri
except URLError, e:
print "URL Error:",e.reason , uri
# extract
command = "rpm2cpio " + filename + " | cpio -idm --quiet"
print "extracting symbols from package"
os.system(command)
# save filename we just extracted (assume there is only one file)
for root, dirs, files in os.walk("usr"):
for file in files:
zipfile = file
# - upload to symbolstore
print "uploading " + zipfile + " to mozilla"
command = "scp ./usr/share/mozilla/" + zipfile + " " + SSHCONF + ":"
os.system(command)
# - unzip data on symbolstore
print "unzipping " + zipfile + " on symbolstore"
command = "ssh mozsymbols 'cd symbols; unzip -o /home/wr/" + zipfile + "'"
os.system(command)
# post-upload command
print "executing post-upload command on server"
command = "ssh mozsymbols '/usr/local/bin/post-symbol-upload.py'"
# clean up working directory
command = "rm -rf usr *.rpm"
os.system(command)
def find_packages(repo, arch):
updates = []
f = urllib2.urlopen(repo)
s = f.read()
parser = MyParser()
parser.parse(s)
datanew = parser.get_buildsymbols()
fn = repo + ".pkl"
fn = fn.replace("/", "_")
dataold = None
if os.path.isfile(fn):
datafile = open(fn, 'rb')
dataold = pickle.load(datafile)
datafile.close()
for k, v in datanew.iteritems():
if dataold is not None:
if dataold.has_key(k) and dataold[k] == v:
continue
# different version, save in updates list
updates.append(k)
# save new dataset for future use (only if anything changed at all)
if len(updates) > 0:
print "updating " + fn
datafile = open(fn, 'wb')
pickle.dump(datanew, datafile)
datafile.close()
else:
print "nothing to do for " + repo
for package in updates:
baseuri = repo + "/"
filename = package + "-buildsymbols-" + datanew[package] + "." + arch + ".rpm"
submit_package(baseuri, filename)
# MAIN
baseurl = "http://download.opensuse.org/"
archs = [ "i586", "x86_64"]
repos = [ "distribution/11.3/repo/oss/suse/", "update/11.3/rpm/", "distribution/11.4/repo/oss/suse/", "update/11.4/rpm/" ]
bsrepos = [ "repositories/mozilla/", "repositories/mozilla:/beta/" ]
bsdists = [ "openSUSE_11.4", "openSUSE_11.3", "openSUSE_11.2" ]
SSHCONF = "mozsymbols"
# product repos
for repo in repos:
for arch in archs:
uri = baseurl + repo + arch
find_packages(uri, arch)
# OBS repos
for repo in bsrepos:
for dist in bsdists:
for arch in archs:
uri = baseurl + repo + dist + "/" + arch
find_packages(uri, arch)