1 #!/usr/bin/python

     2 

     3 import urllib2

     4 import sgmllib

     5 import string

     6 import pickle

     7 import os

     8 

     9 class MyParser(sgmllib.SGMLParser):

    10   "A simple parser class."

    11 

    12   def parse(self, s):

    13     "Parse the given string 's'."

    14     self.feed(s)

    15     self.close()

    16 

    17   def __init__(self, verbose=0):

    18     "Initialise an object, passing 'verbose' to the superclass."

    19     sgmllib.SGMLParser.__init__(self, verbose)

    20     self.packages = {}

    21 

    22   def start_a(self, attributes):

    23     "Process a hyperlink and its 'attributes' and only add buildsymbols rpms to the list."

    24     for name, value in attributes:

    25       if name == "href":

    26 	fn = value.rsplit('.', 1)

    27 	if len(fn) > 1:

    28           if fn[1] == "rpm":

    29 	    if fn[0].find("buildsymbols") != -1:

    30 	      tmpstr = fn[0].rsplit(".", 1)[0]

    31 	      tmpstr = tmpstr.rsplit("-", 2)

    32 	      name = tmpstr[0].rsplit("-", 1)[0]

    33 	      version = tmpstr[1] + "-" + tmpstr[2]

    34 	      self.packages[name] = version

    35 

    36   def get_buildsymbols(self):

    37     "Return the list of packages."

    38     return self.packages

    39 

    40 

    41 def submit_package(baseuri, filename):

    42   uri = baseuri + filename

    43 

    44   # download

    45   print "fetching " + uri

    46   try:

    47     f = urllib2.urlopen(uri)

    48     lf = open(filename, "wb")

    49     lf.write(f.read())

    50     lf.close()

    51   except HTTPError, e:

    52     print "HTTP Error:",e.code , uri

    53   except URLError, e:

    54     print "URL Error:",e.reason , uri

    55 

    56   # extract

    57   command = "rpm2cpio " + filename + " | cpio -idm --quiet"

    58   print "extracting symbols from package"

    59   os.system(command)

    60   # save filename we just extracted (assume there is only one file)

    61   for root, dirs, files in os.walk("usr"):

    62     for file in files:

    63       zipfile = file

    64 

    65   # - upload to symbolstore

    66   print "uploading " + zipfile + " to mozilla"

    67   command = "scp ./usr/share/mozilla/" + zipfile + " " + SSHCONF + ":"

    68   os.system(command)

    69 

    70   # - unzip data on symbolstore

    71   print "unzipping " + zipfile + " on symbolstore"

    72   command = "ssh mozsymbols 'cd symbols; unzip -o /home/wr/" + zipfile + "'"

    73   os.system(command)

    74 

    75   # clean up working directory

    76   command = "rm -rf usr *.rpm"

    77   os.system(command)

    78 

    79 

    80 def find_packages(repo, arch):

    81   updates = []

    82   f = urllib2.urlopen(repo)

    83   s = f.read()

    84 

    85   parser = MyParser()

    86   parser.parse(s)

    87 

    88   datanew = parser.get_buildsymbols()

    89 

    90   fn = repo + ".pkl"

    91   fn = fn.replace("/", "_")

    92 

    93   dataold = None

    94   if os.path.isfile(fn):

    95     datafile = open(fn, 'rb')

    96     dataold = pickle.load(datafile)

    97     datafile.close()

    98     

    99   for k, v in datanew.iteritems():

   100     if dataold is not None:

   101       if dataold[k] == v:

   102         continue

   103 

   104     # different version, save in updates list

   105     updates.append(k)

   106 

   107   # save new dataset for future use (only if anything changed at all)

   108   if len(updates) > 0:

   109     print "updating " + fn

   110     datafile = open(fn, 'wb')

   111     pickle.dump(datanew, datafile)

   112     datafile.close()

   113   else:

   114     print "nothing to do for " + repo

   115 

   116   for package in updates:

   117     baseuri = repo + "/"

   118     filename =  package + "-buildsymbols-" + datanew[package] + "." + arch + ".rpm"

   119     submit_package(baseuri, filename)

   120 

   121 

   122 # MAIN

   123 

   124 baseurl = "http://download.opensuse.org/"

   125 archs = [ "i586", "x86_64"]

   126 repos = [ "distribution/11.3/repo/oss/suse/", "update/11.3/rpm/" ]

   127 bsrepos = [ "repositories/mozilla/", "repositories/mozilla:/beta/" ]

   128 bsdists = [ "openSUSE_11.3", "openSUSE_11.2", "openSUSE_11.1" ]

   129 SSHCONF = "mozsymbols"

   130 

   131 # product repos

   132 for repo in repos:

   133   for arch in archs:

   134     uri = baseurl + repo + arch

   135     find_packages(uri, arch)

   136 

   137 # OBS repos

   138 for repo in bsrepos:

   139   for dist in bsdists:

   140     for arch in archs:

   141       uri = baseurl + repo + dist + "/" + arch

   142       find_packages(uri, arch)

   143 

   144