#!/usr/bin/python

###########################################################
# GNOME dep updater 2005.09.09
# John N. Laliberte <allanonjl@gentoo.org>
#
# This script checks the GNOME release directories specified,
# then checks for the latest release version of the package,
# and then compares both of those versions against the versions
# in portage.  It also generates statistics.
#
# Then it generates an html page which has 3 colors:
# red for not up to date, green for up to date and the latest version,
# and light green for up to date with the current release version only.
#
# In order to use this, you need to have:
# 1.  eix
#
# Easiest way to use this is to use CVS tree as an overlay, and just
# cvs-update && update-eix, and then 
# python gnomedep.py ftp.gnome.org -t -x win32
#
# The sequence "-x win32" just ignores the directory win32.
#
# WARNING - WARNING - WARNING
# This script was created in an ad-hoc fashion, with no forethought
# to design.
# Maybe someday I'll have time to fix and make this thing proper
# and more general.
#
# Some things I'd like to add
# 1. command line options for the directories ( easy )
# 2. links to the files themselves ( easy - med )
# 3. use css instead of straight html ( easy )
# 4. code CLEANUP ( time + design )
#    more specifically:
#   4a. break up objects into seperate modules
#   4b. put the html generation into a sep object / module
#   4c. use only 1 object to store the package information
#   4d. connect to the ftp only once instead of twice
#   4e. make everything non-GNOME specific and more general
#       and accessible via command arguments.
#
# parts of the ftp stuff was taken from:
# http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/436211
############################################################

import fnmatch, ftplib, optparse, os, stat, string, sys, time, re

class FtpWalker:
    def __init__( self, site, user, passwd ):
        self.ftp = ftplib.FTP( site, user, passwd )
        print "logged in"
    def cd( self, path ):
        try:
            self.ftp.cwd( path )
            print "changed to: "+self.pwd()
        except:
            return False
        else:
            return True
    def pwd( self ):
        return self.ftp.pwd()
    def get( self, fileinfo, binary=True, callback=None ):
        status = alreadydownloaded( fileinfo )
        if status == DOWNLOAD_NONE: return
        if not callback:
            localfile = createfile( fileinfo, binary, status == DOWNLOAD_PARTIAL )
            callback  = localfile.write
        try:
            filename = fileinfo.longname
            getstr = "RETR %s" % fileinfo.name
            if binary:
                if status == DOWNLOAD_PARTIAL:
                    self.ftp.retrbinary( getstr, callback, rest=os.path.getsize( filename ) )
                else:
                    self.ftp.retrbinary( getstr, callback )
            else:
                self.ftp.retrlines( getstr, callback )
        finally: 
            if localfile:
                localfile.close()
                os.utime( filename, (fileinfo.date, fileinfo.date) )
                os.chmod( filename, fileinfo.mode )
    def ls( self, cwd ):
        lines = []
        self.ftp.retrlines( "LIST", lines.append )
        return map( lambda x: extract_info( cwd, x ), lines ) 

DOWNLOAD_FULL, DOWNLOAD_PARTIAL, DOWNLOAD_NONE = 0, 1, 2 

def alreadydownloaded( fileinfo ):
    f = fileinfo.longname
    if os.path.isfile( f ):
        ldate, rdate = os.path.getmtime( f ), fileinfo.date
        lsize, rsize = os.path.getsize( f ),  fileinfo.size
        if round( ldate ) == round( rdate ):
            if lsize == rsize:
                return DOWNLOAD_NONE    # already downloaded
            else:
                return DOWNLOAD_PARTIAL # partially downloaded
        else:
            newfilename = mknewversion( fileinfo.path, fileinfo.name )
            os.rename( fileinfo.longname, newfilename )
            return DOWNLOAD_FULL        # old version, rename
    else:
        return DOWNLOAD_FULL            # no file, download

def mknewversion( path, filename ):
    version = 1
    def mkversion( version ):
        return os.path.join( path, ".%s.%03d" % (filename, version) )
    longname = mkversion( version )
    while os.path.exists( longname ):
        version += 1
        longname = mkversion( version )
    return longname
 
def iff( test_, then_, else_ ): # then_, else_ always get evaled so pls be atoms
    if test_:
        return then_
    else:
        return else_

def createfile( fileinfo, binary, append ):
    fname = fileinfo.longname
    if not os.path.isdir( fileinfo.path ):
        os.makedirs( fileinfo.path )
    permissions = iff( binary, 'wb', 'w' )
    if append and os.path.isfile( fname ):
        permissions += 'a'
        perm = os.stat( fname )[stat.ST_MODE]
        if not perm & stat.S_IWUSR:
            os.chmod( fname, perm | stat.S_IWUSR )
    return file( fname, permissions )

curr_year_fmt, prev_year_fmt, unified_fmt = '%b %d %H:%M', '%b %d  %Y', '%Y-%m-%d-%H:%M'

def updatetuple( t, i, x ): # insert x into the ith field of tuple, t
    l = list( t )
    return tuple( l[:i] + [x] + l[i+1:] )

def parsePrevYear( date ): return time.strptime( date, prev_year_fmt )
def parseCurrYear( date ):
    datewith1900 = time.strptime( date, curr_year_fmt )
    currentYear  = time.gmtime()[0]
    return updatetuple( datewith1900, 0, currentYear )

def dateParser( date ): return iff( ':' in date, parseCurrYear, parsePrevYear )
def parseDate( date ):  return time.mktime( dateParser( date )( date ) )

def displayDate( date ):
    date_struct, curr_struct = time.gmtime( date ), time.gmtime()
    date_year, curr_year = date_struct[0], curr_struct[0]
    year_fmt = iff( date_year == curr_year, curr_year_fmt, prev_year_fmt )
    return time.strftime( year_fmt, date_struct )

R_MSK, W_MSK, X_MSK, Z_MSK =   4,   2,   1,   0
R_STR, W_STR, X_STR, Z_STR = 'r', 'w', 'x', '-'

def str2mode( str ):
    r, w, x = str[0] == R_STR,  str[1] == W_STR,  str[2] == X_STR
    return iff( r, R_MSK, Z_MSK ) | iff( w, W_MSK, Z_MSK ) | iff( x, X_MSK, Z_MSK )

def mode2str( mode ):
    r, w, x = mode & R_MSK, mode & W_MSK, mode & X_MSK
    return iff( r, R_STR, Z_STR ) + iff( w, W_STR, Z_STR ) + iff( x, X_STR, Z_STR )

def str2fullmode( str ):
    u, g, o = str[0:3], str[3:6], str[6:9]
    return str2mode( u ) << 6 | str2mode( g ) << 3 | str2mode( o )

def fullmode2str( mode ):
    u, g, o = mode >> 6 & 0x7, mode >> 3 & 0x7, mode & 0x7
    return mode2str( u ) + mode2str( g ) + mode2str( o )

def str2perm( str ):
    return str[0] == 'd', str[0] == 'l', str2fullmode( str[1:] )

def perm2str( isdir, islink, mode ):
    return iff( isdir, 'd', iff( islink, 'l', '-' ) ) + fullmode2str( mode )

def extract_info( cwd, line ):
    fullmode, links, owner, group, size, rest = line.split( None, 5 )
    isdir, islink, mode = str2perm( fullmode )
    dateStr, name = rest[:12], rest[13:]
    if 0 < string.find(name,"->"):
        name, symbolic = name.split("->")
        name = name.strip()
        symbolic = symbolic.strip()
    print name
    date = parseDate( dateStr )
    return FileInfo( cwd, name, fullmode, isdir, islink, mode, int( links ), owner, group, int( size ), dateStr, date, line)

class FileInfo:
    def __init__( self, path, name, modeStr, isdir, islink, mode, links, owner, group, size, dateStr, date, line ):
        self.path, self.name, self.isdir, self.islink = path, name, isdir, islink
        self.modeStr, self.mode, self.owner, self.group = modeStr, mode, owner, group
        self.links, self.size, self.dateStr, self.date = links, size, dateStr, date
        self.longname, self.age, self.line = os.path.join( path, name ), now - self.date, line

def dropslashes( str ): 
    i, n = 0, len( str )
    while i < n and str[i] == '/': i += 1
    return str[i:]

def excluded( exclude_patterns, dir ):
    for exclude_pattern in exclude_patterns:
        if pattern( exclude_pattern, dir ):
            return True
    return False

def listSiteGen( walker, dir, opts ):
    path = walker.pwd()
    if not excluded( opts.exclude, dir ) and walker.cd( dir ):
        for info in walker.ls( dropslashes( os.path.join( path, dir ) ) ):
            if info.isdir:
                for rec_info in listSiteGen( walker, info.name, opts ):
                    yield rec_info
            else:
                yield info
    walker.cd( path )

def ftpfind( walker, dir, opts, packages ):
    for fileinfo in listSiteGen( walker, dir, opts ):
        if opts.expr( fileinfo ):
            print "%s" % opts.printer( fileinfo )
            packages.append(fileinfo.name)
            
            if not opts.test:
                walker.get( fileinfo )

def date( d, f=None ):
    if f:
        return time.mktime( time.strptime( d, f ) )
    else:
        return parseDate( d )

def pattern( p, v ): return fnmatch.fnmatch( v, p )
kilobyte = 1024; megabyte = kilobyte * kilobyte; gigabyte = kilobyte * megabyte; terabyte = kilobyte * gigabyte
second = 1; minute = 60*second; hour = 60*minute; day = 24*hour; week = 7*day; year = 52*week
 
def expr_cb( option, opt_str, value, parser ): parser.values.expr = eval( "lambda file: " + value )
def print_cb( option, opt_str, value, parser ): parser.values.printer = eval( "lambda file: " + value )

now = time.mktime( time.gmtime() ) # used by age filter

def daystart_cb( option, opt_str, value, parser ):
    global now
    x = time.gmtime()
    start_of_day = x[0], x[1], x[2], 0, 0, 0, x[6], x[7], x[8]
    now = time.mktime( start_of_day )

def_printer=lambda file: file.line
def_expr=lambda file: True

def parse_command_line():
    parser = optparse.OptionParser()
    parser.set_defaults( user="anonymous", password="ftpbot@test.com", expr=def_expr, test=False, exclude=[], printer=def_printer  )
    parser.add_option( "-e", "--expr",     action="callback", callback=expr_cb, type="string", help="use the python expression, lambda file: <EXPR>, as a filter (must return boolean)", metavar="EXPR" )
    parser.add_option( "-p", "--password", help="specify the password to use", metavar="PASSWD" )
    parser.add_option( "--print", action="callback", callback=print_cb, type="string", help="use the printer, lambda file: <EXPR>, to print file summary (must return string)", metavar="EXPR" )
    parser.add_option( "-s", "--daystart", action="callback", callback=daystart_cb, help="calculate ages from today @ 00:00" )
    parser.add_option( "-t", "--test",     action ="store_true", help="print filename but do not perform file transfer" )
    parser.add_option( "-u", "--user",     help="specify the username to use", metavar="USER" )
    parser.add_option( "-x", "--exclude",  action="append", help="do not traverse this directory", metavar="DIR" )
    parser.add_option( "--nf", "--nofetch", action="store_true", help="do not connect to gnome.org, read from package.txt", metavar="NOFETCH" )
    return parser.parse_args()

class Package:
    def __init__(self, raw_data):
        # names of packages in portage that are different than the
        # names on the gnome ftp.
        self.special = {"orbit2":"orbit", 
                        "libidl":"libIDL",
                        "libxml++":"libxmlpp",
                        "gnome2-vfs":"gnome-vfs",
                        "gtk2":"gtk2-perl",
                        "gnome2-vfs":"gnome2-vfs-perl",
                        "gtkhtml":"libgtkhtml",
                        "pkg-config":"pkgconfig"
                        }
        
        self.name = None
        self.raw_name = None
        self.package_version = None
        self.revision = None
        self.name_plus_version = None
        self.name_plus_version_plus_revision = None
        self.raw_string = raw_data
        self.latest_version = None
        
        self.parse_raw_string(raw_data)
        
        self.special_cases()
        
        #self.get_latest_version()
        
    def special_cases(self):
        for key in self.special.keys():
            if self.name == key:
                self.name = self.special[key]
    
    def parse_raw_string(self, rs):
        # add a special SPECIAL case for cairo, TAKE THIS OUT LATER WHEN GNOME FTP fixes it.
        if -1 != string.find(rs,"cairo-0.5.2-head"):
            rs = string.replace(rs,"-head","")
            
        # test for the case where we're passing in portage style strings.
        # taken shamelessly from repoman
        ven_cat = r'[\w0-9-]+'                                 # Category
        ven_nam = r'([+a-z0-9-]+(?:[+_a-z0-9-]*[+a-z0-9-]+)*)' # Name
        ven_ver = r'((?:\d+\.)*\d+[a-z]?)'                     # Version
        ven_suf = r'(_(alpha\d*|beta\d*|pre\d*|rc\d*|p\d+))?'  # Suffix
        ven_rev = r'(-r\d+)?'                                  # Revision

        #ven_string=ven_cat+'/'+ven_nam+'-'+ven_ver+ven_suf+ven_rev
        ven_string=ven_nam+'-'+ven_ver+ven_suf+ven_rev
                # get rid of suffixes
        if 0 < rs.find(".tar."):
            # remove any non bz2 files
            if 0 < rs.find(".bz2"):
                # get rid of the .tar.bz2 so we can look it up in portage
                self.name_plus_version = string.replace(rs,".tar.bz2","")
            if 0 < rs.find(".gz"):
                self.name_plus_version = string.replace(rs,".tar.gz","")
        else:
            valid_ebuild_name_re=re.compile(ven_string+'$', re.I)

            if valid_ebuild_name_re.match(rs):
                # do .search(), and then m.groups() returns a tuple
                revisk = re.compile(ven_rev)
                # modified or it'll do blah-3.4.5 to blah-3 ( WRONG! )
                #ven_nam_modified = r'([+a-z0-9-]{0,1}(?:[+_a-z0-9]*[+a-z0-9]+)*)'
                #ven_nam_modified = r'([+a-z0-9-]{0,2}(?:[+_a-z0-9]*[+a-z0-9]+)*)'
                ven_nam_modified = r'([+a-zA-Z0-9-]+(?:[+_a-z0-9-]*[+a-z0-9-]+)*)'
                namere = re.compile(ven_nam_modified)
                vers = re.compile(ven_ver)
                
                # FIXME: revision doesn't work for some reason
                self.revision = revisk.search(rs).groups()[0]
                temp = namere.search(rs).groups()[0]
                self.name = temp[:len(temp)-2] # get rid of the last 2 chars in gnome-mime-data-2 etc.
                # first remove the packagename, this is so that libxml2 does it correctly,
                # and its better anyways.
                temp = string.replace(rs, self.name, "")
                self.package_version = vers.search(str(temp)).groups()[0]
                
                self.name_plus_version = self.name + '-' + self.package_version
                                
        # find the name of the package
        try:
            divider_is_at = string.rfind(self.name_plus_version,"-")
            self.name = string.lower(self.name_plus_version[:divider_is_at])
            self.raw_name = self.name_plus_version[:divider_is_at]
        
            # find the version
            self.package_version = string.lower(self.name_plus_version[(divider_is_at+1):])
        except:
            # this is just a regular file, not a package
            pass
        
    def serialize_to_html(self):
        # do all serialization here
        # return a string
        pass
    
    def print_info(self):
        print "Name: " + str(self.name)
        print "Version: " + str(self.package_version)
        print "Name+Version: " + str(self.name_plus_version)
        print "Raw: " + str(self.raw_string)
        
    def get_latest_version(self):
    
        if self.latest_version == None and self.name != None:
            print "Finding the latest version of the package..."
            
            latest_vers = ""
            # first need to check ftp.gnome.org/pub/GNOME/sources/gnome-terminal/
            site = "ftp.gnome.org"
            ftp_path = "pub/GNOME/sources" + "/" + self.name
            
            # connect and do an "ls" on the directory to retrieve all the version numbers
            walker = FtpWalker(site, "anonymous", "ftpbot@test.com")
            path = walker.pwd()
            #walker.ls( dropslashes( os.path.join( path, dir ) ) )
            walker.cd(ftp_path)
            result = walker.ls(ftp_path)
            #result = walker.ls( dropslashes( os.path.join( path, dir ) ) )
            print "results" + str(result)
            
            # then determine the highest numbering package in the directory
            
            # then you know the full link to the latest package, such as
            # ftp.gnome.org/pub/GNOME/sources/gnome-terminal/2.10/gnome-terminal-2.10.tar.bz2
            self.latest_version = latest_vers
                
        return self.latest_version
        
class PackageUpdate:
    def __init__(self, name, portage_version, gnome_version, latest_version, status):
        self.name = name
        self.portage_version = portage_version
        self.gnome_version = gnome_version
        self.latest_version = latest_version
        self.status = status
        
class Status:
    def Compliant(self):
        return 0
    def NeedUpdate(self):
        return 1
    def NotFound(self):
        return -1
    def NewerVersion(self):
        return 2
    
    property(Compliant)
    property(NeedUpdate)
    property(NotFound)
    property(NewerVersion)
    
class FileStuff:
    def __init__(self, filename):
        self.filename = filename
        self.lines = []
        
    def read(self):
        file = self.open("r")
        # read the file in line by line, and then return it
        for line in file.readlines():
            # replace the newline characters
            line = string.replace(line,'\n','')
            # add it to the collection
            self.lines.append(line)
        file.close()
        
        return self.lines
        
    def write(self, list_of_lines):
        try:
            # open file stream
            file = self.open("w")
        except IOError:
            print "There was an error writing to"+self.filename
            sys.exit()
    
        for line in list_of_lines:
            file.write(line+"\n")
        
        file.close()
    
    def append(self, list_of_lines):
        try:
            # open file stream
            file = self.open("a")
        except IOError:
            print "There was an error writing to"+self.filename
            sys.exit()
    
        for line in list_of_lines:
            file.write(line+"\n")
        
        file.close()
            
    def open(self, type):
        try:
            file = open(self.filename, type)
            return file
        except IOError:
            print "Error reading/writing " + type
            sys.exit()

### Stolen from portage.py
# This takes two version strings and returns an integer to tell you whether
# the versions are the same, val1>val2 or val2>val1.
vcmpcache={}
def vercmp(val1,val2):
        if val1==val2:
                #quick short-circuit
                return 0
        valkey=val1+" "+val2
        try:
                return vcmpcache[valkey]
                try:
                        return -vcmpcache[val2+" "+val1]
                except KeyError:
                        pass
        except KeyError:
                pass

        # consider 1_p2 vc 1.1
        # after expansion will become (1_p2,0) vc (1,1)
        # then 1_p2 is compared with 1 before 0 is compared with 1
        # to solve the bug we need to convert it to (1,0_p2)
        # by splitting _prepart part and adding it back _after_expansion
        val1_prepart = val2_prepart = ''
        if val1.count('_'):
                val1, val1_prepart = val1.split('_', 1)
        if val2.count('_'):
                val2, val2_prepart = val2.split('_', 1)

        # replace '-' by '.'
        # FIXME: Is it needed? can val1/2 contain '-'?
        val1=string.split(val1,'-')
        if len(val1)==2:
                val1[0]=val1[0]+"."+val1[1]
        val2=string.split(val2,'-')
        if len(val2)==2:
                val2[0]=val2[0]+"."+val2[1]

        val1=string.split(val1[0],'.')
        val2=string.split(val2[0],'.')

        #add back decimal point so that .03 does not become "3" !
        for x in range(1,len(val1)):
                if val1[x][0] == '0' :
                        val1[x]='.' + val1[x]
        for x in range(1,len(val2)):
                if val2[x][0] == '0' :
                        val2[x]='.' + val2[x]

        # extend version numbers
        if len(val2)<len(val1):
                val2.extend(["0"]*(len(val1)-len(val2)))
        elif len(val1)<len(val2):
                val1.extend(["0"]*(len(val2)-len(val1)))

        # add back _prepart tails
        if val1_prepart:
                val1[-1] += '_' + val1_prepart
        if val2_prepart:
                val2[-1] += '_' + val2_prepart
        #The above code will extend version numbers out so they
        #have the same number of digits.
        for x in range(0,len(val1)):
                cmp1=relparse(val1[x])
                cmp2=relparse(val2[x])
                for y in range(0,4):
                        myret=cmp1[y]-cmp2[y]
                        if myret != 0:
                                vcmpcache[valkey]=myret
                                return myret
        vcmpcache[valkey]=0
        return 0
### supporting function ripped from portage.py
def relparse(myver):
        "converts last version part into three components"
        number=0
        suffix=0
        endtype=0
        endnumber=0

        mynewver=string.split(myver,"_")
        myver=mynewver[0]
         #normal number or number with letter at end
        divider=len(myver)-1
        if myver[divider:] not in "1234567890":
                #letter at end
                suffix=ord(myver[divider:])
                number=string.atof(myver[0:divider])
        else:
                number=string.atof(myver)

        if len(mynewver)==2:
                #an endversion
                for x in endversion_keys:
                        elen=len(x)
                        if mynewver[1][:elen] == x:
                                match=1
                                endtype=endversion[x]
                                try:
                                        endnumber=string.atof(mynewver[1][elen:])
                                except SystemExit, e:
                                        raise
                                except:
                                        endnumber=0
                                break
        return [number,suffix,endtype,endnumber]

if __name__ == '__main__':
    # Example commandline arguments:
    # This will use the cached file: ftp.gnome.org -t --nofetch
    # This will fetch from the ftp: ftp.gnome.org -t -x win32
    opts, args = parse_command_line()
    site, dirs = args[0], args[1:]
    if len( dirs ) == 0: dirs = ['/']
    
    site="ftp.gnome.org" # default site
    dirs=["pub/GNOME/platform/2.12/2.12.0/", 
          "pub/GNOME/desktop/2.12/2.12.0/sources/",
          "pub/GNOME/bindings/2.12/2.12.0/sources/"
          ]
    packages=[]

    # if you don't want to connect to the FTP and get the updated packs,
    # but use the previously generated txt file.
    print str(opts.nf)
    if not opts.nf:
        walker = FtpWalker( site, "anonymous", "ftpbot@test.com" )

        for dir in dirs:
            ftpfind( walker, dir, opts, packages )
        
        file = FileStuff("packages.txt")
        file.write(packages)
    else:
        # packages.txt is the saved name-version of software on the GNOME FTP
        file = FileStuff("packages.txt")
    

        
    raw_gnome_packages = file.read() # list  

    # create Package objects for the GNOME packs.
    gnome_packages=[]
    for rawstr in raw_gnome_packages:
        g_pack = Package(rawstr)
        # only add it if its not already there
        # pack.name == None if its a file like NEWS, or something not with .tar.gz or .tar.bz2
        if (g_pack.name not in [pack.name for pack in gnome_packages if True]) and g_pack.name != None:
            gnome_packages.append(g_pack)
            g_pack.print_info()
    
    # while we are connected, need to also obtain the absolute current version
    # of the package. ( for example, the package for the "release" might not be
    # the latest version.
    #try:
    walker = FtpWalker( site, "anonymous", "ftpbot@test.com" )
    for pack in gnome_packages:
        print "Finding latest version for: " + pack.name
        if string.rfind(pack.name, "cairo") == -1:
            # retrieve the listing of the directory ->
            # ftp.gnome.org/pub/GNOME/sources/package-name/
            versions_p = []
            path = "/pub/GNOME/sources/" + pack.raw_name + "/"
            
            # if you were able to change directories
            # then do the rest.
            if walker.cd(path):
                result = walker.ls(path)
                # add each name to a list
                for res in result:
                    versions_p.append(res.name)
                    
                # now check to see how x.y on the FTP compares to x.y of "Release"
                # find the last "." , and strip it off, assuming x.y.z here....
                # this is probably NOT the best way to do this. ( only do this if theres 2 dots or more,
                # otherwise, its already in the correct format.
                reduced_version = pack.package_version
                num = string.count(pack.package_version, ".")
                if num == 2:
                    index_of_dot = string.rfind(reduced_version,".")
                    reduced_version = reduced_version[:index_of_dot]
                elif num > 2:
                    # we always want to keep 1 dot.
                    for i in range(num-1):
                        index_of_dot = string.rfind(reduced_version,".")
                        reduced_version = reduced_version[:index_of_dot]
                else:
                    reduced_version = pack.package_version
                
                # figure out if theres a higher version with x.y
                higher_dir = reduced_version
                for vers in versions_p:
                    try:
                        val = vercmp(pack.package_version, vers)
                        if 0 > val:
                            higher_dir = vers
                    except:
                        print "Error comparing for package " + str(pack.name)
                
                # if x.y of ftp = x.y of release, change to that directory,
                # and grab the "LATEST-IS-HERETOEND"
                if higher_dir in versions_p:
                    path = path + higher_dir
                    walker.cd(path)
                    
                    # check to make sure we actually changed directories
                    # or else get bogus entries!
                    print "working dir: " + walker.pwd() + "\nPath: " + path
                    # pwd comes back as /mirror/gnome.org, so substitute
                    # this is a quick HACK
                    #wpwd = walker.pwd()
                    #wpwd = string.replace(wpwd, "mirror","pub")
                    #wpwd = string.replace(wpwd, "gnome.org", "GNOME")
                    #if wpwd == path:
                    #    print "Successfully changed path!"
                        # we did successfully change the path.
                    result = walker.ls(path)
                    
                    # (This could be much more robust, its late and just want a prototype)
                    latest = "LATEST-IS-"
                    for item in result:
                        if 0 <= string.rfind(item.name,"LATEST-IS-"):
                            latest = item.name
                            
                    latest_reduced = latest[10:]
            else:
                print "Did not successfully change path ( does this exist on gnome ftp?"
                latest_reduced = "None"
            
            # compare the current version to all the version in the list
            # to see if theres a newer version.
            highest_number = pack.package_version
            try:
                result_cmp = vercmp(highest_number, latest_reduced)
                if 0 > result_cmp:
                    # its a newer version!
                    highest_number = latest_reduced
            except:
                print "Error comparing for package " + str(pack.name)
                    
            # store the highest number as pack.latest_version            
            pack.latest_version = highest_number
            
    # now go lookup the portage packs and store the objects
    portage_packages = []
    for package in gnome_packages:
        if package.name != None:
            # you need eix to do this, so we can get the LATEST stable version of things
            # eix -e package -n -l
            # go from line "Available versions" to
            # "Installed"
            command = "eix -e " + package.name + " -n"
            print "running command: " + command
            put, get = os.popen4(command)
            
            for line in get.readlines():
                line = string.strip(line)
                index = string.rfind(line,"Available versions:")

                if 0 <= index:
                    # its found!
                    print "Found a version in portage!"
                    versions = line[19:]
                    # strip [M]
                    versions = string.replace(versions,"[M]","")
                    versions = string.replace(versions,"[1]","") # if you have an overlay, remove this too
                    versions = string.split(versions) # split on spaces
                    
                    # give Package class in format it expects, name-version
                    # get rid of last element b/c its faulty! ( thats why its -2 )
                    #inter = versions[len(versions)-2]
                    inter = versions[len(versions)-1]
                    
                    # if the first char is ~, slice it off because thats not valid syntax for portage
                    if inter[0] == "~":
                        inter = inter[1:]
 
                    p_name = package.name + "-" + inter
                    portage_packages.append(Package(p_name))
    
    # now compare 2 of the packages
    # """if returnval is less than zero, then pkg2 is newer than pkg1, zero if equal and positive if older."""

    #print "Comparing: " + str(gnome_packages[4].package_version) + " to " + str(portage_packages[4].package_version)
    #print str(vercmp(gnome_packages[4].package_version, portage_packages[4].package_version))
    
    # there are 3 things, need to update, not found in portage, and no update necessary
    results = []
    for gnome_pack in gnome_packages:
        
        # first find the portage package, they might not be numerically ordered the same way
        print "Searching for: " + "GNOME name: " +str(gnome_pack.name)
        matching_portage_pack = [portage_pack for portage_pack in portage_packages if portage_pack.name == gnome_pack.name]
        #find_pack(pack.name, portage_packages)
        
        # make sure that theres a matching one
        if 0 != len(matching_portage_pack):
            matching_portage_pack = matching_portage_pack[0] # there should only be one
            print "Comparing: " + str(gnome_pack.package_version) + " to " + str(matching_portage_pack.package_version)
            print matching_portage_pack.name
            ret_val = vercmp(gnome_pack.package_version, matching_portage_pack.package_version)
            
            portage_latest = False
            try:
                portage_to_latest = vercmp(matching_portage_pack.package_version, gnome_pack.latest_version) # also compare to latest portage version
            except:
                print "can't compare portage to latest, probably is no latest version."
                portage_to_latest = "NUKED"
                
            if portage_to_latest != "NUKED" and 0 > portage_to_latest:
                # theres a newer portage version!
                portage_latest = True
            
            if 0 < ret_val:
                # package 2 is older
                print gnome_pack.name + "is out of date!" + "need version " + gnome_pack.package_version
                results.append(PackageUpdate(gnome_pack.name, matching_portage_pack.package_version, gnome_pack.package_version, gnome_pack.latest_version, Status.NeedUpdate))
            elif 0 > ret_val:
                # package 2 is newer
                print "how did this happen? portage has a newer version!"
                if portage_latest:
                    results.append(PackageUpdate(gnome_pack.name, matching_portage_pack.package_version, gnome_pack.package_version, gnome_pack.latest_version, Status.NewerVersion))
                else:
                    results.append(PackageUpdate(gnome_pack.name, matching_portage_pack.package_version, gnome_pack.package_version, gnome_pack.latest_version, Status.Compliant))
            else:
                # equal
                print str(gnome_pack.name) + "is up to date!"
                if portage_latest:
                    results.append(PackageUpdate(gnome_pack.name, matching_portage_pack.package_version, gnome_pack.package_version, gnome_pack.latest_version, Status.NewerVersion))
                else:
                    results.append(PackageUpdate(gnome_pack.name, matching_portage_pack.package_version, gnome_pack.package_version, gnome_pack.latest_version, Status.Compliant))
        else:
            print "This package not found in portage: " + str(gnome_pack.name)
            results.append(PackageUpdate(gnome_pack.name, "","", "",Status.NotFound))
            
    
    # variables to hold for stats
    update_needed = float(0)
    compliant = float(0)
    not_found = float(0)
    newer = float(0)
    total_packs = len(results)
    
    # now we have all the results in the results list.
    # just time to generate some kind of "useful" output.
    # for now, lets just make a crappy html file. ( this should use css and the like )
    # name, portage_version, gnome_version, status <-- is whats in the PackageUpdate object
    current_time = str(time.asctime(time.localtime()))
    
    put, get = os.popen4("genlop -nr | tail -n1")            
    for line in get.readlines():
        line = string.strip(line)
    last_sync = line
    
    file = FileStuff("gnome_status.html")
    file.write(["<html>"])
    file.append(["<h2>Gnome 2.12 Progress</h2>"])
    file.append(["Statistics and who's working on what at the bottom of the page, contact AllanonJL if anything is not correct.<br>"])
    file.append(["Generated Date: " + current_time + "<br>"])
    #file.append(["Compared to last sync at: " + last_sync + "<br>"])
    file.append(["Directories checked ( recursively ): <br>"])
    for dir in dirs:
        directory = "ftp.gnome.org/" + dir
        file.append(["<a href=" + "\"" + "ftp://" + directory + "\"" + ">" + directory + "</a>" + "<br>"])
        
    file.append(["<table cellpadding=\"10\">"])
    file.append(["<th>Package Name</th><th>Portage Version</th><th>Official Release Version</th><th>Latest Available Version</th>"])
    
    for pack in results:
        if pack.status == Status.NeedUpdate:
            file.append(["<tr bgcolor=\"red\">"])
            update_needed+=1
        
        if pack.status == Status.Compliant:
            file.append(["<tr bgcolor=\"green\">"])
            compliant+=1
            
        if pack.status == Status.NotFound:
            file.append(["<tr bgcolor=\"grey\">"])
            not_found+=1
            
        if pack.status == Status.NewerVersion:
            file.append(["<tr bgcolor=\"33CC00\">"])
            compliant+=1
        
        file.append(["<td>" + str(pack.name) + "</td>",
                     "<td>" + str(pack.portage_version) + "</td>",
                     "<td>" + str(pack.gnome_version) + "</td>",
                     "<td>" + str(pack.latest_version) + "</td>"
                     ])
            
        file.append(["</tr>"])

    file.append(["</table>"])
    
    file.append(["Statistics:<br>"])
    update_percent = float(update_needed / total_packs) * 100
    compliant_percent = float(compliant / total_packs) * 100
    notfound_percent = float(not_found / total_packs) * 100
    total_percent = float( (update_needed + not_found ) / total_packs ) * 100
    
    file.append(["Compliant Packs: " + str('%0.2f' % compliant_percent)+ "%" + "  Number = " + str(compliant) + "<br>"])
    file.append(["Packs that need to be updated: " + str('%0.2f' % update_percent)+ "%" + "  Number = " + str(update_needed) + "<br>"])
    file.append(["New Packs that need to be added: " + str('%0.2f' % notfound_percent)+ "%" + "  Number = " + str(not_found) + "<br>"])
    
    file.append(["<br>"])
    file.append(["<br>"])
    file.append(["<iframe src=\"./packages\" width=\"100%\" height=\"300\">"])
    file.append(["Your Browser doesn't support iframes d00d!"])
    file.append(["</iframe>"])

    file.append(["<br>"])
    file.append(["</html>"])
    
    print "Done!"