baseUploader.py

#!/usr/bin/env python
import sys, os , ftplib, shelve, time, string
#
#   BaseUploader.py
#
#   Uploads files to base.google.com.  Can be run in a "server"
#   mode which will check a specified directory periodically and
#   upload new items.
#
#   Requires:
#       1. Google account (http://base.google.com/base/welcome)
#       2. Sign up for a base FTP acct. (http://base.google.com/base/updateftp)
#       3. Understand:
#           http://base.google.com/base/howtobulkupload.html
#           http://base.google.com/base/tab_attributes.html#custom
#           http://base.google.com/base/tab_instructions.html
#   
#   Only text and xml files (.txt or .xml), and compressed text and xml files 
#   (.gz, .zip, .Z, or .bz2) will be accepted, so plan accordingly
#
#   Usage:
#
#   Command line usage is simple:
#   
#   %python baseUploader.py  
#
#   Will go through your LDIR (see below) and upload any new items to 
#   base.google.com
#
#   A better way to use this is to just fire this up in the background and 
#   forget about it.
#
#   %nohup python BaseUploader.py -d &
#
#   If you find you have CPU/Process limits (running this on a hosted account)
#   a cron job or scheduled tasks. 
#
#   cron entry (runs at the top of every hour )
#   0  *  *   *   * /full/path/to/BaseUploader.py > /dev/null 2>&1
#
#   You may use this code however you see fit in any form whatsoever, assuming
#   you're abiding by Google's Terms.
#
#
#
#   Command-line options:
#       -d :        Run in "server" mode.  Use this with CHECK_FREQ (below)
#       --debug:    write out debug statements.
#
#
#
#   Things to ponder:
#   
#   You can easily see where you could use base.google.com as an extended
#   personal disk storage device.  Storing your addressbook, recipies, etc.
#   
#   You could also throw in encryption to the mix so that everything stored
#   up at base.google.com is encrypted and safe.  I personally am using
#   gpg to encode all items that go into my LDIR.  
#   
#   November 2005
#   Cameron Mallory   cmallory berserk org
#

##
##  Items you will want to change
##
# Define the local directory name to put data in
SYNC_DIR = "/home/username/base"
BASE_USER = "your.google.base.username"
BASE_PASS = "your.google.base.password"

# How often to check (in minutes)
CHECK_FREQ = 60

##
##  Shouldn't need to change anything below this line
##

class BaseUploader:
    SERVER = "uploads.google.com"
    USER = ""
    PASS = ""
    LDIR = ""
    HIST = None
    SLEEP = ""
    
    ftpCon = None;
    isConnected = False
    isDebugging = False
    
    def __init__( self, u, p, uploadDir, sleep ):
        self.USER = u
        self.PASS = p
        
        try:
            self.SLEEP = int(sleep *60 )
        except:
            self.SLEEP = 60*60
            
        self.LDIR = os.path.normpath( uploadDir )
        if not os.path.isdir( self.LDIR ):
            os.mkdir( self.LDIR )
            

      
    def connectToServer( self ):
        try:
            self.debug( "Connecting to " + self.SERVER )
            self.ftpCon = ftplib.FTP( self.SERVER , self.USER, self.PASS )
            self.ftpCon.set_pasv( 1 )
            self.isConnected = True
        except:
            self.debug( str( sys.exc_info() ) )
            self.isConnected = False
            return
            


    def disconnectFromServer( self ):
        try:
            self.ftpCon.quit()
        except:
            pass
     
     
    def getNewFiles( self ):
        files = []
        foo = os.walk( self.LDIR )
        for data in foo:
            (dirpath, dirnames, filenames) = data
            for f in filenames :
                files.append( os.path.normpath( dirpath + "/" + f ) )
        return files



    def upload( self ):
        newFiles = self.getNewFiles()
        if ( len( newFiles ) > 0 ):
            self.connectToServer()
            self.HIST = shelve.open( "base.history" )
            for f in newFiles:
                if ( self.isConnected ):
                    self.uploadFile( f )
            self.HIST.close()
            self.debug( "Uploaded files: " )
            self.debug( self.ftpCon.retrlines('LIST') )
            self.disconnectFromServer()
 
 
    def uploadFile( self, file ): 
        if ( not self.HIST.has_key( file ) ):
            try:
                self.debug( "Uploading " + file ) 
                fname = file.split('/')[-1]
                f = open( file ,'rb')
                self.ftpCon.storbinary('STOR ' + fname, f)
                f.close()    
                
                self.HIST[ str(file)] =  str( time.asctime(time.localtime()))
            except:
                self.debug( str( sys.exc_info() ) )
                #pass


    def debug( self, mesg ):
        if self.isDebugging:
            print mesg
            
    def debuggingOn( self ):
        self.isDebugging = True
        
        
    def run( self ):
        while ( True ):
            self.upload()
            print "Last check: " , str( time.asctime(time.localtime()))
            time.sleep( self.SLEEP )


if __name__ == "__main__":
    gb = BaseUploader( BASE_USER, BASE_PASS, SYNC_DIR, CHECK_FREQ )
    
    debug = False
    server = False
    if ( len( sys.argv ) > 1 ):
        for a in sys.argv :
            if a == "--debug" or a == "-v" or a == "--debug":
                debug = True
            elif a == "-d" :
                server = True
    if ( debug ):
        gb.debuggingOn()
    
    if ( server ):
        gb.run()
    else:
        gb.upload()