#!/usr/bin/env python2.7

"""A FUSE based filesystem view of VOSpace."""

from sys import argv, exit, platform
import time
import vos
import vos.fuse
from vos.fuse import FUSE, Operations, FuseOSError, LoggingMixIn
import tempfile
from threading import Lock
from errno import EACCES, EIO, ENOENT, EISDIR, ENOTDIR, ENOTEMPTY, EPERM, EEXIST, ENODATA, ECONNREFUSED, EAGAIN, ENOTCONN
ENOATTR=93
import os
import vos
from os import O_RDONLY, O_WRONLY, O_RDWR, O_APPEND
import logging
import sqlite3
from vos.__version__ import version
from ctypes import cdll
from ctypes.util import find_library
import urlparse 
libcPath = find_library('c')
libc = cdll.LoadLibrary(libcPath)

READBUF=2**20
DAEMON_TIMEOUT = 60
READ_SLEEP = 1

def flag2mode(flags):
    md = {O_RDONLY: 'r', O_WRONLY: 'w', O_RDWR: 'w+'}
    m = md[flags & (O_RDONLY | O_WRONLY | O_RDWR)]

    if flags | O_APPEND:
        m = m.replace('w', 'a', 1)

    return m


class VOFS(LoggingMixIn, Operations):
#class VOFS(Operations):
    """The VOFS filesystem opperations class.  Requires the vos (VOSpace) python package.

    To use this you will also need a VOSpace account from the CADC.
    """
    ### VOSpace doesn't support these so I've disabled these operations.
    chown = None
    link = None
    mknode = None
    rmdir = None
    symlink = None
    getxattr = None
    listxattr = None
    removexattr = None
    setxattr = None


    def __init__(self, root, cache_dir, conn=None, cache_limit=1024*1024*1024, cache_nodes=False):
        """Initialize the VOFS.  

        The style here is to use dictionaries to contain information
        about the Node.  The full VOSpace path is used as the Key for
        most of these dictionaries."""

        # This dictionary contains the Node data about the VOSpace node in question
        self.node = {}
        # Standard attribtutes of the Node
        # Where in the file system this Node is currently located
        self.cache={}
        self.loading_dir={}
        # How old is a given reference
        self.cache_nodes=cache_nodes

        # These next dictionaries keep track of pointers 

        # A dictionary or properties about the cached version of the
        # file.  the name of the dictionary should be something else
        # but I started calling this fh for other reasons.
        # Refactoring would help here.
        self.fh={'None': False}

        # What is the 'root' of the VOSpace? (eg vos:MyVOSpace) 
        self.root = root
        # VOSpace is a bit slow so we do some cahcing.
        self.cache_limit=cache_limit
        self.cache_dir = os.path.abspath(os.path.normpath(os.path.join(cache_dir,root)))
        self.cache_db = os.path.abspath(os.path.normpath(os.path.join(cache_dir,"#vofs_cache.db#")))
        
        if not os.access(self.cache_dir,os.F_OK):
            os.makedirs(self.cache_dir)

        ## initialize the md5Cache db
        sqlConn = sqlite3.connect(self.cache_db)
        sqlConn.execute("create table if not exists md5_cache (fname text, md5 text, st_mtime int)")
        sqlConn.commit()
        sqlConn.close()
        ## build cache lookup if doesn't already exists


        ## All communication with the VOSpace goes through this client connection.
        try:
           self.client = vos.Client(rootNode=root,conn=conn)
        except Exception as e:
           e=FuseOSError(e.errno)
           e.filename=root
           e.strerror=getattr(e,'strerror','failed while making mount')
           raise e

        self.rwlock = Lock()


    def __call__(self, op, path, *args):
        return super(VOFS, self).__call__(op, path, *args)

    def __del__(self):
        self.node=None

    def add_to_cache(self,path):
        """Add path to the cache reference.

        path: the vofs location of the file (str)."""

        fname = os.path.normpath(self.cache_dir+path)
        if path not in self.cache:
            self.cache[path] = {'fname': fname,
                                'cached': False,
                                'writing': False}

        ## build the path to this cache, if needed
        ## doing as an error fall through is more efficient
        try:
            dir_path = os.path.dirname(fname)
            os.makedirs(dir_path)
        except OSError as exc: 
            if exc.errno == EEXIST and os.path.isdir(dir_path):
                pass
            else: 
                logging.error("Failed to create cache directory: %s" %( dir_path))
                logging.error(str(exc))
                raise FuseOSError(exc.errno)

        ## Create the cache file, if it doesn't already exist
        ## or open for RDWR if the file already exists
        try:
            fh=os.open(fname,os.O_CREAT | os.O_RDWR)
        except OSError as e:
            e=FuseOSError(e.errno)
            e.strerror=getattr(e,'strerror','failed on open of %s' % ( path))
            e.message="Not able to write cache (Permission denied: %s)" % ( fname) 
            raise e
        finally:
            os.close(fh)

        return

    def get_cache_filename(self, path):
        """Return the name of the file associated with a given vofs path.

        path: vofs location (str)."""
        return self.cache.get('fname',None)

    def list_cache(self):
        """Get a list of the cache files that are in active use"""
        activeCache=[]
        for path in self.cache:
            activeCache.append(self.cache[path]['fname'])
        return activeCache

    def delNode(self,path,force=False):
        """Delete the references associated with this Node"""
        if not self.cache_nodes or force :
            self.node.pop(path,None)
        

    def access(self, path, mode):
        """Check if path is accessible.  

        Only checks read access, mode is currently ignored"""
        logging.debug("Checking if -->%s<-- is accessible" %(path))
        try:
            self.getNode(path)
        except:
            return -1
        return 0


    def chmod(self, path, mode):
        """Set the read/write groups on the VOSpace node based on chmod style modes.

        This function is a bit funny as the VOSpace spec sets the name
        of the read and write groups instead of having mode setting as
        a separate action.  A chmod that adds group permission thus
        becomes a chgrp action.  

        Here I use the logic that the new group will be inherited from
        the container group information.
        """
        logging.debug("Changing mode for %s to %d" % ( path, mode))

        node = self.getNode(path)
        parent = self.getNode(os.path.dirname(path))

        if node.groupread == "NONE":
            node.groupread=parent.groupread
        if node.groupwrite == "NONE":
            node.groupwrite=parent.groupwrite
        # The 'node' object returned by getNode has a chmod method
        # that we now call to set the mod, since we set the group(s)
        # above.  NOTE: If the parrent doesn't have group then NONE is
        # passed up and the groupwrite and groupread will be set to
        # the string NONE.
        if node.chmod(mode):
            # Now set the time of change/modification on the path...
            self.getattr(path)['st_ctime']=time.time()
            ## if node.chmod returns false then no changes were made.
            try:
                self.client.update(node)
                self.getNode(path, force=True)
            except Exception as e:
                logging.debug(str(e))
                logging.debug(type(e))
                e=FuseOSError(getattr(e,'errno',EIO))
                e.filename=path
                e.strerror=getattr(e,'strerror','failed to chmod on %s' %(path))
                raise e

        # may also need to also update the cache properties.
        if self.is_cached(path):
            logging.debug("Changing permissions on %s also" % ( self.cache[path]['fname']))
            os.chmod(self.cache[path]['fname'],mode)
        


        
    def create(self, path, flags):
        """Create a node. Currently ignores the ownership mode"""
        import re,os

        logging.debug("Creating a node: %s with flags %s" % (path, str(flags)))

        # Create is handle by the client. 
        # This should fail if the basepath doesn't exist
        try: 
            self.client.open(path,os.O_CREAT).close()
            
            node = self.getNode(path)
            parent=  self.getNode(os.path.dirname(path))

            # Force inheritance of group settings. 
            node.groupread = parent.groupread
            node.groupwrite = parent.groupwrite
            if node.chmod(flags):
                ## chmod returns True if the mode changed but doesn't do update.
                self.client.update(node)
                node = self.getNode(path,force=True)
                
        except Exception as e:
            logging.error(str(e))
            logging.error("Error trying to create Node %s" %(path))
            f=FuseOSError(getattr(e,'errno',EIO))
            f.strerror=getattr(e,'strerror','failed to create %s' %(path))
            raise f

        ## now we can just open the file in the usual way and return the handle
        return self.open(path,os.O_WRONLY)

    def flushnode(self,path,fh):
        """Flush the data associated with this fh to the Node at path
        in VOSpace.
        
        Flushing the VOSpace object involves pushing the entire file
        back over the network. This should only be done when really
        needed as the network connection can be slow."""
        mode = flag2mode(self.fh[fh]['flags'])
        if not ('w' in mode or 'a' in mode ):
            logging.debug("file was not opened for writing")
            return 

        if opt.readonly:
            logging.debug("File system is readonly... no flushnode allowed")
            return 

        destination = self.getNode(path).uri
        source = self.cache[path]['fname']
        ### if we are writing then we need to wait for writing to finish
        self.client.copy(source,destination)
        self.cache[path]['writing']=False

        return        

    def flush(self,path,fh):
        """Flush the cached version of the file.

        This could be a problem since users expect the file in VOSpace
        to be updated too.  But we only do that with a flushnode call
        which we do on close of the file."""
        

        if opt.readonly:
            logging.debug("File system is readonly, no sync allowed")
            return

        return os.fsync(fh) 

  
    def fsync(self,path,datasync,fh):
        if opt.readonly:
            logging.debug("File system is readonly, no sync allowed")
            return 
        mode=''
        if self.fh.get(fh,None) is not None:
            locked = self.fh[fh].get('locked',False)
            if locked:
                logging.info("%s is locked, no sync allowed" % path)
                raise FuseOSError(EPERM)
            mode = flag2mode(self.fh[fh]['flags'])
        if 'w' in mode or 'a' in mode:
            try:
                os.fsync(fh)
                # set the modification time on this node..
                # if the system asks for information about this node
                # we need to know that this node was modified.
                self.getattr(path)['st_mtime']=time.time()
            except:
                logging.critical("Failed to sync fh %d?" % ( fh))
                pass


    def getNode(self,path,force=False,limit=0):
        """Use the client and pull the node from VOSpace.  
        
        Currently force=False is the default... so we never check
        VOSpace to see if the node metadata is different from what we
        have.  This doesn't keep the node metadata current but is
        faster if VOSpace is slow.
        """

      
        logging.debug("force? -> %s path -> %s" % ( force, path))
        ### force if this is a container we've not looked in before
        if path in self.node and not force:
            node = self.node[path]
            if node.isdir() and not limit==0 and not len(node.getNodeList()) > 0 :
               force = True
            if not force:
               logging.debug("Sending back cached metadata for %s" % ( path))
               return node
            

        ## Pull the node meta data from VOSpace.
        try:
            logging.debug("requesting node %s from VOSpace" % ( path))
            self.node[path]=self.client.getNode(path,force=True,limit=limit)
        except Exception as e:
            logging.error(str(e))
            logging.error(type(e))
            ex=FuseOSError(getattr(e,'errno',ENOENT))
            ex.filename=path
            ex.strerror=getattr(e,'strerror','Error getting %s' % (path))
            raise ex

        if self.node[path].isdir() and self.node[path]._nodeList is not None:
            for node in self.node[path]._nodeList:
               subPath=os.path.join(path,node.name)
               self.node[subPath]=node
        return self.node[path]

    def is_cached(self,path):
    	return self.cache.get(path,{'cached': False}).get('cached', False)

    def getattr(self, path, fh=None):
        """Build some attributes for this file, we have to make-up some stuff"""

        import os
        logging.debug("getting attributes of %s" % ( path))

        if self.is_cached(path):
            f = os.stat(self.cache[path]['fname'])
            return dict((name, getattr(f, name)) for name in dir(f) if not name.startswith('__'))

        ## not in cache -> add to dictionary of Node attributes if need be
        attr = self.getNode(path).attr

        atime=attr.get('st_atime',time.time())
        mtime=attr.get('st_mtime',atime)
        ctime=attr.get('st_ctime',atime)

        if mtime > atime or ctime > atime:
            ### the modification/change times are after the last access
            ### so we should access this VOSpace node again.
            logging.debug("Getting node details for stale node %s" % ( path))
            attr=self.getNode(path,limit=0,force=True).attr
        return attr

    def dead_getxattr(self, path, name, position=0):
        """Get the value of an extended attribute"""
        value=self.getNode(path).xattr.get(name,None)
        if value is None:
            raise FuseOSError(ENOATTR)
        import binascii
        return binascii.a2b_base64(value)

    def dead_listxattr(self, path):
        """Send back the list of extended attributes"""
        return self.getNode(path).xattr.keys()
        

    def mkdir(self, path, mode):
        """Create a container node in the VOSpace at the correct location.

        set the mode after creation. """
        #try:
        if 1==1:
            parentNode = self.getNode(os.path.dirname(path), force=False, limit=1)
            if parentNode and parentNode.props.get('islocked', False):
                logging.info("Parent node of %s is locked." % path)
                raise FuseOSError(EPERM)
            self.client.mkdir(path)
            self.chmod(path,mode)
        #except Exception as e:
        #   logging.error(str(e))
        #   ex=FuseOSError(e.errno)
        #   ex.filename=path
        #   ex.strerror=e.strerror
        #   raise ex
        return


    def open(self, path, flags, *mode):
        """Open file with the desired modes

        Here we return a handle to the cached version of the file
        which is updated if older and out of synce with VOSpace.

        """
        import io

        logging.debug("Opening %s with flags %s" % ( path, flag2mode(flags)))

        ## see if this node exists
        try:
            node = self.getNode(path)
        except Exception as e:
            if e.errno == ENOENT:
                if flags | os.O_RDONLY:
                    # file openned for readonly doens't exist
                    FuseOSError(ENOENT)
                else:
                    ## no problem, maybe we should have created first... 
                    pass
            else:
                raise FuseOSError(e.errno)
            
        ### check if this file is locked, if locked on vospace then don't open
        locked=False

        if node and node.props.get('islocked', False):
             logging.info("%s is locked." % path)
             locked = True

        if not locked:
             if node.type == "vos:DataNode":
                  parentNode = self.getNode(os.path.dirname(path),force=False,limit=1)
                  if parentNode.props.get('islocked', False):
                       logging.info("%s is locked by parent node." % path)
                       locked=True
             elif node.type == "vos:LinkNode":
	          try:
		     ## sometimes targetNodes aren't internall... so then not locked
                      targetNode = self.getNode(node.target,force=False,limit=1)  
                      if targetNode.props.get('islocked', False):
                          logging.info("%s target node is locked." % path)
                          locked=True
                      else:
                          targetParentNode = self.getNode(os.path.dirname(node.target),force=False,limit=1)
                          if targetParentNode.props.get('islocked', False):
                              logging.info("%s is locked by target parent node." % path)
                              locked=True
                  except Exception as e:
                      logging.error("Got an error while checking for lock: "+str(e))
                      pass



        ## setup the data cache structure, which returns a 
        self.add_to_cache(path)

        ## open the cache file, return that handle to fuse layer.
        fh = os.open(self.cache[path]['fname'], os.O_RDWR )

        ## also open a direct io path to this handle, use that for file write operations 
        fs = io.open(fh,mode='r+b')
        os.lseek(fh,0,os.SEEK_SET)

        ## the fh dictionary provides lookups to some critical info... 
        self.fh[fh]={'flags': flags, 'fs': fs, 'locked': locked}

        return fh

    def read(self, path, size=0, offset=0, fh=None):
        """ Read the entire file from the VOSpace server, place in a temporary file and then offset
        and return size bytes  

        """

        ## Read from the requested filehandle, which was set during 'open'
        if fh is None:
            raise FuseOSError(EIO)

        ## raise an error if cache structure doens't exist
        if path not in self.cache:
            logging.error("Tried to read but cached wasn't initialied for %s" % ( path))
            raise FuseOSError(EIO)

        import os
        from ctypes import create_string_buffer    

        with self.rwlock:
            if self.cache[path]['writing'] :
                ## the cache is currently being written to... so wait for bytes to arrive
                logging.debug("checking if cache writing has gotten far enought")
                st_size = os.stat(self.cache[path]['fname']).st_size
                waited = 0
                logging.debug("loop until %s + %s < %s or read from vospace reaches EOF." % ( str(size), str(offset), str(st_size)))
                while ( st_size < offset + size and self.cache[path]['writing'] ):
                    logging.debug("sleeping for %d ..." % (READ_SLEEP))
                    time.sleep(READ_SLEEP)
                    logging.debug("wake up ")
                    waited += READ_SLEEP
                    if waited > DAEMON_TIMEOUT - 3:
                        logging.debug("Slept for %d, data not ready" % (waited))
                        raise FuseOSError(EAGAIN)
                    st_size = os.stat(self.cache[path]['fname']).st_size
                logging.debug("Done waiting. %s + %s < %s or reached EOF" % ( str(size), str(offset), str(st_size)))
                os.lseek(fh,offset,os.SEEK_SET)
                logging.debug("sending back %s for %s" % ( str(fh), str(size)))
                buf = create_string_buffer(size)
                retsize = libc.read(fh,buf,size)
                return buf


            logging.debug("%s is in data cache: %s " % ( path, self.is_cached(path)))
                
            vosMD5 = self.getNode(path).props.get('MD5','d41d8cd98f00b204e9800998ecf8427e')
            cacheMD5 = self.get_md5_db(self.cache[path]['fname'])
            if cacheMD5['md5'] == vosMD5 :
                ## return a libc.read buffer
                fs = self.fh[fh]['fs']
                fs.seek(offset,os.SEEK_SET)
                logging.debug("returning bytes %s starting at  %s" % ( offset, fs.tell()))
                buf = fs.read(size)
                retsize = len(buf)
                return create_string_buffer( buf[:retsize], retsize )

            ## get a copy from VOSpace if the version we have is not currnet or cached.
            import thread
            if not self.cache[path]['writing']:
                self.cache[path]['writing'] = True
                thread.start_new_thread( self.load_into_cache, (path, int(fh)))

        return self.read(path, size, offset, fh)
        
    def readlink(self, path):
        """Return a string representing the path to which the symbolic link points.

	path: filesystem location that is a link

	returns the file that path is a link to. 

	Currently doesn't provide correct capabilty for VOSpace FS.
        """
        return self.getNode(path).target
        
    def load_into_cache(self, path, fh ):
        """Load path from VOSpace and store into fh"""
        logging.debug("fh: %d" % ( fh))
        logging.debug("self: %s" % ( str(self.fh)))
        try:
            os.fsync(fh)
            os.ftruncate(fh,0)
            wh = os.open(self.cache[path]['fname'],os.O_WRONLY)
            r = self.client.open(path,mode=os.O_RDONLY,view="data")
            logging.debug("writing to %s" % ( self.cache[path]['fname']))
            logging.debug("reading from %s" % ( str(r)))
            wrote = 0
            while True:
                buf = r.read(READBUF)
                if not buf:
                    break
                if os.write(wh,buf)!=len(buf):
                    raise FuseOSError(EIO)
                wrote = wrote + len(buf)
            os.fsync(wh)
            logging.debug("Wrote: %d" % (wrote))
            vosMD5 = self.getNode(path).props.get('MD5','d41d8cd98f00b204e9800998ecf8427e')
            cacheMD5 = self.get_md5_db(self.cache[path]['fname'])
            if vosMD5 != cacheMD5['md5']:
                logging.debug("vosMD5: %s cacheMD5: %s" % ( vosMD5, cacheMD5['md5']))
                raise FuseOSError(EIO)
            self.cache[path]['cached'] = True
            logging.debug("Finished caching file %s" %  ( str(self.fh)))
        except Exception as e:
            logging.error("ERROR: %s" % (str(e)))
            ex = FuseOSError(e.errno)
            ex.strerror = getattr(e,'strerror','failed while reading from %s' %(path))
            raise ex
        finally:
            r.close()
            os.close(wh)
            self.cache[path]['writing'] = False
        return 


    def get_md5_db(self,fname):
        """Get the MD5 for this fname from the SQL cache"""
        if not os.access(fname,os.R_OK):
            raise FuseOSError(EACCES)
        sqlConn=sqlite3.connect(self.cache_db)
        sqlConn.row_factory = sqlite3.Row
        cursor= sqlConn.cursor()
        cursor.execute("SELECT * FROM md5_cache WHERE fname = ?", (fname,))
        md5Row=cursor.fetchone()
        cursor.close()
        sqlConn.close()
	if md5Row is None or md5Row['st_mtime'] < os.stat(fname).st_mtime:
            self.update_md5_db(fname)
            return self.get_md5_db(fname)
        if md5Row is None:
            md5Row={}
        return md5Row

    def delete_md5_db(self,fname):
        """Delete a record from the cache MD5 database"""
        sqlConn=sqlite3.connect(self.cache_db)
        sqlConn.row_factory =  sqlite3.Row
        cursor = sqlConn.cursor()
        cursor.execute("DELETE from md5_cache WHERE fname = ?", ( fname,))
        sqlConn.commit()
        cursor.close()
        sqlConn.close()
        return 
        

    def update_md5_db(self,fname):
        """Update a record in the cache MD5 database"""
        import hashlib
	st_mtime_end = None
	st_mtime_start = os.stat(fname).st_mtime
        ## make sure the file doesn't change while were computing the MD5
	while st_mtime_end != st_mtime_start:
            md5 = hashlib.md5()  
	    ### get the modifcation time prior to reading the file
            st_mtime_start = os.stat(fname).st_mtime
            r=open(fname,'r')
            while True:
                buf=r.read(READBUF)
                if len(buf)==0:
                    break
                md5.update(buf)
            r.close()
	    ### now get mtime again
            st_mtime_end = os.stat(fname).st_mtime
        ## UPDATE the MD5 database
        sqlConn=sqlite3.connect(self.cache_db)
        sqlConn.row_factory = sqlite3.Row
        cursor=sqlConn.cursor()
        cursor.execute("DELETE FROM md5_cache WHERE fname = ?", (fname,))
        if md5 is not None:
            cursor.execute("INSERT INTO md5_cache (fname, md5, st_mtime) VALUES ( ?, ?, ?)", (fname, md5.hexdigest(), st_mtime_end))
        sqlConn.commit()
        cursor.close()
        sqlConn.close()
        return 


    def readdir(self, path, fh):
        """Send a list of entries in this directory"""
        logging.debug("Getting directory list for %s " % ( path))
        ## reading from VOSpace can be slow, we'll do this in a thread
        import thread, time
        if not self.loading_dir.get(path,False):
            self.loading_dir[path]=True
            thread.start_new_thread(self.load_dir,(path, ))
        thread_wait=0
        while self.loading_dir.get(path,False):
            time.sleep(READ_SLEEP)
            thread_wait += READ_SLEEP
            logging.error("Waiting ... %ds" % ( thread_wait))
            if thread_wait > DAEMON_TIMEOUT - 3:
                e = FuseOSError(EAGAIN)
                e.strerror = "Timeout waiting for directory listing"
                raise e
        return ['.','..'] + [e.name.encode('utf-8') for e in self.getNode(path,force=False,limit=None).getNodeList() ]

    def load_dir(self,path):
        """Load the dirlist from VOSpace"""
        logging.debug("Starting getNodeList thread")
        self.getNode(path,force=not opt.cache_nodes,limit=None).getNodeList()
        self.loading_dir[path] = False
        logging.debug("Got listing for %s" %(path))
        return 

    def release(self, path, fh):
        """Close the file, but if this was a holding spot for writes, then write the file to the node"""
        import os

        ## get the MODE of the original open, if 'w/a/w+/a+' we should write to VOSpace
        ## we do that here before closing the filehandle since we delete the reference to 
        ## the file handle at this point.
        mode = os.O_RDONLY
        if self.fh.get(fh,None) is not None:
            mode = self.fh[fh]['flags']

        ## remove references to this file handle.
        writing_wait = 0
        while self.cache[path]['writing'] :
            time.sleep(READ_SLEEP) 
            writing_wait += READ_SLEEP
            if writing_wait > DAEMON_TIMEOUT -3 :
                raise FuseOSError(EAGAIN)

        ## On close, if this was a WRITE opperation then update VOSpace
        ## unless the VOSpace is actually newer than this cache file.
        ### copy the staging file to VOSpace if needed
        logging.debug("node %s currently open with mode %s, releasing" % ( path, mode))
        if mode & ( os.O_RDWR | os.O_WRONLY | os.O_APPEND | os.O_CREAT ):
            ## check if the cache MD5 is up-to-date
            md5Row = self.get_md5_db(self.cache[path]['fname'])
            voMD5 = self.getNode(path,force=True).props.get('MD5','d41d8cd98f00b204e9800998ecf8427e')
            if md5Row['md5'] != voMD5:
                logging.debug("cache MD5 %s",md5Row['md5'])
                logging.debug("VOSpace MD5 %s",voMD5)
                logging.debug("PUSHING contents of %s to VOSpace location %s " % (self.cache[path]['fname'],path))
                ## replace VOSpace copy with cache version.
                self.fsync(path,False,fh)
                if not self.cache[path]['writing']:
                    import thread
                    self.cache[path]['writing']=True
                    thread.start_new_thread(self.flushnode,(path,fh))
                    return self.release(path,fh)

        ##  now close the fh
        try:
            os.close(fh)
        except Exception as e:
            ex = FuseOSError(getattr(e,'errno',EIO))
            ex.strerror = getattr(ex,'strerror','failed while closing %s' %(path))
            logging.debug(str(e))
            raise ex
        finally:
            self.fh.pop(fh,None)
            self.delNode(path)
            #self.cache.pop(path,None)
        ## clear up the cache
        self.clear_cache()
        return 



    def dead_removexattr(self, path, name):
        """Remove the named attribute from the xattr dictionary"""
        node=self.getNode(path)
        node.changeProp(name,None)
        try:
            del self.getNode(path).attr[name]
        except KeyError:
            raise FuseOSError(ENOATTR)
        return 0

    def rename(self,src,dest):
        """Rename a data node into a new container"""
        logging.debug("Original %s -> %s" % ( src,dest))
        #if not self.client.isfile(src):
        #   return -1
        #if not self.client.isdir(os.path.dirname(dest)):
        #    return -1
        try:
            logging.debug("Moving %s to %s" % ( src,dest))
            result=self.client.move(src,dest)
            logging.debug(str(result))
            if result:
                srcPath=os.path.normpath(self.cache_dir+src)
                destPath=os.path.normpath(self.cache_dir+dest)
                if os.access(srcPath,os.F_OK):
                    # only rename in cache if the source exists
                    dirs=os.path.dirname(destPath)
                    if not os.path.exists(dirs):
                        os.makedirs(dirs)
                    os.rename(srcPath,destPath)
                return 0
            return -1
        except Exception, e:
            logging.error("%s" % str(e))
            import re
            if re.search('NodeLocked', str(e)) != None:
                raise FuseOSError(EPERM)
            return -1
    
    def dead_setxattr(self, path, name, value, size, options, *args):
        """Simple xattr setting, ignorring options for now"""
        node=self.getNode(path)
        ### call changeProp on the node so that the XML is updated
        import binascii
        value=binascii.b2a_base64(value)
        if node.changeProp(name,value)==1:
            """The node properties changed so force node update back to VOSpace"""
            self.client.update(node)
        return 0

        

    def cache_size(self):
        """Determine how much disk space is being used by the local cache"""
        import os
        start_path = self.cache_dir
        total_size = 0
        self.atimes={}
        oldest_time=time.time()
        for dirpath, dirnames, filenames in os.walk(start_path):
            for f in filenames:
                fp = os.path.join(dirpath, f)
                if oldest_time > os.stat(fp).st_atime and fp not in self.list_cache():
                    oldest_time = os.stat(fp).st_atime
                    self.oldest_file = fp
                total_size += os.path.getsize(fp)
        return total_size

    def clear_cache(self):
        """Clear the oldest files until cache_size < cache_limit"""
        while ( self.cache_size() > self.cache_limit) :
            logging.debug("Removing file %s from the local cache" % ( self.oldest_file))
            os.unlink(self.oldest_file)
            self.oldest_file=None

    def rmdir(self,path):
        node=self.getNode(path)
        #if not node.isdir():
        #    raise FuseOSError(ENOTDIR)
        #if len(node.getNodeList())>0:
        #    raise FuseOSError(ENOTEMPTY)
        if node and node.props.get('islocked', False):
            logging.info("%s is locked." % path)
            raise FuseOSError(EPERM)
        fname=os.path.normpath(self.cache_dir+path)
        if os.access(fname,os.F_OK):
            os.rmdir(fname)
        self.client.delete(path)
        self.delNode(path,force=True)

        
    def statfs(self, path):
        node=self.getNode(path)
        block_size=512
        bytes=2**33
        free=2**33
        
        if 'quota' in node.props:
            bytes=int(node.props.get('quota',2**33))
            used=int(node.props.get('length',2**33))
            free=bytes-used
        sfs={}
        sfs['f_bsize']=block_size
        sfs['f_frsize']=block_size
        sfs['f_blocks']=int(bytes/block_size)
        sfs['f_bfree']=int(free/block_size)
        sfs['f_bavail']=int(free/block_size)
        sfs['f_files']=len(node.getNodeList())
        sfs['f_ffree']=2*10
        sfs['f_favail']=2*10
        sfs['f_flags']=0
        sfs['f_namemax']=256
        return sfs
            
    
    def truncate(self, path, length, fh=None):
        """Perform a file truncation to length bytes"""
        logging.debug("Attempting to truncate %s (%d)" % ( path,length))


        ## do we have a fildes?  If so then don't close on truncate
        close=False
        if fh is None:
            logging.debug("don't have an open file handle, so creating one")
            close=True
            fh=self.open(path,os.O_RDWR)
	    self.add_to_cache(path)

        ## Check if we have a valid cached version of this file.
        ## but don't use standard read since we only want at most 'length' of this file
        if length > 0 :
            vosMD5  = self.getNode(path).props.get('MD5','d41d8cd98f00b204e9800998ecf8427e')
            cacheMD5 = self.get_md5_db(self.cache[path]['fname'])
            if not self.is_cached(path) and cacheMD5['md5'] != vosMD5 : 
                ## cache file  (really we don't need the entire file, just upto length
                ## TODO:  Fix this loop so it can exit on certain errors.
                success=False
                while not success:
                    os.lseek(fh,0,os.SEEK_SET)
                    try:
                        r = self.client.open(path,mode=os.O_RDONLY,view='data')
                        fpos=0
                        while fpos < length:
                            buf=r.read(READBUF)
                            if not buf :
                                ## stop reading at end of file.
                                success=True
                                break
                            chunk=min(length-fpos,len(READBUF)) 
                            if os.write(fh,buf[:chunk])!=chunk:
                                raise FuseOSError(EIO)
                            fpos=fpos+chunk
                            if fpos >= length:
                                success=True
                                break
                    except Exception as e:
                        logging.error("Failed during truncate cache bulding (%s), trying again"  % ( str(e)))
                    finally: 
                        r.close()

        ## now we can truncate the file.
        os.ftruncate(fh,length)
        ## set cached true since this version is now the keeper.
        self.cache[path]['cached'] = True
        if close :
            ### fuse normally calls truncate without first opening the file
            self.release(path,fh)
        ## Update the access/mod times 
        self.utimens(path)
        return

    def unlink(self,path):
    	node = self.getNode(path, force=False, limit=1)
        fname=os.path.normpath(self.cache_dir+path)
        if node and node.props.get('islocked', False):
            logging.info("%s is locked." % path)
            raise FuseOSError(EPERM)
        if os.access(fname,os.F_OK):
            os.unlink(fname)
        if node:
            self.client.delete(path)
        self.delNode(path,force=True)
        ## update the access times on the parent node


    def utimens(self, path, times=None):
        """Set the access and modification times of path"""
        logging.debug("Setting the access and modification times for %s " % ( path))
        logging.debug("%s" % (str(times)))
        if times is None:
          logging.debug("No times specified so using the current system time for access and modifcation")
          t=time.time()
          times = (t,t)
        else:
          logging.debug("Setting the access and modification times using times provided")
        logging.debug("Getting cache file name")
        fname=os.path.normpath(self.cache_dir+path)
        logging.debug("Setting access times on cached version at location %s" % ( fname))
        if os.access(fname,os.W_OK):
            logging.debug("Setting access times on cached version at location %s" % ( fname))
            try:
                os.utime(fname,times)
            except Exception as e:
                ex = FuseOSError(getattr(e,'errno',EIO))
                ex.strerror = getattr(e,'strerror','failed to set times on %s' %(path))
                raise ex
        logging.debug("Attempting to set the st_mtime and st_atime attributes")
        self.getattr(path)['st_mtime']=times[1]
        self.getattr(path)['st_atime']=times[0]
        return 

    def write(self, path, data, size, offset, fh=None):
        import ctypes

        if opt.readonly:
            logging.debug("File system is readonly.. so writing 0 bytes\n")
            return 0


        mode = flag2mode(self.fh[fh]['flags'])
        if not ('w' in mode or 'a' in mode ):
            logging.debug("file was not opened for writing")
            return 0

        locked = self.fh[fh].get('locked',False)
        if locked:
            logging.debug("file %s is locked, no write allowed" % ( path ) )
            raise FuseOSError(EPERM)

        logging.debug("%s -> %s" % ( path,fh))
        if not self.cache[path]['cached'] and offset > 0 :
            ## we are writing but never cached the original file, do that now
            try:
                logging.debug("Getting data from VOSpace cause the cache is empty")
                self.read(path,fh=fh)
            except IOError as e:
                f=FuseOSError(e.errno)
                f.strerror=getattr(e,'strerror','failed during write of %s' %(path))
                raise f
        ## Update the access/mod times and delete the md5 from the cache db
        ## self.utimens(path)
        logging.debug("%d --> %d" % ( offset, offset+size))
        self.delete_md5_db(self.cache[path]['fname'])
        self.cache[path]['cached']=True
        os.lseek(fh, offset, os.SEEK_SET)
        return libc.write(fh, data, size)


if __name__ == "__main__":

    import optparse

    #usage="%prog <root> <mountpoint>"


    parser = optparse.OptionParser(description='mount vospace as a filesystem.')

    parser.add_option("--vospace",help="the VOSpace to mount",default="vos:")
    parser.add_option("--mountpoint",help="the mountpoint on the local filesystem",default="/tmp/vospace")
    parser.add_option("--version",action="store_true",default=False,help="Print the version (%s)" % ( version))
    parser.add_option("-d","--debug",action="store_true")
    parser.add_option("-v","--verbose",action="store_true")
    parser.add_option("--warning",action="store_true",default=False,help="Print warning level log messages" )
    parser.add_option("-f","--foreground",action="store_true",help="Mount the filesystem as a foreground opperation and produce copious amounts of debuging information")
    parser.add_option("--log",action="store",help="File to store debug log to",default="/tmp/vos.err")
    parser.add_option("--cache_limit",action="store",type=int,help="upper limit on local diskspace to use for file caching",default=50*2**(10+10+10))
    parser.add_option("--cache_dir",action="store",help="local directory to use for file caching",default=None)
    parser.add_option("--certfile",help="location of your CADC security certificate file",default=os.path.join(os.getenv("HOME","."),".ssl/cadcproxy.pem"))
    parser.add_option("--readonly",action="store_true",help="mount vofs readonly",default=False)
    parser.add_option("--cache_nodes",action="store_true",default=False,help="cache dataNode properties, containerNodes are not cached")
    parser.add_option("--allow_other",action="store_true",default=False,help="Allow all users access to this mountpoint")

    parser.version = version
    (opt,args)=parser.parse_args()
    import sys
    if opt.version:
        parser.print_version()
        sys.exit(0)



    if opt.verbose:
        log_level = logging.INFO
    elif opt.debug:
        log_level = logging.DEBUG
    elif opt.warning:
        log_level = logging.WARNING
    else:
        log_level = logging.ERROR

    format = "%(asctime)s %(thread)d vos-"+str(version)+" %(module)s.%(funcName)s.%(lineno)d %(message)s"
    logging.basicConfig(level=log_level, format=format, filename=opt.log)

    logging.getLogger('vos').addHandler(logging.StreamHandler())

    logging.debug("Checking connection to VOSpace ")
    if not os.access(opt.certfile,os.F_OK):
        certfile=None
    else:
        certfile=opt.certfile
    conn=vos.Connection(certfile=certfile)
    logging.debug("Got a certificate, connections should work")

    root = opt.vospace
    mount = opt.mountpoint
    if opt.cache_dir is None:
        opt.cache_dir=os.path.normpath(os.path.join(os.getenv('HOME',default='.'),root.replace(":","_")))
    if not os.access(mount,os.F_OK):
        os.makedirs(mount)
    if platform=="darwin":
        fuse = FUSE(VOFS(root,opt.cache_dir,conn=conn,cache_limit=opt.cache_limit,cache_nodes=opt.cache_nodes), mount,
                    fsname=root,
                    volname=root,
                    defer_permissions=True,
                    daemon_timeout=DAEMON_TIMEOUT,
                    readonly=opt.readonly,
                    #auto_cache=True,
                    allow_other=opt.allow_other,
                    noapplexattr=True,
                    noappledouble=True,
                    foreground=opt.foreground)
    else:
        fuse = FUSE(VOFS(root,opt.cache_dir,conn=conn,cache_limit=opt.cache_limit,cache_nodes=opt.cache_nodes), mount,
                    fsname=root,
                    readonly=opt.readonly,
                    allow_other=opt.allow_other,
                    #auto_cache=True,
                    foreground=opt.foreground)

