Package Products :: Package ZenUtils :: Module FileCache |
[hide private]
[frames] | no frames]

# Source Code for Module Products.ZenUtils.FileCache

      1  ##############################################################################
      2  #
      3  # Copyright (C) Zenoss, Inc. 2010, all rights reserved.
      4  #
      5  # This content is made available according to terms specified in
      6  # License.zenoss under the directory where your Zenoss product is installed.
      7  #
      8  ##############################################################################
      9
     10
     11  ## Based on Python recipe http://code.activestate.com/recipes/543263/ (r1)
     12  # by Brian O. Bush, Thu 24-Jan-2008 06:53 bushbo
     13
     14  import os, sys, pickle, base64, threading, glob
     15  import tempfile
     16
     17  _DEFAULT_NOT_SPECIFIED = object()
     18
     19  # This file cache is thread-safe
     20 -class FileCache(object):
     21 -    def __init__(self, path, protocol=-1):
     22          self.path = path # path assumed existing; check externally
     23          if not os.path.exists(self.path):
     24              os.makedirs(self.path)
     25          self.gen_key = lambda x: '%s.pickle' % base64.b64encode(x)
     26          self.lock = threading.Lock()
     27          self._pickleProtocol = protocol
     28 -    def _makeFileNameFromKey(self, key):
     29          return os.path.join(self.path, self.gen_key(key))
     30 -    def _allFileNames(self):
     31          return glob.glob(os.path.join(self.path,'*.pickle'))
     32 -    def get(self, key, default=_DEFAULT_NOT_SPECIFIED):
     33          retval = default
     34          fn = self._makeFileNameFromKey(key)
     35          with self.lock:
     36              try:
     37                  with open(fn, 'rb') as f:
     38                      retval = pickle.load(f)
     39              except IOError:
     40                  if default is _DEFAULT_NOT_SPECIFIED:
     41                      raise KeyError('no such key ' + key)
     42                  else:
     43                      return default
     44              return retval[1]
     45 -    def __getitem__(self, key):
     46          return self.get(key)
     47 -    def __setitem__(self, key, value):
     48          fn = self._makeFileNameFromKey(key)
     49          with self.lock:
     50              # use temp file to avoid partially written out pickles
     51              tempFd = None # file descriptor
     52              tempFn = None # file name
     53              try:
     54                  # dump to a temp file in pickle dir
     55                  tempFd, tempFn = tempfile.mkstemp(dir=os.path.dirname(fn))
     56                  # open a file object
     57                  with os.fdopen(tempFd, "wb") as tempF:
     58                      tempFd = None
     59                      pickle.dump((key, value), tempF, protocol=self._pickleProtocol)
     60                  # rename the temp file
     61                  # this is an atomic operation on most filesystems
     62                  os.rename(tempFn, fn)
     63              except Exception as ex:
     64                  if tempFd is not None:
     65                      os.close(tempFd)
     66                  raise ex
     67              finally:
     68                  if os.path.exists(tempFn):
     69                      try:
     70                          os.remove(tempFn)
     71                      except (OSError, IOError):
     72                          pass
     73 -    def __delitem__(self, key):
     74          fn = self._makeFileNameFromKey(key)
     75          with self.lock:
     76              try:
     77                  os.remove(fn)
     78              except (OSError, IOError):
     79                  raise KeyError('no such key ' + key)
     80 -    def clear(self):
     81          with self.lock:
     82              for fn in self._allFileNames():
     83                  try:
     84                      os.remove(fn)
     85                  except (OSError, IOError):
     86                      pass
     87 -    def items(self):
     88          with self.lock:
     89              return list(self.iteritems())
     90 -    def keys(self):
     91          with self.lock:
     92              return list(self.iterkeys())
     93 -    def values(self):
     94          with self.lock:
     95              return list(self.itervalues())
     96 -    def iterkeys(self):
     97          for fn in self._allFileNames():
     98              yield base64.b64decode(os.path.split(fn)[1][:-7])
     99 -    def itervalues(self):
    100          for k,v in self.iteritems():
    101              yield v
    102 -    def iteritems(self):
    103          for fn in self._allFileNames():
    104              try:
    105                  with open(fn,'rb') as f:
    106                      yield pickle.load(f)
    107              except IOError:
    108                  pass
    109 -    def __contains__(self, key):
    110          with self.lock:
    111              fn = self._makeFileNameFromKey(key)
    112              return os.path.exists(fn)
    113 -    def __len__(self):
    114          with self.lock:
    115              return len(self._allFileNames())
    116 -    def __bool__(self):
    117          with self.lock:
    118              return bool(self._allFileNames())
    119      __nonzero__ = __bool__
    120
    121  if __name__=='__main__':
    122 -    class Site:
    123 -        def __init__(self, name, hits=0):
    124              self.name = name
    125              self.hits = hits
    126 -        def __str__(self):
    127              return '%s, %d hits' % (self.name, self.hits)
    128      cache = FileCache('test')
    129      sites = [Site('cnn.com'), Site('kd7yhr.org', 1), Site('asdf.com', 3)]
    130      # We will use the site url as the key for our cache
    131      # Comment out the next two lines to test cache reading
    132      for site in sites:
    133          cache[site.name] = site
    134      testitemname = sites[-1].name
    135
    136      entry = cache.get(testitemname)
    137      if entry:
    138          print type(entry), entry
    139
    140      print cache.keys()
    141      import glob
    142      for fn in glob.glob('test/*'):
    143          print fn
    144
    145      print testitemname in cache
    146
    147      del cache[testitemname]
    148      print cache.keys()
    149      print testitemname in cache
    150
    151      cache.clear()
    152      print cache.keys()
    153