Package Products :: Package ZenUtils :: Module FileCache |
[hide private]
[frames] | no frames]

# Source Code for Module Products.ZenUtils.FileCache

      1  ###########################################################################
      2  #
      3  # This program is part of Zenoss Core, an open source monitoring platform.
      4  # Copyright (C) 2010, Zenoss Inc.
      5  #
      6  # This program is free software; you can redistribute it and/or modify it
      7  # under the terms of the GNU General Public License version 2 or (at your
      8  # option) any later version as published by the Free Software Foundation.
      9  #
     10  # For complete information please visit: http://www.zenoss.com/oss/
     11  #
     12  ###########################################################################
     13  ## Based on Python recipe http://code.activestate.com/recipes/543263/ (r1)
     14  # by Brian O. Bush, Thu 24-Jan-2008 06:53 bushbo
     15
     16  import os, sys, pickle, base64, threading, glob
     17  import tempfile
     18
     19  _DEFAULT_NOT_SPECIFIED = object()
     20
     21  # This file cache is thread-safe
     22 -class FileCache(object):
     23 -    def __init__(self, path, protocol=-1):
     24          self.path = path # path assumed existing; check externally
     25          if not os.path.exists(self.path):
     26              os.makedirs(self.path)
     27          self.gen_key = lambda x: '%s.pickle' % base64.b64encode(x)
     28          self.lock = threading.Lock()
     29          self._pickleProtocol = protocol
     30 -    def _makeFileNameFromKey(self, key):
     31          return os.path.join(self.path, self.gen_key(key))
     32 -    def _allFileNames(self):
     33          return glob.glob(os.path.join(self.path,'*.pickle'))
     34 -    def get(self, key, default=_DEFAULT_NOT_SPECIFIED):
     35          retval = default
     36          fn = self._makeFileNameFromKey(key)
     37          with self.lock:
     38              try:
     39                  with open(fn, 'rb') as f:
     40                      retval = pickle.load(f)
     41              except IOError:
     42                  if default is _DEFAULT_NOT_SPECIFIED:
     43                      raise KeyError('no such key ' + key)
     44                  else:
     45                      return default
     46              return retval[1]
     47 -    def __getitem__(self, key):
     48          return self.get(key)
     49 -    def __setitem__(self, key, value):
     50          fn = self._makeFileNameFromKey(key)
     51          with self.lock:
     52              # use temp file to avoid partially written out pickles
     53              tempFd = None # file descriptor
     54              tempFn = None # file name
     55              try:
     56                  # dump to a temp file in pickle dir
     57                  tempFd, tempFn = tempfile.mkstemp(dir=os.path.dirname(fn))
     58                  # open a file object
     59                  with os.fdopen(tempFd, "wb") as tempF:
     60                      tempFd = None
     61                      pickle.dump((key, value), tempF, protocol=self._pickleProtocol)
     62                  # rename the temp file
     63                  # this is an atomic operation on most filesystems
     64                  os.rename(tempFn, fn)
     65              except Exception as ex:
     66                  if tempFd is not None:
     67                      os.close(tempFd)
     68                  raise ex
     69              finally:
     70                  if os.path.exists(tempFn):
     71                      try:
     72                          os.remove(tempFn)
     73                      except (OSError, IOError):
     74                          pass
     75 -    def __delitem__(self, key):
     76          fn = self._makeFileNameFromKey(key)
     77          with self.lock:
     78              try:
     79                  os.remove(fn)
     80              except (OSError, IOError):
     81                  raise KeyError('no such key ' + key)
     82 -    def clear(self):
     83          with self.lock:
     84              for fn in self._allFileNames():
     85                  try:
     86                      os.remove(fn)
     87                  except (OSError, IOError):
     88                      pass
     89 -    def items(self):
     90          with self.lock:
     91              return list(self.iteritems())
     92 -    def keys(self):
     93          with self.lock:
     94              return list(self.iterkeys())
     95 -    def values(self):
     96          with self.lock:
     97              return list(self.itervalues())
     98 -    def iterkeys(self):
     99          for fn in self._allFileNames():
    100              yield base64.b64decode(os.path.split(fn)[1][:-7])
    101 -    def itervalues(self):
    102          for k,v in self.iteritems():
    103              yield v
    104 -    def iteritems(self):
    105          for fn in self._allFileNames():
    106              try:
    107                  with open(fn,'rb') as f:
    108                      yield pickle.load(f)
    109              except IOError:
    110                  pass
    111 -    def __contains__(self, key):
    112          with self.lock:
    113              fn = self._makeFileNameFromKey(key)
    114              return os.path.exists(fn)
    115 -    def __len__(self):
    116          with self.lock:
    117              return len(self._allFileNames())
    118 -    def __bool__(self):
    119          with self.lock:
    120              return bool(self._allFileNames())
    121      __nonzero__ = __bool__
    122
    123  if __name__=='__main__':
    124 -    class Site:
    125 -        def __init__(self, name, hits=0):
    126              self.name = name
    127              self.hits = hits
    128 -        def __str__(self):
    129              return '%s, %d hits' % (self.name, self.hits)
    130      cache = FileCache('test')
    131      sites = [Site('cnn.com'), Site('kd7yhr.org', 1), Site('asdf.com', 3)]
    132      # We will use the site url as the key for our cache
    133      # Comment out the next two lines to test cache reading
    134      for site in sites:
    135          cache[site.name] = site
    136      testitemname = sites[-1].name
    137
    138      entry = cache.get(testitemname)
    139      if entry:
    140          print type(entry), entry
    141
    142      print cache.keys()
    143      import glob
    144      for fn in glob.glob('test/*'):
    145          print fn
    146
    147      print testitemname in cache
    148
    149      del cache[testitemname]
    150      print cache.keys()
    151      print testitemname in cache
    152
    153      cache.clear()
    154      print cache.keys()
    155