Source Code for Module Products.ZenUtils.FileCache

  1  ############################################################################## 
  2  #  
  3  # Copyright (C) Zenoss, Inc. 2010, all rights reserved. 
  4  #  
  5  # This content is made available according to terms specified in 
  6  # License.zenoss under the directory where your Zenoss product is installed. 
  7  #  
  8  ############################################################################## 
  9   
 10   
 11  ## Based on Python recipe http://code.activestate.com/recipes/543263/ (r1) 
 12  # by Brian O. Bush, Thu 24-Jan-2008 06:53 bushbo 
 13   
 14  import os, sys, pickle, base64, threading, glob 
 15  import tempfile 
 16   
 17  _DEFAULT_NOT_SPECIFIED = object() 
 18   
 19  # This file cache is thread-safe 
 20 -class FileCache(object): 
 21 -    def __init__(self, path, protocol=-1):  
 22          self.path = path # path assumed existing; check externally 
 23          if not os.path.exists(self.path):  
 24              os.makedirs(self.path)         
 25          self.gen_key = lambda x: '%s.pickle' % base64.b64encode(x) 
 26          self.lock = threading.Lock() 
 27          self._pickleProtocol = protocol 
 28 -    def _makeFileNameFromKey(self, key): 
 29          return os.path.join(self.path, self.gen_key(key)) 
 30 -    def _allFileNames(self): 
 31          return glob.glob(os.path.join(self.path,'*.pickle')) 
 32 -    def get(self, key, default=_DEFAULT_NOT_SPECIFIED): 
 33          retval = default 
 34          fn = self._makeFileNameFromKey(key) 
 35          with self.lock: 
 36              try: 
 37                  with open(fn, 'rb') as f: 
 38                      retval = pickle.load(f) 
 39              except IOError: 
 40                  if default is _DEFAULT_NOT_SPECIFIED: 
 41                      raise KeyError('no such key ' + key) 
 42                  else: 
 43                      return default 
 44              return retval[1] 
 45 -    def __getitem__(self, key): 
 46          return self.get(key) 
 47 -    def __setitem__(self, key, value): 
 48          fn = self._makeFileNameFromKey(key) 
 49          with self.lock: 
 50              # use temp file to avoid partially written out pickles 
 51              tempFd = None # file descriptor 
 52              tempFn = None # file name 
 53              try: 
 54                  # dump to a temp file in pickle dir 
 55                  tempFd, tempFn = tempfile.mkstemp(dir=os.path.dirname(fn)) 
 56                  # open a file object 
 57                  with os.fdopen(tempFd, "wb") as tempF: 
 58                      tempFd = None 
 59                      pickle.dump((key, value), tempF, protocol=self._pickleProtocol) 
 60                  # rename the temp file 
 61                  # this is an atomic operation on most filesystems 
 62                  os.rename(tempFn, fn) 
 63              except Exception as ex: 
 64                  if tempFd is not None: 
 65                      os.close(tempFd) 
 66                  raise ex 
 67              finally: 
 68                  if os.path.exists(tempFn): 
 69                      try: 
 70                          os.remove(tempFn) 
 71                      except (OSError, IOError): 
 72                          pass 
 73 -    def __delitem__(self, key): 
 74          fn = self._makeFileNameFromKey(key) 
 75          with self.lock: 
 76              try: 
 77                  os.remove(fn) 
 78              except (OSError, IOError): 
 79                  raise KeyError('no such key ' + key) 
 80 -    def clear(self): 
 81          with self.lock: 
 82              for fn in self._allFileNames(): 
 83                  try: 
 84                      os.remove(fn) 
 85                  except (OSError, IOError): 
 86                      pass 
 87 -    def items(self): 
 88          with self.lock: 
 89              return list(self.iteritems()) 
 90 -    def keys(self): 
 91          with self.lock: 
 92              return list(self.iterkeys()) 
 93 -    def values(self): 
 94          with self.lock: 
 95              return list(self.itervalues()) 
 96 -    def iterkeys(self): 
 97          for fn in self._allFileNames(): 
 98              yield base64.b64decode(os.path.split(fn)[1][:-7]) 
 99 -    def itervalues(self): 
100          for k,v in self.iteritems(): 
101              yield v 
102 -    def iteritems(self): 
103          for fn in self._allFileNames(): 
104              try: 
105                  with open(fn,'rb') as f: 
106                      yield pickle.load(f) 
107              except IOError: 
108                  pass 
109 -    def __contains__(self, key): 
110          with self.lock: 
111              fn = self._makeFileNameFromKey(key) 
112              return os.path.exists(fn) 
113 -    def __len__(self): 
114          with self.lock: 
115              return len(self._allFileNames()) 
116 -    def __bool__(self): 
117          with self.lock: 
118              return bool(self._allFileNames()) 
119      __nonzero__ = __bool__ 
120   
121  if __name__=='__main__': 
122 -    class Site: 
123 -        def __init__(self, name, hits=0): 
124              self.name = name 
125              self.hits = hits 
126 -        def __str__(self): 
127              return '%s, %d hits' % (self.name, self.hits) 
128      cache = FileCache('test') 
129      sites = [Site('cnn.com'), Site('kd7yhr.org', 1), Site('asdf.com', 3)] 
130      # We will use the site url as the key for our cache 
131      # Comment out the next two lines to test cache reading 
132      for site in sites:     
133          cache[site.name] = site 
134      testitemname = sites[-1].name 
135   
136      entry = cache.get(testitemname) 
137      if entry:  
138          print type(entry), entry 
139   
140      print cache.keys() 
141      import glob 
142      for fn in glob.glob('test/*'): 
143          print fn 
144   
145      print testitemname in cache 
146   
147      del cache[testitemname] 
148      print cache.keys() 
149      print testitemname in cache 
150   
151      cache.clear() 
152      print cache.keys() 
153