Source Code for Module Products.ZenUtils.AutoGCObjectReader

  1  ############################################################################## 
  2  #  
  3  # Copyright (C) Zenoss, Inc. 2011, all rights reserved. 
  4  #  
  5  # This content is made available according to terms specified in 
  6  # License.zenoss under the directory where your Zenoss product is installed. 
  7  #  
  8  ############################################################################## 
  9   
 10   
 11  from contextlib import contextmanager 
 12  from functools import wraps, partial 
 13   
 14  import logging 
 15  log = logging.getLogger("zenUtils.AutoGCObjectReader") 
 16  from ZODB.serialize import ObjectReader 
 17   
 18  __all__ = ["gc_cache_every", "gc_cache_every_decorator"] 
 19   
 20   
 21 -class AutoGCObjectReader(ObjectReader): 
 22      """ 
 23      ZODB has semipeculiar behavior wherein the object cache is only garbage 
 24      collected at the transaction boundaries. If, within a transaction, one 
 25      wishes to read a number of objects greater than the configured object cache 
 26      size, the cache, and therefore memory, will simply grow with each read 
 27      object. 
 28   
 29      This makes sense, when you think about it. Removing cached objects out from 
 30      under existing references could have horrible effects, so it is only safe 
 31      when beginning a new transaction (or aborting an existing one). 
 32      Unfortunately, we have several cases where we need to read an enormous 
 33      number of objects within a transaction, but don't need to write anything. 
 34   
 35      This class replaces a ZODB.Connection.Connection's existing ObjectReader. 
 36      It will garbage-collect the cache every n objects. To enforce integrity, it 
 37      will also abort any open transaction after cleaning the cache. It is safe 
 38      ONLY when you are certain that the open transaction has not modified any 
 39      objects. 
 40      """ 
 41      _orig_reader = None 
 42      _counter = 0 
 43      _chunk_size = 1000 
 44   
 45 -    def __init__(self, orig_reader, chunk_size=1000): 
 46          self._counter = 0 
 47          self._orig_reader = orig_reader 
 48          self._conn = orig_reader._conn 
 49          self._cache = orig_reader._cache 
 50          self._factory = orig_reader._factory 
 51          self._chunk_size = chunk_size 
 52          self._orig_cache_size = self._cache.cache_size 
 53          self._cache.cache_size = chunk_size 
 54   
 55 -    def garbage_collect_cache(self): 
 56          self._cache.incrgc() 
 57          log.info("GC: reduced cache to %d/%d (total/active) objects", len(self._cache), self._cache.ringlen()) 
 58          self._counter = 0 
 59   
 60 -    def load_persistent(self, oid, klass): 
 61          if self._counter >= self._chunk_size: 
 62              self.garbage_collect_cache() 
 63          ob = ObjectReader.load_persistent(self, oid, klass) 
 64          self._counter += 1 
 65          return ob 
 66   
 67 -    def get_original(self): 
 68          self._cache.cache_size = self._orig_cache_size 
 69          return self._orig_reader 
 70   
 71   
 72 -def _normal_to_auto(connection, chunk_size=1000): 
 73      """ 
 74      Replace the ObjectReader on a Connection with an AutoGCObjectReader. 
 75      """ 
 76      if not isinstance(connection._reader, AutoGCObjectReader): 
 77          connection._reader = AutoGCObjectReader(connection._reader, 
 78                                              chunk_size=chunk_size) 
 79   
 80 -def _auto_to_normal(connection): 
 81      """ 
 82      Uninstall an AutoGCObjectReader from a Connection, replacing it with the 
 83      original ObjectReader. 
 84      """ 
 85      if isinstance(connection._reader, AutoGCObjectReader): 
 86          try: 
 87              connection._reader.garbage_collect_cache() 
 88          finally: 
 89              connection._reader = connection._reader.get_original() 
 90   
 91   
 92  @contextmanager 
 93 -def gc_cache_every(chunk_size=1000, db=None): 
 94      """ 
 95      Temporarily replace the ObjectReaders on the current database with 
 96      AutoGCObjectReaders. 
 97   
 98      WARNING: Will abort any open transaction! 
 99      """ 
100      db._connectionMap( 
101          partial(_normal_to_auto, chunk_size=chunk_size)) 
102      try: 
103          yield 
104      finally: 
105          db._connectionMap(_auto_to_normal) 
106   
107   
108 -def gc_cache_every_decorator(chunk_size=1000, db=None): 
109      """ 
110      Temporarily replace the ObjectReaders on the current database with 
111      AutoGCObjectReaders. 
112   
113      WARNING: Will abort any open transaction! 
114      """ 
115      def decorator(f): 
116          @wraps(f) 
117          def inner(*args, **kwargs): 
118              with gc_cache_every(chunk_size, db): 
119                  return f(*args, **kwargs) 
120          return inner 
121      return decorator 
122