Package Products :: Package ZenUtils :: Module AutoGCObjectReader |
[hide private]
[frames] | no frames]

# Source Code for Module Products.ZenUtils.AutoGCObjectReader

      1  ##############################################################################
      2  #
      3  # Copyright (C) Zenoss, Inc. 2011, all rights reserved.
      4  #
      5  # This content is made available according to terms specified in
      6  # License.zenoss under the directory where your Zenoss product is installed.
      7  #
      8  ##############################################################################
      9
     10
     11  from contextlib import contextmanager
     12  from functools import wraps, partial
     13
     14  import logging
     15  log = logging.getLogger("zenUtils.AutoGCObjectReader")
     16  from ZODB.serialize import ObjectReader
     17
     18  __all__ = ["gc_cache_every", "gc_cache_every_decorator"]
     19
     20
     21 -class AutoGCObjectReader(ObjectReader):
     22      """
     23      ZODB has semipeculiar behavior wherein the object cache is only garbage
     24      collected at the transaction boundaries. If, within a transaction, one
     25      wishes to read a number of objects greater than the configured object cache
     26      size, the cache, and therefore memory, will simply grow with each read
     27      object.
     28
     29      This makes sense, when you think about it. Removing cached objects out from
     30      under existing references could have horrible effects, so it is only safe
     31      when beginning a new transaction (or aborting an existing one).
     32      Unfortunately, we have several cases where we need to read an enormous
     33      number of objects within a transaction, but don't need to write anything.
     34
     35      This class replaces a ZODB.Connection.Connection's existing ObjectReader.
     36      It will garbage-collect the cache every n objects. To enforce integrity, it
     37      will also abort any open transaction after cleaning the cache. It is safe
     38      ONLY when you are certain that the open transaction has not modified any
     39      objects.
     40      """
     41      _orig_reader = None
     42      _counter = 0
     43      _chunk_size = 1000
     44
     45 -    def __init__(self, orig_reader, chunk_size=1000):
     46          self._counter = 0
     47          self._orig_reader = orig_reader
     48          self._conn = orig_reader._conn
     49          self._cache = orig_reader._cache
     50          self._factory = orig_reader._factory
     51          self._chunk_size = chunk_size
     52          self._orig_cache_size = self._cache.cache_size
     53          self._cache.cache_size = chunk_size
     54
     55 -    def garbage_collect_cache(self):
     56          self._cache.incrgc()
     57          log.info("GC: reduced cache to %d/%d (total/active) objects", len(self._cache), self._cache.ringlen())
     58          self._counter = 0
     59
     60 -    def load_persistent(self, oid, klass):
     61          if self._counter >= self._chunk_size:
     62              self.garbage_collect_cache()
     63          ob = ObjectReader.load_persistent(self, oid, klass)
     64          self._counter += 1
     65          return ob
     66
     67 -    def get_original(self):
     68          self._cache.cache_size = self._orig_cache_size
     69          return self._orig_reader
     70
     71
     72 -def _normal_to_auto(connection, chunk_size=1000):
     73      """
     74      Replace the ObjectReader on a Connection with an AutoGCObjectReader.
     75      """
     76      if not isinstance(connection._reader, AutoGCObjectReader):
     77          connection._reader = AutoGCObjectReader(connection._reader,
     78                                              chunk_size=chunk_size)
     79
     80 -def _auto_to_normal(connection):
     81      """
     82      Uninstall an AutoGCObjectReader from a Connection, replacing it with the
     83      original ObjectReader.
     84      """
     85      if isinstance(connection._reader, AutoGCObjectReader):
     86          try:
     87              connection._reader.garbage_collect_cache()
     88          finally:
     89              connection._reader = connection._reader.get_original()
     90
     91
     92  @contextmanager
     93 -def gc_cache_every(chunk_size=1000, db=None):
     94      """
     95      Temporarily replace the ObjectReaders on the current database with
     96      AutoGCObjectReaders.
     97
     98      WARNING: Will abort any open transaction!
     99      """
    100      db._connectionMap(
    101          partial(_normal_to_auto, chunk_size=chunk_size))
    102      try:
    103          yield
    104      finally:
    105          db._connectionMap(_auto_to_normal)
    106
    107
    108 -def gc_cache_every_decorator(chunk_size=1000, db=None):
    109      """
    110      Temporarily replace the ObjectReaders on the current database with
    111      AutoGCObjectReaders.
    112
    113      WARNING: Will abort any open transaction!
    114      """
    115      def decorator(f):
    116          @wraps(f)
    117          def inner(*args, **kwargs):
    118              with gc_cache_every(chunk_size, db):
    119                  return f(*args, **kwargs)
    120          return inner
    121      return decorator
    122