Package Products :: Package ZenUtils :: Module MultiPathIndex |
[hide private]
[frames] | no frames]

# Source Code for Module Products.ZenUtils.MultiPathIndex

      1  ##############################################################################
      2  #
      3  # Copyright (C) Zenoss, Inc. 2008, all rights reserved.
      4  #
      5  # This content is made available according to terms specified in
      6  # License.zenoss under the directory where your Zenoss product is installed.
      7  #
      8  ##############################################################################
      9
     10
     11  import time
     12  import logging
     13  LOG = logging.getLogger('ZenUtils.MultiPathIndex')
     14
     15  from Globals import DTMLFile
     16
     17  from ExtendedPathIndex import ExtendedPathIndex
     18  from Products.PluginIndexes.common import safe_callable
     19  from BTrees.OOBTree import OOSet
     20  from BTrees.IIBTree import IISet, intersection, union, multiunion
     21
     22 -def _isSequenceOfSequences(seq):
     23      if not seq:
     24          return False
     25      return (isinstance(seq, (tuple, list)) and
     26              all(isinstance(item, (tuple, list)) for item in seq))
     27
     28 -def _recursivePathSplit(seq):
     29      if isinstance(seq, (tuple, list)):
     30          return map(_recursivePathSplit, seq)
     31      if '/' in seq:
     32          return seq.split('/')
     33      else:
     34          return seq
     35
     36
     37 -class MultiPathIndex(ExtendedPathIndex):
     38      """
     39      A path index that is capable of indexing multiple paths per object.
     40      """
     41      meta_type = "MultiPathIndex"
     42
     43
     44 -    def search(self, path, default_level=0, depth=-1, navtree=0,
     45                                                               navtree_start=0):
     46          """
     47          path is either a string representing a
     48          relative URL or a part of a relative URL or
     49          a tuple (path,level).
     50
     51          level >= 0  starts searching at the given level
     52          level <  0  not implemented yet
     53          """
     54
     55          if isinstance(path, basestring):
     56              startlevel = default_level
     57          else:
     58              startlevel = int(path[1])
     59              path = path[0]
     60
     61          absolute_path = isinstance(path, basestring) and path.startswith('/')
     62          comps = filter(None, path.split('/'))
     63
     64          orig_comps = [''] + comps[:]
     65
     66          if depth > 0:
     67              raise ValueError, "Can't do depth searches anymore"
     68
     69          if not comps:
     70              comps = ['dmd']
     71              startlevel = 1
     72          elif comps[0] == 'zport':
     73              comps = comps[1:]
     74          elif comps[0] != 'dmd':
     75              raise ValueError, "Depth searches must start with 'dmd'"
     76          startlevel = len(comps)
     77          #startlevel = len(comps)-1 if len(comps) > 1 else 1
     78
     79          if len(comps) == 0:
     80              if depth == -1 and not navtree:
     81                  return IISet(self._unindex.keys())
     82
     83          # Make sure that we get depth = 1 if in navtree mode
     84          # unless specified otherwise
     85
     86          orig_depth = depth
     87          if depth == -1:
     88              depth = 0 or navtree
     89
     90          # Optimized navtree starting with absolute path
     91          if absolute_path and navtree and depth == 1 and default_level==0:
     92              set_list = []
     93              # Insert root element
     94              if navtree_start >= len(orig_comps):
     95                  navtree_start = 0
     96              # create a set of parent paths to search
     97              for i in range(len(orig_comps), navtree_start, -1):
     98                  parent_path = '/'.join(orig_comps[:i])
     99                  parent_path = parent_path and parent_path or '/'
    100                  try:
    101                      set_list.append(self._index_parents[parent_path])
    102                  except KeyError:
    103                      pass
    104              return multiunion(set_list)
    105          # Optimized breadcrumbs
    106          elif absolute_path and navtree and depth == 0 and default_level==0:
    107              item_list = IISet()
    108              # Insert root element
    109              if navtree_start >= len(orig_comps):
    110                  navtree_start = 0
    111              # create a set of parent paths to search
    112              for i in range(len(orig_comps), navtree_start, -1):
    113                  parent_path = '/'.join(orig_comps[:i])
    114                  parent_path = parent_path and parent_path or '/'
    115                  try:
    116                      item_list.insert(self._index_items[parent_path])
    117                  except KeyError:
    118                      pass
    119              return item_list
    120          # Specific object search
    121          elif absolute_path and orig_depth == 0 and default_level == 0:
    122              try:
    123                  return IISet([self._index_items[path]])
    124              except KeyError:
    125                  return IISet()
    126          # Single depth search
    127          elif absolute_path and orig_depth == 1 and default_level == 0:
    128              # only get objects contained in requested folder
    129              try:
    130                  return self._index_parents[path]
    131              except KeyError:
    132                  return IISet()
    133          # Sitemaps, relative paths, and depth queries
    134          elif startlevel >= 0:
    135
    136              pathset = None # Same as pathindex
    137              navset  = None # For collecting siblings along the way
    138              depthset = None # For limiting depth
    139
    140              if navtree and depth and \
    141                     self._index.has_key(None) and \
    142                     self._index[None].has_key(startlevel):
    143                  navset = self._index[None][startlevel]
    144              for level in range(startlevel, startlevel+len(comps)):
    145                  if level <= len(comps):
    146                      comp = "/".join(comps[:level])
    147                      if (not self._index.has_key(comp)
    148                          or not self._index[comp].has_key(level)):
    149                          # Navtree is inverse, keep going even for
    150                          # nonexisting paths
    151                          if navtree:
    152                              pathset = IISet()
    153                          else:
    154                              return IISet()
    155                      else:
    156                          return self._index[comp][level]
    157                      if navtree and depth and \
    158                             self._index.has_key(None) and \
    159                             self._index[None].has_key(level+depth):
    160                          navset  = union(navset, intersection(pathset,
    161                                                self._index[None][level+depth]))
    162                  if level-startlevel >= len(comps) or navtree:
    163                      if (self._index.has_key(None)
    164                          and self._index[None].has_key(level)):
    165                          depthset = union(depthset, intersection(pathset,
    166                                                      self._index[None][level]))
    167
    168              if navtree:
    169                  return union(depthset, navset) or IISet()
    170              elif depth:
    171                  return depthset or IISet()
    172              else:
    173                  return pathset or IISet()
    174
    175          else:
    176              results = IISet()
    177              for level in range(0,self._depth + 1):
    178                  ids = None
    179                  error = 0
    180                  for cn in range(0,len(comps)):
    181                      comp = comps[cn]
    182                      try:
    183                          ids = intersection(ids,self._index[comp][level+cn])
    184                      except KeyError:
    185                          error = 1
    186                  if error==0:
    187                      results = union(results,ids)
    188              return results
    189
    190
    191 -    def getIndexSourceNames(self):
    192          """ return names of indexed attributes """
    193          return (self.id, )
    194
    195 -    def index_object(self, docid, obj, threshold=100):
    196          """ hook for (Z)Catalog """
    197
    198          f = getattr(obj, self.id, None)
    199          if f is not None:
    200              if safe_callable(f):
    201                  try:
    202                      paths = f()
    203                  except AttributeError:
    204                      return 0
    205              else:
    206                  paths = f
    207          else:
    208              try:
    209                  paths = obj.getPhysicalPath()
    210              except AttributeError:
    211                  return 0
    212
    213          if not paths: return 0
    214          paths = _recursivePathSplit(paths)
    215          if not _isSequenceOfSequences(paths):
    216              paths = [paths]
    217
    218          if docid in self._unindex:
    219              unin = self._unindex[docid]
    220              # Migrate old versions of the index to use OOSet
    221              if isinstance(unin, set):
    222                  unin = self._unindex[docid] = OOSet(unin)
    223              for oldpath in list(unin):
    224                  if list(oldpath.split('/')) not in paths:
    225                      self.unindex_paths(docid, (oldpath,))
    226          else:
    227              self._unindex[docid] = OOSet()
    228              self._length.change(1)
    229
    230          self.index_paths(docid, paths)
    231
    232          return 1
    233
    234
    235 -    def index_paths(self, docid, paths):
    236          for path in paths:
    237              if isinstance(path, (list, tuple)):
    238                  path = '/'+ '/'.join(path[1:])
    239              comps = filter(None, path.split('/'))
    240              parent_path = '/' + '/'.join(comps[:-1])
    241
    242              for i in range(len(comps)):
    243                  comp = "/".join(comps[1:i+1])
    244                  if comp:
    245                      self.insertEntry(comp, docid, i)
    246
    247              # Add terminator
    248              self.insertEntry(None, docid, len(comps)-1, parent_path, path)
    249
    250              self._unindex.setdefault(docid, OOSet()).insert(path)
    251
    252
    253 -    def unindex_paths(self, docid, paths):
    254
    255          if not self._unindex.has_key(docid):
    256              return
    257
    258          def unindex(comp, level, docid=docid, parent_path=None,
    259                      object_path=None):
    260              try:
    261                  self._index[comp][level].remove(docid)
    262
    263                  if not self._index[comp][level]:
    264                      del self._index[comp][level]
    265
    266                  if not self._index[comp]:
    267                      del self._index[comp]
    268                  # Remove parent_path and object path elements
    269                  if parent_path is not None:
    270                      self._index_parents[parent_path].remove(docid)
    271                      if not self._index_parents[parent_path]:
    272                          del self._index_parents[parent_path]
    273                  if object_path is not None:
    274                      del self._index_items[object_path]
    275              except KeyError:
    276                  # Failure
    277                  pass
    278
    279          old = set(self._unindex.get(docid, ()))
    280          mkstr = lambda path:'/'.join(path) if isinstance(path, tuple) else path
    281          paths = map(mkstr, paths)
    282          toremove = set(paths) & old
    283          tokeep = old - toremove
    284          for path in toremove:
    285              if not path.startswith('/'):
    286                  path = '/'+path
    287              comps =  path.split('/')
    288              parent_path = '/'.join(comps[:-1])
    289
    290              for level in range(1, len(comps[2:])+1):
    291                  comp = "/".join(comps[2:level+2])
    292                  unindex(comp, level, docid, parent_path, path)
    293              # Remove the terminator
    294              level = len(comps[1:])
    295              comp = None
    296              unindex(comp, level-1, parent_path=parent_path, object_path=path)
    297
    298              self._unindex[docid].remove(path)
    299
    300          if tokeep:
    301              self.index_paths(docid, tokeep)
    302          else:
    303              # Cleared out all paths for the object
    304              self._length.change(-1)
    305              del self._unindex[docid]
    306
    307
    308 -    def unindex_object(self, docid):
    309          """ hook for (Z)Catalog """
    310          if not self._unindex.has_key(docid):
    311              return
    312          self.unindex_paths(docid, self._unindex[docid])
    313
    314      manage = manage_main = DTMLFile('dtml/manageMultiPathIndex', globals())
    315      manage_main._setName('manage_main')
    316
    317
    318  manage_addMultiPathIndexForm = DTMLFile('dtml/addMultiPathIndex', globals())
    319
    320 -def manage_addMultiPathIndex(self, id, REQUEST=None, RESPONSE=None,
    321                               URL3=None):
    322      """
    323      Add a MultiPathIndex.
    324      """
    325      return self.manage_addIndex(id, 'MultiPathIndex', extra=None,
    326                                  REQUEST=REQUEST, RESPONSE=RESPONSE,
    327                                  URL1=URL3)
    328