Package Products :: Package ZenUtils :: Module MultiPathIndex |
[hide private]
[frames] | no frames]

# Source Code for Module Products.ZenUtils.MultiPathIndex

      1  ###########################################################################
      2  #
      3  # This program is part of Zenoss Core, an open source monitoring platform.
      4  # Copyright (C) 2008, Zenoss Inc.
      5  #
      6  # This program is free software; you can redistribute it and/or modify it
      7  # under the terms of the GNU General Public License version 2 as published by
      8  # the Free Software Foundation.
      9  #
     10  # For complete information please visit: http://www.zenoss.com/oss/
     11  #
     12  ###########################################################################
     13  import time
     14  import logging
     15  LOG = logging.getLogger('ZenUtils.MultiPathIndex')
     16
     17  from Globals import DTMLFile
     18
     19  from ExtendedPathIndex import ExtendedPathIndex
     20  from Products.PluginIndexes.common import safe_callable
     21  from BTrees.OOBTree import OOSet
     22  from BTrees.IIBTree import IISet, intersection, union, multiunion
     23
     24 -def _isSequenceOfSequences(seq):
     25      if not seq:
     26          return False
     27      return (isinstance(seq, (tuple, list)) and
     28              all(isinstance(item, (tuple, list)) for item in seq))
     29
     30 -def _recursivePathSplit(seq):
     31      if isinstance(seq, (tuple, list)):
     32          return map(_recursivePathSplit, seq)
     33      if '/' in seq:
     34          return seq.split('/')
     35      else:
     36          return seq
     37
     38
     39 -class MultiPathIndex(ExtendedPathIndex):
     40      """
     41      A path index that is capable of indexing multiple paths per object.
     42      """
     43      meta_type = "MultiPathIndex"
     44
     45
     46 -    def search(self, path, default_level=0, depth=-1, navtree=0,
     47                                                               navtree_start=0):
     48          """
     49          path is either a string representing a
     50          relative URL or a part of a relative URL or
     51          a tuple (path,level).
     52
     53          level >= 0  starts searching at the given level
     54          level <  0  not implemented yet
     55          """
     56
     57          if isinstance(path, basestring):
     58              startlevel = default_level
     59          else:
     60              startlevel = int(path[1])
     61              path = path[0]
     62
     63          absolute_path = isinstance(path, basestring) and path.startswith('/')
     64          comps = filter(None, path.split('/'))
     65
     66          orig_comps = [''] + comps[:]
     67
     68          if depth > 0:
     69              raise ValueError, "Can't do depth searches anymore"
     70
     71          if not comps:
     72              comps = ['dmd']
     73              startlevel = 1
     74          elif comps[0] == 'zport':
     75              comps = comps[1:]
     76          elif comps[0] != 'dmd':
     77              raise ValueError, "Depth searches must start with 'dmd'"
     78          startlevel = len(comps)
     79          #startlevel = len(comps)-1 if len(comps) > 1 else 1
     80
     81          if len(comps) == 0:
     82              if depth == -1 and not navtree:
     83                  return IISet(self._unindex.keys())
     84
     85          # Make sure that we get depth = 1 if in navtree mode
     86          # unless specified otherwise
     87
     88          orig_depth = depth
     89          if depth == -1:
     90              depth = 0 or navtree
     91
     92          # Optimized navtree starting with absolute path
     93          if absolute_path and navtree and depth == 1 and default_level==0:
     94              set_list = []
     95              # Insert root element
     96              if navtree_start >= len(orig_comps):
     97                  navtree_start = 0
     98              # create a set of parent paths to search
     99              for i in range(len(orig_comps), navtree_start, -1):
    100                  parent_path = '/'.join(orig_comps[:i])
    101                  parent_path = parent_path and parent_path or '/'
    102                  try:
    103                      set_list.append(self._index_parents[parent_path])
    104                  except KeyError:
    105                      pass
    106              return multiunion(set_list)
    107          # Optimized breadcrumbs
    108          elif absolute_path and navtree and depth == 0 and default_level==0:
    109              item_list = IISet()
    110              # Insert root element
    111              if navtree_start >= len(orig_comps):
    112                  navtree_start = 0
    113              # create a set of parent paths to search
    114              for i in range(len(orig_comps), navtree_start, -1):
    115                  parent_path = '/'.join(orig_comps[:i])
    116                  parent_path = parent_path and parent_path or '/'
    117                  try:
    118                      item_list.insert(self._index_items[parent_path])
    119                  except KeyError:
    120                      pass
    121              return item_list
    122          # Specific object search
    123          elif absolute_path and orig_depth == 0 and default_level == 0:
    124              try:
    125                  return IISet([self._index_items[path]])
    126              except KeyError:
    127                  return IISet()
    128          # Single depth search
    129          elif absolute_path and orig_depth == 1 and default_level == 0:
    130              # only get objects contained in requested folder
    131              try:
    132                  return self._index_parents[path]
    133              except KeyError:
    134                  return IISet()
    135          # Sitemaps, relative paths, and depth queries
    136          elif startlevel >= 0:
    137
    138              pathset = None # Same as pathindex
    139              navset  = None # For collecting siblings along the way
    140              depthset = None # For limiting depth
    141
    142              if navtree and depth and \
    143                     self._index.has_key(None) and \
    144                     self._index[None].has_key(startlevel):
    145                  navset = self._index[None][startlevel]
    146              for level in range(startlevel, startlevel+len(comps)):
    147                  if level <= len(comps):
    148                      comp = "/".join(comps[:level])
    149                      if (not self._index.has_key(comp)
    150                          or not self._index[comp].has_key(level)):
    151                          # Navtree is inverse, keep going even for
    152                          # nonexisting paths
    153                          if navtree:
    154                              pathset = IISet()
    155                          else:
    156                              return IISet()
    157                      else:
    158                          return self._index[comp][level]
    159                      if navtree and depth and \
    160                             self._index.has_key(None) and \
    161                             self._index[None].has_key(level+depth):
    162                          navset  = union(navset, intersection(pathset,
    163                                                self._index[None][level+depth]))
    164                  if level-startlevel >= len(comps) or navtree:
    165                      if (self._index.has_key(None)
    166                          and self._index[None].has_key(level)):
    167                          depthset = union(depthset, intersection(pathset,
    168                                                      self._index[None][level]))
    169
    170              if navtree:
    171                  return union(depthset, navset) or IISet()
    172              elif depth:
    173                  return depthset or IISet()
    174              else:
    175                  return pathset or IISet()
    176
    177          else:
    178              results = IISet()
    179              for level in range(0,self._depth + 1):
    180                  ids = None
    181                  error = 0
    182                  for cn in range(0,len(comps)):
    183                      comp = comps[cn]
    184                      try:
    185                          ids = intersection(ids,self._index[comp][level+cn])
    186                      except KeyError:
    187                          error = 1
    188                  if error==0:
    189                      results = union(results,ids)
    190              return results
    191
    192
    193 -    def getIndexSourceNames(self):
    194          """ return names of indexed attributes """
    195          return (self.id, )
    196
    197 -    def index_object(self, docid, obj, threshold=100):
    198          """ hook for (Z)Catalog """
    199
    200          f = getattr(obj, self.id, None)
    201          if f is not None:
    202              if safe_callable(f):
    203                  try:
    204                      paths = f()
    205                  except AttributeError:
    206                      return 0
    207              else:
    208                  paths = f
    209          else:
    210              try:
    211                  paths = obj.getPhysicalPath()
    212              except AttributeError:
    213                  return 0
    214
    215          if not paths: return 0
    216          paths = _recursivePathSplit(paths)
    217          if not _isSequenceOfSequences(paths):
    218              paths = [paths]
    219
    220          if docid in self._unindex:
    221              unin = self._unindex[docid]
    222              # Migrate old versions of the index to use OOSet
    223              if isinstance(unin, set):
    224                  unin = self._unindex[docid] = OOSet(unin)
    225              for oldpath in list(unin):
    226                  if list(oldpath.split('/')) not in paths:
    227                      self.unindex_paths(docid, (oldpath,))
    228          else:
    229              self._unindex[docid] = OOSet()
    230              self._length.change(1)
    231
    232          self.index_paths(docid, paths)
    233
    234          return 1
    235
    236
    237 -    def index_paths(self, docid, paths):
    238          for path in paths:
    239              if isinstance(path, (list, tuple)):
    240                  path = '/'+ '/'.join(path[1:])
    241              comps = filter(None, path.split('/'))
    242              parent_path = '/' + '/'.join(comps[:-1])
    243
    244              for i in range(len(comps)):
    245                  comp = "/".join(comps[1:i+1])
    246                  if comp:
    247                      self.insertEntry(comp, docid, i)
    248
    249              # Add terminator
    250              self.insertEntry(None, docid, len(comps)-1, parent_path, path)
    251
    252              self._unindex.setdefault(docid, OOSet()).insert(path)
    253
    254
    255 -    def unindex_paths(self, docid, paths):
    256
    257          if not self._unindex.has_key(docid):
    258              return
    259
    260          def unindex(comp, level, docid=docid, parent_path=None,
    261                      object_path=None):
    262              try:
    263                  self._index[comp][level].remove(docid)
    264
    265                  if not self._index[comp][level]:
    266                      del self._index[comp][level]
    267
    268                  if not self._index[comp]:
    269                      del self._index[comp]
    270                  # Remove parent_path and object path elements
    271                  if parent_path is not None:
    272                      self._index_parents[parent_path].remove(docid)
    273                      if not self._index_parents[parent_path]:
    274                          del self._index_parents[parent_path]
    275                  if object_path is not None:
    276                      del self._index_items[object_path]
    277              except KeyError:
    278                  # Failure
    279                  pass
    280
    281          old = set(self._unindex.get(docid, ()))
    282          mkstr = lambda path:'/'.join(path) if isinstance(path, tuple) else path
    283          paths = map(mkstr, paths)
    284          toremove = set(paths) & old
    285          tokeep = old - toremove
    286          for path in toremove:
    287              if not path.startswith('/'):
    288                  path = '/'+path
    289              comps =  path.split('/')
    290              parent_path = '/'.join(comps[:-1])
    291
    292              for level in range(1, len(comps[2:])+1):
    293                  comp = "/".join(comps[2:level+2])
    294                  unindex(comp, level, docid, parent_path, path)
    295              # Remove the terminator
    296              level = len(comps[1:])
    297              comp = None
    298              unindex(comp, level-1, parent_path=parent_path, object_path=path)
    299
    300              self._unindex[docid].remove(path)
    301
    302          if tokeep:
    303              self.index_paths(docid, tokeep)
    304          else:
    305              # Cleared out all paths for the object
    306              self._length.change(-1)
    307              del self._unindex[docid]
    308
    309
    310 -    def unindex_object(self, docid):
    311          """ hook for (Z)Catalog """
    312          if not self._unindex.has_key(docid):
    313              return
    314          self.unindex_paths(docid, self._unindex[docid])
    315
    316      manage = manage_main = DTMLFile('dtml/manageMultiPathIndex', globals())
    317      manage_main._setName('manage_main')
    318
    319
    320  manage_addMultiPathIndexForm = DTMLFile('dtml/addMultiPathIndex', globals())
    321
    322 -def manage_addMultiPathIndex(self, id, REQUEST=None, RESPONSE=None,
    323                               URL3=None):
    324      """
    325      Add a MultiPathIndex.
    326      """
    327      return self.manage_addIndex(id, 'MultiPathIndex', extra=None,
    328                                  REQUEST=REQUEST, RESPONSE=RESPONSE,
    329                                  URL1=URL3)
    330