Package Products :: Package ZenUtils :: Module ExtendedPathIndex |
[hide private]
[frames] | no frames]

# Source Code for Module Products.ZenUtils.ExtendedPathIndex

      1  # Copyright (c) 2004 Zope Corporation and Plone Solutions
      2
      3  # ZPL 2.1 license
      4
      5  import logging
      6
      7  from Globals import DTMLFile
      8  from BTrees.IIBTree import IISet, intersection, union, multiunion
      9  from BTrees.OOBTree import OOBTree
     10  from BTrees.OIBTree import OIBTree
     11
     12  from Products.PluginIndexes.common.util import parseIndexRequest
     13  from Products.PluginIndexes.common import safe_callable
     14  from Products.PluginIndexes.PathIndex.PathIndex import PathIndex
     15
     16  _marker = []
     17  logger = logging.getLogger('ExtendedPathIndex')
     18
     19 -class ExtendedPathIndex(PathIndex):
     20      """ A path index stores all path components of the physical
     21      path of an object:
     22
     23      Internal datastructure (regular pathindex):
     24
     25      - a physical path of an object is split into its components
     26
     27      - every component is kept as a  key of a OOBTree in self._indexes
     28
     29      - the value is a mapping 'level of the path component' to
     30        'all docids with this path component on this level'
     31
     32      In addition
     33
     34      - there is a terminator (None) signifying the last component in the path
     35
     36      """
     37
     38      meta_type = "ExtendedPathIndex"
     39
     40      manage_options= (
     41          {'label': 'Settings',
     42           'action': 'manage_main',
     43           'help': ('ExtendedPathIndex','ExtendedPathIndex_Settings.stx')},
     44      )
     45
     46      query_options = ("query", "level", "operator", "depth", "navtree",
     47                                                                "navtree_start")
     48
     49 -    def __init__(self, id, extra=None, caller=None):
     50          """ ExtendedPathIndex supports indexed_attrs """
     51          PathIndex.__init__(self, id, caller)
     52
     53          def get(o, k, default):
     54              if isinstance(o, dict):
     55                  return o.get(k, default)
     56              else:
     57                  return getattr(o, k, default)
     58
     59          attrs = get(extra, 'indexed_attrs', None)
     60          if attrs is None:
     61              return
     62          if isinstance(attrs, str):
     63              attrs = attrs.split(',')
     64          attrs = filter(None, [a.strip() for a in attrs])
     65
     66          if attrs:
     67              # We only index the first attribute so snip off the rest
     68              self.indexed_attrs = tuple(attrs[:1])
     69
     70 -    def clear(self):
     71          PathIndex.clear(self)
     72          self._index_parents = OOBTree()
     73          self._index_items = OIBTree()
     74
     75 -    def insertEntry(self, comp, id, level, parent_path=None, object_path=None):
     76          """Insert an entry.
     77
     78             parent_path is the path of the parent object
     79
     80             path is the object path, it is assumed to be unique, i.e. there
     81             is a one to one mapping between physical paths and docids.  This
     82             will be large, and is only used for breadcrumbs.
     83
     84             id is the docid
     85          """
     86
     87          PathIndex.insertEntry(self, comp, id, level)
     88
     89          if parent_path is not None:
     90              if not self._index_parents.has_key(parent_path):
     91                  self._index_parents[parent_path] = IISet()
     92
     93              self._index_parents[parent_path].insert(id)
     94
     95          # We make the assumption that a full path corresponds one and only
     96          # one object.
     97
     98          if object_path is not None:
     99              self._index_items[object_path] = id
    100
    101 -    def index_object(self, docid, obj ,threshold=100):
    102          """ hook for (Z)Catalog """
    103
    104          # PathIndex first checks for an attribute matching its id and
    105          # falls back to getPhysicalPath only when failing to get one.
    106          # The presence of 'indexed_attrs' overrides this behavior and
    107          # causes indexing of the custom attribute.
    108
    109          attrs = getattr(self, 'indexed_attrs', None)
    110          if attrs:
    111              index = attrs[0]
    112          else:
    113              index = self.id
    114
    115          f = getattr(obj, index, None)
    116          if f is not None:
    117              if safe_callable(f):
    118                  try:
    119                      path = f()
    120                  except AttributeError:
    121                      return 0
    122              else:
    123                  path = f
    124
    125              if not isinstance(path, (str, tuple)):
    126                  raise TypeError('path value must be string or tuple '
    127                                  'of strings: (%r, %s)' % (index, repr(path)))
    128          else:
    129              try:
    130                  path = obj.getPhysicalPath()
    131              except AttributeError:
    132                  return 0
    133
    134          if isinstance(path, (list, tuple)):
    135              path = '/'+ '/'.join(path[1:])
    136          comps = filter(None, path.split('/'))
    137          parent_path = '/' + '/'.join(comps[:-1])
    138
    139          # Make sure we reindex properly when path change
    140          if self._unindex.has_key(docid) and self._unindex.get(docid) != path:
    141              self.unindex_object(docid)
    142
    143          if not self._unindex.has_key(docid):
    144              self._length.change(1)
    145
    146          for i in range(len(comps)):
    147              self.insertEntry(comps[i], docid, i)
    148
    149          # Add terminator
    150          self.insertEntry(None, docid, len(comps)-1, parent_path, path)
    151
    152          self._unindex[docid] = path
    153          return 1
    154
    155 -    def unindex_object(self, docid):
    156          """ hook for (Z)Catalog """
    157
    158          if not self._unindex.has_key(docid):
    159              logger.log(logging.INFO,
    160                         'Attempt to unindex nonexistent object with id '
    161                         '%s' % docid)
    162              return
    163
    164          # There is an assumption that paths start with /
    165          path = self._unindex[docid]
    166          if not path.startswith('/'):
    167              path = '/'+path
    168          comps =  path.split('/')
    169          parent_path = '/'.join(comps[:-1])
    170
    171          def unindex(comp, level, docid=docid, parent_path=None,
    172                                                              object_path=None):
    173              try:
    174                  self._index[comp][level].remove(docid)
    175
    176                  if not self._index[comp][level]:
    177                      del self._index[comp][level]
    178
    179                  if not self._index[comp]:
    180                      del self._index[comp]
    181                  # Remove parent_path and object path elements
    182                  if parent_path is not None:
    183                      self._index_parents[parent_path].remove(docid)
    184                      if not self._index_parents[parent_path]:
    185                          del self._index_parents[parent_path]
    186                  if object_path is not None:
    187                      del self._index_items[object_path]
    188              except KeyError:
    189                  logger.log(logging.INFO,
    190                             'Attempt to unindex object with id '
    191                             '%s failed' % docid)
    192
    193          for level in range(len(comps[1:])):
    194              comp = comps[level+1]
    195              unindex(comp, level)
    196
    197          # Remove the terminator
    198          level = len(comps[1:])
    199          comp = None
    200          unindex(comp, level-1, parent_path=parent_path, object_path=path)
    201
    202          self._length.change(-1)
    203          del self._unindex[docid]
    204
    205 -    def search(self, path, default_level=0, depth=-1, navtree=0,
    206                                                               navtree_start=0):
    207          """
    208          path is either a string representing a
    209          relative URL or a part of a relative URL or
    210          a tuple (path,level).
    211
    212          level >= 0  starts searching at the given level
    213          level <  0  not implemented yet
    214          """
    215
    216          if isinstance(path, basestring):
    217              startlevel = default_level
    218          else:
    219              startlevel = int(path[1])
    220              path = path[0]
    221
    222          absolute_path = isinstance(path, basestring) and path.startswith('/')
    223
    224          comps = filter(None, path.split('/'))
    225
    226          orig_comps = [''] + comps[:]
    227          # Optimization - avoid using the root set
    228          # as it is common for all objects anyway and add overhead
    229          # There is an assumption about catalog/index having
    230          # the same container as content
    231          if default_level == 0:
    232              indexpath = list(filter(None, self.getPhysicalPath()))
    233              while min(len(indexpath), len(comps)):
    234                  if indexpath[0] == comps[0]:
    235                      del indexpath[0]
    236                      del comps[0]
    237                      startlevel += 1
    238                  else:
    239                      break
    240
    241          if len(comps) == 0:
    242              if depth == -1 and not navtree:
    243                  return IISet(self._unindex.keys())
    244
    245          # Make sure that we get depth = 1 if in navtree mode
    246          # unless specified otherwise
    247
    248          orig_depth = depth
    249          if depth == -1:
    250              depth = 0 or navtree
    251
    252          # Optimized navtree starting with absolute path
    253          if absolute_path and navtree and depth == 1 and default_level==0:
    254              set_list = []
    255              # Insert root element
    256              if navtree_start >= len(orig_comps):
    257                  navtree_start = 0
    258              # create a set of parent paths to search
    259              for i in range(len(orig_comps), navtree_start, -1):
    260                  parent_path = '/'.join(orig_comps[:i])
    261                  parent_path = parent_path and parent_path or '/'
    262                  try:
    263                      set_list.append(self._index_parents[parent_path])
    264                  except KeyError:
    265                      pass
    266              return multiunion(set_list)
    267          # Optimized breadcrumbs
    268          elif absolute_path and navtree and depth == 0 and default_level==0:
    269              item_list = IISet()
    270              # Insert root element
    271              if navtree_start >= len(orig_comps):
    272                  navtree_start = 0
    273              # create a set of parent paths to search
    274              for i in range(len(orig_comps), navtree_start, -1):
    275                  parent_path = '/'.join(orig_comps[:i])
    276                  parent_path = parent_path and parent_path or '/'
    277                  try:
    278                      item_list.insert(self._index_items[parent_path])
    279                  except KeyError:
    280                      pass
    281              return item_list
    282          # Specific object search
    283          elif absolute_path and orig_depth == 0 and default_level == 0:
    284              try:
    285                  return IISet([self._index_items[path]])
    286              except KeyError:
    287                  return IISet()
    288          # Single depth search
    289          elif absolute_path and orig_depth == 1 and default_level == 0:
    290              # only get objects contained in requested folder
    291              try:
    292                  return self._index_parents[path]
    293              except KeyError:
    294                  return IISet()
    295          # Sitemaps, relative paths, and depth queries
    296          elif startlevel >= 0:
    297
    298              pathset = None # Same as pathindex
    299              navset  = None # For collecting siblings along the way
    300              depthset = None # For limiting depth
    301
    302              if navtree and depth and \
    303                     self._index.has_key(None) and \
    304                     self._index[None].has_key(startlevel):
    305                  navset = self._index[None][startlevel]
    306
    307              for level in range(startlevel, startlevel+len(comps) + depth):
    308                  if level-startlevel < len(comps):
    309                      comp = comps[level-startlevel]
    310                      if not self._index.has_key(comp) or not self._index[comp].has_key(level):
    311                          # Navtree is inverse, keep going even for
    312                          # nonexisting paths
    313                          if navtree:
    314                              pathset = IISet()
    315                          else:
    316                              return IISet()
    317                      else:
    318                          pathset = intersection(pathset,
    319                                                       self._index[comp][level])
    320                      if navtree and depth and \
    321                             self._index.has_key(None) and \
    322                             self._index[None].has_key(level+depth):
    323                          navset  = union(navset, intersection(pathset,
    324                                                self._index[None][level+depth]))
    325                  if level-startlevel >= len(comps) or navtree:
    326                      if self._index.has_key(None) and self._index[None].has_key(level):
    327                          depthset = union(depthset, intersection(pathset,
    328                                                      self._index[None][level]))
    329
    330              if navtree:
    331                  return union(depthset, navset) or IISet()
    332              elif depth:
    333                  return depthset or IISet()
    334              else:
    335                  return pathset or IISet()
    336
    337          else:
    338              results = IISet()
    339              for level in range(0,self._depth + 1):
    340                  ids = None
    341                  error = 0
    342                  for cn in range(0,len(comps)):
    343                      comp = comps[cn]
    344                      try:
    345                          ids = intersection(ids,self._index[comp][level+cn])
    346                      except KeyError:
    347                          error = 1
    348                  if error==0:
    349                      results = union(results,ids)
    350              return results
    351
    352 -    def _apply_index(self, request, cid=''):
    353          """ hook for (Z)Catalog
    354              'request' --  mapping type (usually {"path": "..." }
    355               additionaly a parameter "path_level" might be passed
    356               to specify the level (see search())
    357
    358              'cid' -- ???
    359          """
    360
    361          record = parseIndexRequest(request,self.id,self.query_options)
    362          if record.keys==None: return None
    363
    364          level    = record.get("level", 0)
    365          operator = record.get('operator', self.useOperator).lower()
    366          depth    = getattr(record, 'depth', -1) # Set to 0 or navtree later
    367                                                  # use getattr to get 0 value
    368          navtree  = record.get('navtree', 0)
    369          navtree_start  = record.get('navtree_start', 0)
    370
    371          # depending on the operator we use intersection of union
    372          if operator == "or":  set_func = union
    373          else: set_func = intersection
    374
    375          res = None
    376          for k in record.keys:
    377              rows = self.search(k,level, depth, navtree, navtree_start)
    378              res = set_func(res,rows)
    379
    380          if res:
    381              return res, (self.id,)
    382          else:
    383              return IISet(), (self.id,)
    384
    385 -    def getIndexSourceNames(self):
    386          """ return names of indexed attributes """
    387
    388          # By default PathIndex advertises getPhysicalPath even
    389          # though the logic in index_object is different.
    390
    391          try:
    392              return tuple(self.indexed_attrs)
    393          except AttributeError:
    394              return ('getPhysicalPath',)
    395
    396      index_html = DTMLFile('dtml/index', globals())
    397      manage_workspace = DTMLFile('dtml/manageExtendedPathIndex', globals())
    398
    399
    400  manage_addExtendedPathIndexForm = DTMLFile('dtml/addExtendedPathIndex', globals())
    401
    402 -def manage_addExtendedPathIndex(self, id, extra=None, REQUEST=None, RESPONSE=None, URL3=None):
    403      """Add an extended path index"""
    404      return self.manage_addIndex(id, 'ExtendedPathIndex', extra=extra,
    405                  REQUEST=REQUEST, RESPONSE=RESPONSE, URL1=URL3)
    406