Source Code for Module Products.ZenUtils.MultiPathIndex

  1  ############################################################################## 
  2  #  
  3  # Copyright (C) Zenoss, Inc. 2008, all rights reserved. 
  4  #  
  5  # This content is made available according to terms specified in 
  6  # License.zenoss under the directory where your Zenoss product is installed. 
  7  #  
  8  ############################################################################## 
  9   
 10   
 11  import time 
 12  import logging 
 13  LOG = logging.getLogger('ZenUtils.MultiPathIndex') 
 14   
 15  from Globals import DTMLFile 
 16   
 17  from ExtendedPathIndex import ExtendedPathIndex 
 18  from Products.PluginIndexes.common import safe_callable 
 19  from BTrees.OOBTree import OOSet 
 20  from BTrees.IIBTree import IISet, intersection, union, multiunion 
 21   
 22 -def _isSequenceOfSequences(seq): 
 23      if not seq: 
 24          return False 
 25      return (isinstance(seq, (tuple, list)) and 
 26              all(isinstance(item, (tuple, list)) for item in seq)) 
 27   
 28 -def _recursivePathSplit(seq): 
 29      if isinstance(seq, (tuple, list)): 
 30          return map(_recursivePathSplit, seq) 
 31      if '/' in seq: 
 32          return seq.split('/') 
 33      else: 
 34          return seq 
 35   
 36   
 37 -class MultiPathIndex(ExtendedPathIndex): 
 38      """ 
 39      A path index that is capable of indexing multiple paths per object. 
 40      """ 
 41      meta_type = "MultiPathIndex" 
 42   
 43   
 44 -    def search(self, path, default_level=0, depth=-1, navtree=0, 
 45                                                               navtree_start=0): 
 46          """ 
 47          path is either a string representing a 
 48          relative URL or a part of a relative URL or 
 49          a tuple (path,level). 
 50   
 51          level >= 0  starts searching at the given level 
 52          level <  0  not implemented yet 
 53          """ 
 54   
 55          if isinstance(path, basestring): 
 56              startlevel = default_level 
 57          else: 
 58              startlevel = int(path[1]) 
 59              path = path[0] 
 60   
 61          absolute_path = isinstance(path, basestring) and path.startswith('/') 
 62          comps = filter(None, path.split('/')) 
 63   
 64          orig_comps = [''] + comps[:] 
 65   
 66          if depth > 0: 
 67              raise ValueError, "Can't do depth searches anymore" 
 68   
 69          if not comps: 
 70              comps = ['dmd'] 
 71              startlevel = 1 
 72          elif comps[0] == 'zport': 
 73              comps = comps[1:] 
 74          elif comps[0] != 'dmd': 
 75              raise ValueError, "Depth searches must start with 'dmd'" 
 76          startlevel = len(comps) 
 77          #startlevel = len(comps)-1 if len(comps) > 1 else 1 
 78   
 79          if len(comps) == 0: 
 80              if depth == -1 and not navtree: 
 81                  return IISet(self._unindex.keys()) 
 82   
 83          # Make sure that we get depth = 1 if in navtree mode 
 84          # unless specified otherwise 
 85   
 86          orig_depth = depth 
 87          if depth == -1: 
 88              depth = 0 or navtree 
 89   
 90          # Optimized navtree starting with absolute path 
 91          if absolute_path and navtree and depth == 1 and default_level==0: 
 92              set_list = [] 
 93              # Insert root element 
 94              if navtree_start >= len(orig_comps): 
 95                  navtree_start = 0 
 96              # create a set of parent paths to search 
 97              for i in range(len(orig_comps), navtree_start, -1): 
 98                  parent_path = '/'.join(orig_comps[:i]) 
 99                  parent_path = parent_path and parent_path or '/' 
100                  try: 
101                      set_list.append(self._index_parents[parent_path]) 
102                  except KeyError: 
103                      pass 
104              return multiunion(set_list) 
105          # Optimized breadcrumbs 
106          elif absolute_path and navtree and depth == 0 and default_level==0: 
107              item_list = IISet() 
108              # Insert root element 
109              if navtree_start >= len(orig_comps): 
110                  navtree_start = 0 
111              # create a set of parent paths to search 
112              for i in range(len(orig_comps), navtree_start, -1): 
113                  parent_path = '/'.join(orig_comps[:i]) 
114                  parent_path = parent_path and parent_path or '/' 
115                  try: 
116                      item_list.insert(self._index_items[parent_path]) 
117                  except KeyError: 
118                      pass 
119              return item_list 
120          # Specific object search 
121          elif absolute_path and orig_depth == 0 and default_level == 0: 
122              try: 
123                  return IISet([self._index_items[path]]) 
124              except KeyError: 
125                  return IISet() 
126          # Single depth search 
127          elif absolute_path and orig_depth == 1 and default_level == 0: 
128              # only get objects contained in requested folder 
129              try: 
130                  return self._index_parents[path] 
131              except KeyError: 
132                  return IISet() 
133          # Sitemaps, relative paths, and depth queries 
134          elif startlevel >= 0: 
135   
136              pathset = None # Same as pathindex 
137              navset  = None # For collecting siblings along the way 
138              depthset = None # For limiting depth 
139   
140              if navtree and depth and \ 
141                     self._index.has_key(None) and \ 
142                     self._index[None].has_key(startlevel): 
143                  navset = self._index[None][startlevel] 
144              for level in range(startlevel, startlevel+len(comps)): 
145                  if level <= len(comps): 
146                      comp = "/".join(comps[:level]) 
147                      if (not self._index.has_key(comp) 
148                          or not self._index[comp].has_key(level)): 
149                          # Navtree is inverse, keep going even for 
150                          # nonexisting paths 
151                          if navtree: 
152                              pathset = IISet() 
153                          else: 
154                              return IISet() 
155                      else: 
156                          return self._index[comp][level] 
157                      if navtree and depth and \ 
158                             self._index.has_key(None) and \ 
159                             self._index[None].has_key(level+depth): 
160                          navset  = union(navset, intersection(pathset, 
161                                                self._index[None][level+depth])) 
162                  if level-startlevel >= len(comps) or navtree: 
163                      if (self._index.has_key(None) 
164                          and self._index[None].has_key(level)): 
165                          depthset = union(depthset, intersection(pathset, 
166                                                      self._index[None][level])) 
167   
168              if navtree: 
169                  return union(depthset, navset) or IISet() 
170              elif depth: 
171                  return depthset or IISet() 
172              else: 
173                  return pathset or IISet() 
174   
175          else: 
176              results = IISet() 
177              for level in range(0,self._depth + 1): 
178                  ids = None 
179                  error = 0 
180                  for cn in range(0,len(comps)): 
181                      comp = comps[cn] 
182                      try: 
183                          ids = intersection(ids,self._index[comp][level+cn]) 
184                      except KeyError: 
185                          error = 1 
186                  if error==0: 
187                      results = union(results,ids) 
188              return results 
189   
190   
191 -    def getIndexSourceNames(self): 
192          """ return names of indexed attributes """ 
193          return (self.id, ) 
194   
195 -    def index_object(self, docid, obj, threshold=100): 
196          """ hook for (Z)Catalog """ 
197   
198          f = getattr(obj, self.id, None) 
199          if f is not None: 
200              if safe_callable(f): 
201                  try: 
202                      paths = f() 
203                  except AttributeError: 
204                      return 0 
205              else: 
206                  paths = f 
207          else: 
208              try: 
209                  paths = obj.getPhysicalPath() 
210              except AttributeError: 
211                  return 0 
212   
213          if not paths: return 0 
214          paths = _recursivePathSplit(paths) 
215          if not _isSequenceOfSequences(paths): 
216              paths = [paths] 
217   
218          if docid in self._unindex: 
219              unin = self._unindex[docid] 
220              # Migrate old versions of the index to use OOSet 
221              if isinstance(unin, set): 
222                  unin = self._unindex[docid] = OOSet(unin) 
223              for oldpath in list(unin): 
224                  if list(oldpath.split('/')) not in paths: 
225                      self.unindex_paths(docid, (oldpath,)) 
226          else: 
227              self._unindex[docid] = OOSet() 
228              self._length.change(1) 
229   
230          self.index_paths(docid, paths) 
231   
232          return 1 
233   
234   
235 -    def index_paths(self, docid, paths): 
236          for path in paths: 
237              if isinstance(path, (list, tuple)): 
238                  path = '/'+ '/'.join(path[1:]) 
239              comps = filter(None, path.split('/')) 
240              parent_path = '/' + '/'.join(comps[:-1]) 
241   
242              for i in range(len(comps)): 
243                  comp = "/".join(comps[1:i+1]) 
244                  if comp: 
245                      self.insertEntry(comp, docid, i) 
246   
247              # Add terminator 
248              self.insertEntry(None, docid, len(comps)-1, parent_path, path) 
249   
250              self._unindex.setdefault(docid, OOSet()).insert(path) 
251   
252   
253 -    def unindex_paths(self, docid, paths): 
254   
255          if not self._unindex.has_key(docid): 
256              return 
257   
258          def unindex(comp, level, docid=docid, parent_path=None, 
259                      object_path=None): 
260              try: 
261                  self._index[comp][level].remove(docid) 
262   
263                  if not self._index[comp][level]: 
264                      del self._index[comp][level] 
265   
266                  if not self._index[comp]: 
267                      del self._index[comp] 
268                  # Remove parent_path and object path elements 
269                  if parent_path is not None: 
270                      self._index_parents[parent_path].remove(docid) 
271                      if not self._index_parents[parent_path]: 
272                          del self._index_parents[parent_path] 
273                  if object_path is not None: 
274                      del self._index_items[object_path] 
275              except KeyError: 
276                  # Failure 
277                  pass 
278   
279          old = set(self._unindex.get(docid, ())) 
280          mkstr = lambda path:'/'.join(path) if isinstance(path, tuple) else path 
281          paths = map(mkstr, paths) 
282          toremove = set(paths) & old 
283          tokeep = old - toremove 
284          for path in toremove: 
285              if not path.startswith('/'): 
286                  path = '/'+path 
287              comps =  path.split('/') 
288              parent_path = '/'.join(comps[:-1]) 
289   
290              for level in range(1, len(comps[2:])+1): 
291                  comp = "/".join(comps[2:level+2]) 
292                  unindex(comp, level, docid, parent_path, path) 
293              # Remove the terminator 
294              level = len(comps[1:]) 
295              comp = None 
296              unindex(comp, level-1, parent_path=parent_path, object_path=path) 
297   
298              self._unindex[docid].remove(path) 
299   
300          if tokeep: 
301              self.index_paths(docid, tokeep) 
302          else: 
303              # Cleared out all paths for the object 
304              self._length.change(-1) 
305              del self._unindex[docid] 
306   
307   
308 -    def unindex_object(self, docid): 
309          """ hook for (Z)Catalog """ 
310          if not self._unindex.has_key(docid): 
311              return 
312          self.unindex_paths(docid, self._unindex[docid]) 
313   
314      manage = manage_main = DTMLFile('dtml/manageMultiPathIndex', globals()) 
315      manage_main._setName('manage_main') 
316   
317   
318  manage_addMultiPathIndexForm = DTMLFile('dtml/addMultiPathIndex', globals()) 
319   
320 -def manage_addMultiPathIndex(self, id, REQUEST=None, RESPONSE=None, 
321                               URL3=None): 
322      """ 
323      Add a MultiPathIndex. 
324      """ 
325      return self.manage_addIndex(id, 'MultiPathIndex', extra=None, 
326                                  REQUEST=REQUEST, RESPONSE=RESPONSE, 
327                                  URL1=URL3) 
328