Changeset 332

Timestamp:

30/04/06 14:09:58 (4 years ago)

Author:

athomas

Message:

pyndexter:

• Added a horribly inefficient built-in indexer, default.DefaultIndexer. There seems to be a memory leak somewhere, so on large datasets the indexer will consume large amounts of memory.
• Added a util module with CacheDict, currently used by the default indexer.
• Added some more CAP_ bits.
• Source state data is now accumulated by the Source classes __iter__() method, no longer requiring a full walk of the source to collect state. This means an Indexer.update() will automagically do the right thing.
• Factored out some common environment initialisation code into Indexer._init_env().
• Factored FileSource include/exclude/predicate code into base Source class so it can be reused.

Location:

pyndexter/trunk

Files:

• .todo (modified) (1 diff)
• pyndexter/__init__.py (modified) (11 diffs)
• pyndexter/default.py (added)
• pyndexter/file.py (modified) (5 diffs)
• pyndexter/hyperestraier.py (modified) (2 diffs)
• pyndexter/metasource.py (modified) (4 diffs)
• pyndexter/portalocker.py (added)
• pyndexter/util.py (added)
• pyndexter/xapian.py (modified) (2 diffs)

pyndexter/trunk/.todo

@@ -9 +9 @@
         Finish PyLucene adapter
     </note>
-    <note priority="medium" time="1145854608">
+    <note priority="medium" time="1145854608" done="1146296772">
         Finish MetaSource
     </note>
+    <note priority="medium" time="1146296806">
+        Optimise on disk format for DefaultIndexer. Use URI/word "ids" rather than full word.
+    </note>
+    <note priority="medium" time="1146321654">
+        I think it might need a MIME filter system, for translating known content types to plain text for indexing. eg. Just the content of HTML pages. This could get out of hand.
+    </note>
+    <note priority="medium" time="1146328561" done="1146368244">
+        state() is being called, which in the naive implementation simply walks the entire source. Need some way around this. Should the state() be accumulated somehow when the source is being walked?
+    </note>
+    <note priority="medium" time="1146331225" done="1146368238">
+        HTTPSource should be able to handle multiple iterations, but self._traversed renders this impossible.
+    </note>
 </todo>

pyndexter/trunk/pyndexter/__init__.py

@@ -23 +23 @@
 
 CAP_READONLY CAP_ORDERING CAP_CONTENT CAP_ATTRIBUTES CAP_RELEVANCE CAP_HITCOUNT
-CAP_LIST CAP_ITERATION
+CAP_LIST CAP_ITERATION CAP_ASTERISK CAP_QUESTION CAP_WHOLEWORD CAP_UNION
+CAP_INTERSECTION
 
 Document Source Indexer Search Hit
@@ -46 +47 @@
 CAP_LIST = 64           # Search result supports list-style lookup
 CAP_ITERATION = 128     # Supports index iteration
+CAP_ASTERISK = 256      # Supports the asterisk wildcard (*<term>*)
+CAP_QUESTION = 512      # Supports the single character wildcard (a?c)
+CAP_WHOLEWORD = 512     # Performs whole word searches by default
+CAP_UNION = 1024        # Supports unions (ie. matches documents with any word)
+CAP_INTERSECTION = 2048 # Supports intersections (ie. matches documents with
+                        # all words)
 
 
@@ -97 +104 @@
             return self.attributes[key]
         except KeyError, e:
-            raise AttributeError(str(e))
+            raise AttributeError(unicode(e))
 
     def __hash__(self):
@@ -116 +123 @@
     """ A source of indexable documents. A Source object is responsible for not
     only fetching documents and iterating over them, but for determining what
-    has changed in the source. This is achieved with the state() and
-    difference() methods. The ''state'' of a source is the minimum information
-    required to be able to determine what has changed. For FileSource this is a
-    list of all files and their modification times, for a SubversionSource it
-    would be as simple as the changeset number.
+    has changed in the source.
+    
+    Determing what has changed is achieved with the state() and difference()
+    methods. The ''state'' of a source is the minimum information required to
+    be able to determine what has changed. For FileSource this is a list of all
+    files and their modification times, for a SubversionSource it would be as
+    simple as the changeset number. The default state() and difference()
+    methods use the data in self._state.
 
     (All attributes, including document contents and URI's must be in unicode)
     """
+
+    def __init__(self, include=['*'], exclude=[], predicate=None):
+        self.include = include
+        self.exclude = exclude
+        self.predicate = predicate or self._glob_predicate
+        self._state = {}
 
     def matches(self, uri):
@@ -158 +174 @@
         """ Return a raw byte string representing the current state of this
         source.  Storage and retrieval of this byte string is typically handled
-        by the Indexer. """
-        state = {}
-        for uri in self:
-            state[uri] = self.fetch(uri).changed
-        state = pickle.dumps(state, 2)
-        compressed = StringIO()
-        gzip.GzipFile(filename='pyndexer source state', fileobj=compressed,
-                      mode='wb', compresslevel=1).write(state)
-        return compressed.getvalue()
+        by the Indexer. If this method returns false, the Indexer will assume
+        that state information is not available, and do nothing. """
+        if not self._state:
+            return None
+        return self._marshal_state(self._state)
 
     def difference(self, state):
@@ -173 +185 @@
         tuple is in the form `(<transition>, uri)`, where <transition> is one
         of ADDED, REMOVED or MODIFIED. """
-        state = StringIO(state)
-        try:
-            ungzipped = gzip.GzipFile(fileobj=state, mode='rb').read()
-            state = pickle.loads(ungzipped)
-        except Exception, e:
-            raise InvalidState('Invalid state provided to document source. '
-                               'Exception was %s: %s' % (e.__class__.__name__, e))
         current = set()
+        state = self._unmarshal_state(state)
         for uri in self:
             current.add(uri)
@@ -190 +196 @@
             yield (REMOVED, removed)
 
+    # Useful helper methods
+    def _glob_predicate(self, uri):
+        """ Given a list of include and exclude pattern lists, return whether
+        the given uri matches. """
+        from fnmatch import fnmatch
+        for pattern in self.exclude:
+            if fnmatch(uri, pattern):
+                return False
+        for pattern in self.include:
+            if fnmatch(uri, pattern):
+                return True
+        return False
+
+    def _marshal_state(self, state):
+        """ Pickle and compress state. This is used by the default state()
+        implementation, but can be reused. """
+        state = pickle.dumps(state, 2)
+        compressed = StringIO()
+        gzip.GzipFile(filename='pyndexer source state', fileobj=compressed,
+                      mode='wb', compresslevel=1).write(state)
+        return compressed.getvalue()
+
+    def _unmarshal_state(self, state):
+        """ Uncompress and unpickle state. Used by the default difference()
+        method, but can be reused. """
+        state = StringIO(state)
+        try:
+            ungzipped = gzip.GzipFile(fileobj=state, mode='rb').read()
+            return pickle.loads(ungzipped)
+        except Exception, e:
+            raise InvalidState('Invalid state provided to document source. '
+                               'Exception was %s: %s' % (e.__class__.__name__, e))
 
 class Indexer(object):
@@ -209 +247 @@
         on the Source copy, if available. """
         if not self.source:
-            raise IndexerError("This indexer has no Source object associated "
-                               "with and as such can not fetch() documents.")
+            raise IndexerError("This indexer has no associated Source object "
+                               "and as such can not fetch() documents.")
         return self.source.fetch(uri)
 
@@ -226 +264 @@
                                "capable of automatic updates.")
         if os.path.exists(self.state_path):
-            state = open(self.state_path).read()
+            try:
+                state = open(self.state_path).read()
+            except Exception, e:
+                raise IndexerError("Source state '%s' is not readable. "
+                                   "Exception was %s: %s" % 
+                                   (self.state_path, e.__class__.__name__,
+                                    unicode(e)))
+
             for transition, uri in self.source.difference(state):
                 if transition == REMOVED:
@@ -282 +327 @@
         constructor. """
         if self.mode == READWRITE and self.source and self.state_path:
-            open(self.state_path, 'w').write(self.source.state())
+            state = self.source.state()
+            if state:
+                open(self.state_path, 'w').write(self.source.state())
+
+    def _init_env(self, path):
+        """ Create a default environment with a <path> base directory. """
+        if not os.path.exists(path):
+            if self.mode != READWRITE:
+                raise IndexError("Indexer environment has not been initialised")
+            os.makedirs(path)
 
 class Search(object):
@@ -329 +383 @@
             return self.attributes[key]
         except KeyError, e:
-            raise AttributeError(str(e))
+            raise AttributeError(unicode(e))
 
     def _get_document(self):

pyndexter/trunk/pyndexter/file.py

@@ -1 +1 @@
 import sys
 import codecs
-import os.path
-from fnmatch import fnmatch
-from dircache import listdir
+import os
+from stat import *
 from urlparse import urlsplit, urlunsplit
 
@@ -11 +10 @@
     def __init__(self, root, include=['*'], exclude=[], predicate=None):
         """ Expose a subset of the file system for searching. """
+        Source.__init__(self, include, exclude, predicate)
         self.root = os.path.normpath(root)
-        self.include = include
-        self.exclude = exclude
-        self.predicate = predicate or self._glob_predicate
         self.encoding = sys.getfilesystemencoding()
 
@@ -21 +18 @@
             path = path.strip(os.path.sep)
             root_path = os.path.join(self.root, path)
-            for file in listdir(root_path):
+            for file in os.listdir(root_path):
                 full_path = os.path.join(root_path, file)
-                if os.path.isdir(full_path):
+                try:
+                    stat = os.lstat(full_path)
+                except OSError:
+                    continue
+                if not self.predicate(full_path) or not os.access(full_path, os.R_OK):
+                    continue
+                if S_ISDIR(stat.st_mode):
                     for file in walk_path(os.path.join(path, file)):
                         yield file
-                elif self.predicate(full_path) and os.path.exists(full_path):
-                    # TODO Stat for normal files + readability
-                    yield self._file2uri(full_path)
+                elif S_ISREG(stat.st_mode):
+                    yield (self._file2uri(full_path).decode(self.encoding), stat)
 
-        for file in walk_path('/'):
-            yield file.decode(self.encoding)
+        for file, stat in walk_path('/'):
+            self._state[file] = stat.st_mtime
+            yield file
 
     def matches(self, uri):
-        scheme, netloc, path, query, fragment = urlsplit(uri)
+        scheme, netloc, path, query, fragment = urlsplit(uri, 'file')
         path = os.path.normpath(path)
-        return scheme in ('file', '') and \
+        return scheme == 'file' and \
                path.startswith(self.root) and \
                self.predicate(path)
@@ -67 +70 @@
 
     def _uri2file(self, uri):
-        scheme, location, path, query, fragment = urlsplit(uri)
-        if scheme not in ('file', ''):
+        scheme, location, path, query, fragment = urlsplit(uri, 'file')
+        if scheme not in 'file':
             raise InvalidURI("URI scheme in '%s' not supported by FileSource"
                              % scheme)
@@ -76 +79 @@
                              % uri)
         return path.decode(self.encoding)
-
-    def _glob_predicate(self, file):
-        for pattern in self.exclude:
-            if fnmatch(file, pattern):
-                return False
-        for pattern in self.include:
-            if fnmatch(file, pattern):
-                return True
-        return False
-        

pyndexter/trunk/pyndexter/hyperestraier.py

@@ -5 +5 @@
 class HyperestraierIndexer(Indexer):
     capabilities = CAP_READONLY | CAP_CONTENT | CAP_ATTRIBUTES | CAP_ORDERING |\
-                   CAP_HITCOUNT | CAP_LIST | CAP_RELEVANCE
+                   CAP_HITCOUNT | CAP_LIST | CAP_RELEVANCE | CAP_WHOLEWORD | \
+                   CAP_ASTERISK | CAP_INTERSECTION
 
     def __init__(self, path, source=None, mode=READWRITE, hype_mode=None):
         Indexer.__init__(self, source, mode, os.path.join(path, 'state.db'))
         self.path = path
-        if not os.path.exists(self.path):
-            if mode != READWRITE:
-                raise IndexerError("Index directory has not been initialised")
-            os.makedirs(self.path)
+        self._init_env(self.path)
         self.hype_path = os.path.join(self.path, 'hyperestraier.db')
         if hype_mode is None:
@@ -60 +58 @@
             else:
                 order_type = 'STR'
-            print order_ascending
             order = u'@%s %s%s' % (order_by, order_type,
                                    order_ascending and 'A' or 'D')

pyndexter/trunk/pyndexter/metasource.py

@@ -5 +5 @@
 class MetaSource(Source):
     """ A collection of sources. If sources serve the same documents the
-    results are undefined, and probably not good.  """
+    results will be undefined, and probably not good. """
     def __init__(self, sources=[]):
         self.sources = sources
@@ -31 +31 @@
             if source.matches(uri):
                 return source.fetch(uri)
-        raise DocumentNotFound
+        raise DocumentNotFound(uri)
 
     def exists(self, uri):
@@ -49 +49 @@
             state = pickle.loads(state)
         except Exception, e:
-            raise InvalidState('Invalid state provided to document source. '
+            raise InvalidState('Invalid state provided to MetaSource. '
                                'Exception was %s: %s' % (e.__class__.__name__, e))
         for source in self.sources:
@@ -58 +58 @@
                 for change in source.difference(state[hash(source)]):
                     yield change
-

pyndexter/trunk/pyndexter/xapian.py

@@ -16 +16 @@
 
 class XapianIndexer(Indexer):
-    capabilities = CAP_ORDERING | CAP_READONLY | CAP_ATTRIBUTES | CAP_RELEVANCE | \
-                   CAP_HITCOUNT | CAP_LIST
+    capabilities = CAP_ORDERING | CAP_READONLY | CAP_ATTRIBUTES | \
+                   CAP_RELEVANCE | CAP_HITCOUNT | CAP_LIST | CAP_WHOLEWORD | \
+                   CAP_INTERSECTION
 
     def __init__(self, path, source=None, mode=READWRITE):
         Indexer.__init__(self, source, mode, os.path.join(path, 'state.db'))
         self.path = path
-        if not os.path.exists(self.path):
-            if mode != READWRITE:
-                raise IndexerError("Index directory has not been initialised")
-            os.makedirs(self.path)
+        self._init_env(self.path)
         self.idx_path = os.path.join(path, 'xapian.db')
         if mode == READWRITE:
@@ -68 +66 @@
     def search(self, phrase, order_by=None, order_ascending=True,
                order_type=str, intersection=True):
+        phrase = phrase.encode('utf-8')
         if order_by == 'relevance':
             order_args = {'sortByRelevence': True}