Changeset 332 for pyndexter/trunk/pyndexter/file.py

Timestamp:

30/04/06 14:09:58 (4 years ago)

Author:

athomas

Message:

pyndexter:

• Added a horribly inefficient built-in indexer, default.DefaultIndexer. There seems to be a memory leak somewhere, so on large datasets the indexer will consume large amounts of memory.
• Added a util module with CacheDict, currently used by the default indexer.
• Added some more CAP_ bits.
• Source state data is now accumulated by the Source classes __iter__() method, no longer requiring a full walk of the source to collect state. This means an Indexer.update() will automagically do the right thing.
• Factored out some common environment initialisation code into Indexer._init_env().
• Factored FileSource include/exclude/predicate code into base Source class so it can be reused.

Files:

• pyndexter/trunk/pyndexter/file.py (modified) (5 diffs)

pyndexter/trunk/pyndexter/file.py

@@ -1 +1 @@
 import sys
 import codecs
-import os.path
-from fnmatch import fnmatch
-from dircache import listdir
+import os
+from stat import *
 from urlparse import urlsplit, urlunsplit
 
@@ -11 +10 @@
     def __init__(self, root, include=['*'], exclude=[], predicate=None):
         """ Expose a subset of the file system for searching. """
+        Source.__init__(self, include, exclude, predicate)
         self.root = os.path.normpath(root)
-        self.include = include
-        self.exclude = exclude
-        self.predicate = predicate or self._glob_predicate
         self.encoding = sys.getfilesystemencoding()
 
@@ -21 +18 @@
             path = path.strip(os.path.sep)
             root_path = os.path.join(self.root, path)
-            for file in listdir(root_path):
+            for file in os.listdir(root_path):
                 full_path = os.path.join(root_path, file)
-                if os.path.isdir(full_path):
+                try:
+                    stat = os.lstat(full_path)
+                except OSError:
+                    continue
+                if not self.predicate(full_path) or not os.access(full_path, os.R_OK):
+                    continue
+                if S_ISDIR(stat.st_mode):
                     for file in walk_path(os.path.join(path, file)):
                         yield file
-                elif self.predicate(full_path) and os.path.exists(full_path):
-                    # TODO Stat for normal files + readability
-                    yield self._file2uri(full_path)
+                elif S_ISREG(stat.st_mode):
+                    yield (self._file2uri(full_path).decode(self.encoding), stat)
 
-        for file in walk_path('/'):
-            yield file.decode(self.encoding)
+        for file, stat in walk_path('/'):
+            self._state[file] = stat.st_mtime
+            yield file
 
     def matches(self, uri):
-        scheme, netloc, path, query, fragment = urlsplit(uri)
+        scheme, netloc, path, query, fragment = urlsplit(uri, 'file')
         path = os.path.normpath(path)
-        return scheme in ('file', '') and \
+        return scheme == 'file' and \
                path.startswith(self.root) and \
                self.predicate(path)
@@ -67 +70 @@
 
     def _uri2file(self, uri):
-        scheme, location, path, query, fragment = urlsplit(uri)
-        if scheme not in ('file', ''):
+        scheme, location, path, query, fragment = urlsplit(uri, 'file')
+        if scheme not in 'file':
             raise InvalidURI("URI scheme in '%s' not supported by FileSource"
                              % scheme)
@@ -76 +79 @@
                              % uri)
         return path.decode(self.encoding)
-
-    def _glob_predicate(self, file):
-        for pattern in self.exclude:
-            if fnmatch(file, pattern):
-                return False
-        for pattern in self.include:
-            if fnmatch(file, pattern):
-                return True
-        return False
-