Changeset 384

Timestamp:

02/11/07 06:40:56 (2 years ago)

Author:

athomas

Message:

pyndexter:

• Framework filters will now be passed the Framework object as their first
  argument, in addition to context and stream.
  
• Added util.TimingFilter, a Framework filter useful for collecting
  timing statistics.
  
• Added some doctests to URI.
  

Files:

• pyndexter/trunk/pyndexter/__init__.py (modified) (3 diffs)
• pyndexter/trunk/pyndexter/sources/file.py (modified) (1 diff)
• pyndexter/trunk/pyndexter/util.py (modified) (4 diffs)
• pyndexter/trunk/.todo (modified) (1 diff)

pyndexter/trunk/pyndexter/__init__.py

@@ -820 +820 @@
-        """ Update the index with the current state of the document source.
-
-context, stream)`, where `stream`
-
+framework, context, stream)`,
+where `stream` 
-        self._assert_rw()
-        if not self.state_store:
@@ -834 +834 @@
-        if self.state_store.exists():
-            store = self.state_store.retrieve()
-
+self, 
-                                          self.source.difference(store)):
-                if transition == REMOVED:
@@ -847 +847 @@
-                    yield ADDED, uri
-
-
+self, 
-                self.index(uri)
-

pyndexter/trunk/pyndexter/sources/file.py

@@ -28 +28 @@
-
-class FileSource(Source):
- 
+
-
-    def __init__(self, framework, path, include=None, exclude=None, predicate=None):

pyndexter/trunk/pyndexter/util.py

@@ -9 +9 @@
-import re
-import posixpath
+import sys
-from StringIO import StringIO
-from urllib import quote, unquote
+from datetime import datetime, timedelta
-try:
-    set = set
@@ -22 +24 @@
-set frozenset
-quote unquote
-
+ TimingFilter
-""".split()
-
@@ -36 +38 @@
-        scheme://username:password@host/path;parameters?query#fragment
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-    _pattern = re.compile(r'(?:(?P<scheme>[^:]+)://)?(?:(?P<username>[^:@]*)(?::(?P<password>[^@]*))?@)?(?P<host>[^?/#:]*)(?::(P<port>[\d+]+))?(?P<path>/[^#?]*)?(?:\?(?P<query>[^#]*))?(?:#(?P<fragment>.*))?')
@@ -104 +130 @@
-            uri += '#' + quote(self.fragment)
-        return uri
+
+
+class TimingFilter(object):
+    """A Framework filter for collecting timing statistics."""
+    def __init__(self, next_filter=None, progressive=False):
+        """`next_filter` is the next filter in the chain.
+
+        `progressive` will print statistics while the indexer is running."""
+        if next_filter:
+            self.next_filter = next_filter
+        self.times = []
+        self.total = timedelta()
+        self.average = timedelta()
+        self.progressive = progressive
+
+    def next_filter(self, framework, context, stream):
+        for transition, uri in stream:
+            yield transition, uri
+
+    def __call__(self, framework, context, stream):
+        self.times = []
+        for transition, uri in self.next_filter(framework, context, stream):
+            start = datetime.now()
+            yield transition, uri
+            end = datetime.now()
+            line = (transition, uri, start, end)
+            self.times.append(line)
+            if self.progressive:
+                self.print_line(*line)
+
+        self.total = timedelta()
+        self.average = timedelta()
+        for transition, uri, start, end in self.times:
+            self.total += end - start
+        if self.total:
+            self.average = self.total / len(self.times)
+        if self.progressive:
+            self.print_summary()
+
+    def print_line(self, transition, uri, start, end, out=sys.stdout):
+        from pyndexter import MODIFIED, ADDED, REMOVED
+        mapping = {MODIFIED: 'MODIFIED', ADDED: 'ADDED', REMOVED: 'REMOVED'}
+        print >>out, '%s %s (in %s)' % (mapping[transition], uri, end - start)
+
+    def print_summary(self, out=sys.stdout):
+        print >>out
+        print >>out, "Indexed %i documents" % len(self.times)
+        print >>out, 'Total time to index: %s' % self.total
+        print >>out, 'Average time to index: %s' % self.average
+
+    def __str__(self):
+        from StringIO import StringIO
+        out = StringIO()
+        for transition, uri, start, end in self.times:
+            self.print_line(transition, uri, start, end, out=out)
+        self.print_summary(out)
+        return out.getvalue()

pyndexter/trunk/.todo

@@ -123 +123 @@
-        Fix port parsing in util.URI.
-    </note>
+    <note priority="medium" time="1171055477">
+        Write a decent test suite.
+    </note>
-</todo>