Index: pyndexter/branches/simplification/pyndexter/indexers/_hyperestraier.py =================================================================== --- pyndexter/branches/simplification/pyndexter/indexers/_hyperestraier.py (revision 458) +++ pyndexter/branches/simplification/pyndexter/indexers/_hyperestraier.py (revision 465) @@ -48,4 +48,5 @@ import hyperestraier from pyndexter import * +from pyndexter import errors from pyndexter.util import URI @@ -59,4 +60,5 @@ uri = URI(uri) uri.scheme = 'http' + uri.path = 'node/' + uri.path return HyperestraierIndexer(uri) @@ -68,23 +70,32 @@ uri = URI(uri, port=1978) + scrubbed = URI(uri) + scrubbed.username = None + scrubbed.password = None self.db = hyperestraier.Node() - self.db.set_url(str(uri)) + self.db.set_url(str(scrubbed)) + self.db.set_auth(uri.username, uri.password) def index(self, document): + uri = unicode(document.uri) hdoc = hyperestraier.Document() - for k, v in document.attributes.iteritems(): + for k, v in document.iteritems(): hdoc.add_attr(u'@' + k, v) - for line in document.content.splitlines(): + hdoc.add_attr(u'@uri', uri) + for line in document.texts: hdoc.add_text(line) - self.db.put_doc(hdoc, 1) + if not self.db.put_doc(hdoc): + raise errors.IndexerError('Failed to index %s' % document.uri) def discard(self, uri): uri = unicode(uri) if not self.db.out_doc_by_uri(uri): - raise DocumentNotFound(uri) + raise errors.DocumentNotFound(uri) def fetch(self, uri): uri = unicode(uri) doc = self.db.get_doc_by_uri(uri) + if not doc: + raise errors.DocumentNotFound(uri) attributes = self._translate_attributes(doc) return Document(uri, texts=doc.dtexts, quality=0.99, @@ -98,9 +109,6 @@ self.db.optimize() - def flush(self): - self.db.sync() - def close(self): - self.db.close() + self.flush() self.db = None Index: pyndexter/branches/simplification/pyndexter/util.py =================================================================== --- pyndexter/branches/simplification/pyndexter/util.py (revision 457) +++ pyndexter/branches/simplification/pyndexter/util.py (revision 465) @@ -25,4 +25,5 @@ """.split() + class URI(object): """Parse a URI into its component parts. The `query` component is passed @@ -40,7 +41,7 @@ The URI constructor can be passed a string: - >>> u = URI('http://user:password@www.example.com/some/path?parm=1&parm=2&other=3#fragment') + >>> u = URI('http://user:password@www.example.com:12345/some/path?parm=1&parm=2&other=3#fragment') >>> u - URI(u'http://user:password@www.example.com/some/path?other=3&parm=1&parm=2#fragment') + URI(u'http://user:password@www.example.com:12345/some/path?other=3&parm=1&parm=2#fragment') >>> u.scheme 'http' @@ -60,6 +61,6 @@ ...or the individual URI components as keyword arguments: - >>> URI(scheme='http', username='user', password='password', host='www.example.com', path='/some/path', query={'parm': [1, 2], 'other': [3]}, fragment='fragment') - URI(u'http://user:password@www.example.com/some/path?other=3&parm=1&parm=2#fragment') + >>> URI(scheme='http', username='user', password='password', host='www.example.com', port=12345, path='/some/path', query={'parm': [1, 2], 'other': [3]}, fragment='fragment') + URI(u'http://user:password@www.example.com:12345/some/path?other=3&parm=1&parm=2#fragment') ...or finally, another URI object: @@ -71,5 +72,5 @@ False >>> v - URI(u'http://user:password@www.example.com/some/path?other=3&parm=1&parm=2#fragment') + URI(u'http://user:password@www.example.com:12345/some/path?other=3&parm=1&parm=2#fragment') URI also normalises the path component: @@ -79,8 +80,17 @@ """ - _pattern = re.compile(r'(?:(?P[^:]+)://)?(?:(?P[^:@]*)(?::(?P[^@]*))?@)?(?P[^?/#:]*)(?::(P[\d+]+))?(?P/[^#?]*)?(?:\?(?P[^#]*))?(?:#(?P.*))?') - - __slots__ = ('scheme', 'username', 'password', 'host', 'port', '_path', - 'query', 'fragment') + _pattern = re.compile(r""" + (?:(?P[^:]+)://)? + (?:(?P[^:@]*) + (?::(?P[^@]*))?@)? + (?P[^?/#:]*) + (?::(?P[^/]+))? + (?P/[^#?]*)? + (?:\?(?P[^#]*))? + (?:\#(?P.*))? + """, re.VERBOSE) + + __slots__ = ['scheme', 'username', 'password', 'host', 'port', '_path', + 'query', 'fragment'] def __init__(self, uri=None, scheme=None, username=None, password=None, @@ -155,4 +165,7 @@ def __str__(self): + return unicode(self).encode('utf-8') + + def __unicode__(self): uri = unicode(self.scheme and (quote(self.scheme) + u'://') or u'') if self.username or self.password: Index: pyndexter/branches/simplification/pyndexter/__init__.py =================================================================== --- pyndexter/branches/simplification/pyndexter/__init__.py (revision 459) +++ pyndexter/branches/simplification/pyndexter/__init__.py (revision 465) @@ -18,5 +18,4 @@ from StringIO import StringIO from UserDict import DictMixin -from pyndexter.errors import * from pyndexter.util import URI from pyndexter.query import Query @@ -34,6 +33,5 @@ __all__ = ['Document', 'Query', 'Hit', 'Indexer', 'ResultSet', 'Stemmer', - 'Error', 'DocumentNotFound', 'InvalidURI', 'InvalidIndexer', - 'connect'] + 'connect', 'URI'] @@ -46,15 +44,4 @@ -class Error(Exception): - """Base of all pyndexter exceptions.""" - -class DocumentNotFound(Error): - """Raised when a document could not be found, usually by the fetch() - methods.""" - -class PluginError(Error): - """An error occurred in the plugin system.""" - - class Stemmer(object): """Abstraction for a stemming algorithm.""" @@ -81,5 +68,5 @@ def __init__(self, uri, quality=1.0, attributes=None, text=None, texts=None): - self.uri = uri + self.uri = URI(uri) self.quality = quality self._attributes = {} @@ -206,10 +193,16 @@ def __len__(self): """ Return the length of the result set. """ - raise NotImplementedError + count = 0 + for i in self: + count += 1 + return count def __getitem__(self, index): """Return a Hit object for a specific index in the search result. Not necessarily implemented by all Indexers.""" - raise NotImplementedError + for i, value in enumerate(self): + if i == index: + return value + raise IndexError(index) def __getslice__(self, i, j): @@ -279,5 +272,6 @@ class IndexerWrapper(Indexer): - """An Indexer wrapper that does type translation for end users.""" + """An Indexer wrapper that does some convenient type conversion for end + users.""" def __init__(self, indexer): Indexer.__init__(self) @@ -296,7 +290,7 @@ def discard(self, uri): - if isinstance(uri, document): - uri = document.uri - return self.indexer.discard(URI(uri)) + if isinstance(uri, Document): + uri = uri.uri + return self.indexer.discard(uri) def search(self, query): @@ -309,5 +303,5 @@ -def load_plugin(name, entry_point, plugin_paths=None): +def iter_plugins(entry_point, plugin_paths=None): import pkg_resources @@ -340,14 +334,21 @@ for entry in pkg_resources.working_set.iter_entry_points(entry_point): - if entry.name == name: - try: - _debug('Loading %s' % entry) - factory = entry.load(require=True) - return factory - except (ImportError, pkg_resources.DistributionNotFound, - pkg_resources.VersionConflict, pkg_resources.UnknownExtra), e: - _debug('Failed to load %s: %s' % (name, _format_error(entry, e))) - raise PluginError('%s (%s)' % (name, _format_error(entry, e))) - raise PluginError('No suitable plugins found for %s' % name) + try: + _debug('Loading %s' % entry) + plugin = entry.load(require=True) + yield entry.name, plugin + except (ImportError, pkg_resources.DistributionNotFound, + pkg_resources.VersionConflict, pkg_resources.UnknownExtra), e: + _debug('Failed to load %s: %s' % (entry.name, _format_error(entry, e))) + yield None, PluginError('%s (%s)' % (entry.name, _format_error(entry, e))) + + +def load_plugin(plugin_name, entry_point, plugin_paths=None): + for name, plugin in iter_plugins(entry_point, plugin_paths): + if name is None: + raise plugin + if name == plugin_name: + return plugin + raise PluginError('No suitable plugins found named "%s"' % name) Index: pyndexter/branches/simplification/pyndexter/query.py =================================================================== --- pyndexter/branches/simplification/pyndexter/query.py (revision 458) +++ pyndexter/branches/simplification/pyndexter/query.py (revision 465) @@ -9,12 +9,8 @@ import re -from pyndexter import Error +from pyndexter.errors import InvalidQuery __all__ = ['Query'] - - -class InvalidQuery(Error): - """Invalid query string.""" Index: pyndexter/branches/simplification/.todo =================================================================== --- pyndexter/branches/simplification/.todo (revision 465) +++ pyndexter/branches/simplification/.todo (revision 465) @@ -0,0 +1,156 @@ + + + + Pyndexter, pronounced 'poindexter', a full text indexing and search abstraction layer + + + Callbacks for index() and discard(), perhaps something similar for Source objects? + + Framework.update() accepts a filter callback. This could be sufficient. + + + + Finish PyLucene adapter + + Functional enough for a first commit. + + + + Finish MetaSource + + + I think it might need a MIME filter system, for translating known content types to plain text for indexing. eg. Just the content of HTML pages. This could get out of hand. + + + state() is being called, which in the naive implementation simply walks the entire source. Need some way around this. Should the state() be accumulated somehow when the source is being walked? + + + HTTPSource should be able to handle multiple iterations, but self._traversed renders this impossible. + + + For storing state, perhaps there should be default store_state(store)/restore_state(store) methods. Also need a Store class, or just use a file object... + + + Refactor Indexer into two classes: the Indexer itself, and a class that glues Source and the Indexer together. This would remove the duplication I'm getting in all the stock methods (update, index, fetch, etc.) + + Done as the Framework class. + + + + Add slicing to Result objects. This will allow fast pagination in result displays. + + + Add some "stock" query translators (eg. a AND b OR c style, a b or c, +a +b c, etc.) + + Added a general to_boolean() method to the Query object. Operators can be overridden for variants. + + + + Incremental updates for the indexer state. Waiting until the end of the index, then writing the state, is bad. A single document error can render the entire index useless. + + "Transactions" for state updates? + + + I think an anydbm style interface for storing state could be useful. + + + + Add a swish-e adapter. The Python module SwishE only appears to expose searching :( + + Done, but only for searching. + + + + Why is Xapian not returning all the hits? + + + I'd like to add database Sources, but I can't see a way to handle updated rows without doing a full table scan. + + + Use metakit for pure-Python implementation? (Check out "divmod pyndex" for ideas) + + + Deprecate Hit and just use Document - they're almost identical in functionality. + + Bad idea. Hit now has indexed and current members, which lazily fetch from the Indexer and Framework, respectively. + + + Perhaps Results should use the framework to try and fetch a Document, then "underlay" the hit attributes? + + + + Add generalised "field" indexing. + + + Search result ordering. + + + How do we detect when sources have been removed from the index? If file:///tmp changes to file:///usr, the Framework has no real way of detecting which URI's in the index are no longer valid. + + + Default indexer tasks + + Optimise on disk format for DefaultIndexer. Use URI/word "ids" rather than full word. + + + Abstract storage mechanism so that sqlite, metakit, anydbm, etc. can be used. This would allow for wide use. + + + Use bigrams same as the current 'default' search? This is a good solution I think. Allows for sub-word searches, start and end of word searches, etc. + + + Optionally use snowball stemmer. + + + Have a built-in stemmer? Porter? + + + Use "nltk" stemmer? + + + + http://www.biais.org/blog/index.php/2007/01/31/25-spelling-correction-using-the-python-natural-language-toolkit-nltk <- interesting + + + Pyndex adapter. + + + Add utility function for converting attribute dictionary keys to plain strings (common pattern). + + + Normalise URI usage everywhere. + + + Fix port parsing in util.URI. + + + Write a decent test suite. + + Test that searches return the right hits. Don't care about order. + + + Test that all interfaces pass and receive unicode correctly. + + + Test that all indexers and sources pass URI objects correctly. + + + + http://www.liris.org/tech/program/hyperestraier-purepython/ <- Client library for HE server. + + + Refactor Query.as_string() into dialects. + + + Add support for "sphinx" (http://www.sphinxsearch.com) + + + Hyperestraier backend was failing to fetch or delete documents that *were* in the index. Searching through the UI showed them, but attempts to delete or fetch them failed. + + + Hyperestraier client module doesn't report the actual errors. Not useful for troubleshooting. + + + If Hyperestraier tests abort estmaster may not be stopped. Need to fix this. + + Index: pyndexter/branches/simplification/setup.py =================================================================== --- pyndexter/branches/simplification/setup.py (revision 459) +++ pyndexter/branches/simplification/setup.py (revision 465) @@ -4,9 +4,9 @@ name='pyndexter', description="An abstraction layer for full-text indexing engines.", - long_description="""Pyndexter (pronounced 'poindexter') is an abstraction - layer for full-text indexing and search engines. It presents a uniform - query syntax to the user, includes a basic but functional pure-Python - indexer, and has adapters for Hype, Hyperestraier, Lucene, Lupy, - Pyndex, Swish-e and Xapian.""", + long_description="""Pyndexter is an abstraction layer for full-text + indexing and search engines. It presents a uniform query syntax to the + user, includes a basic but functional pure-Python indexer, and has + adapters for Hype, Hyperestraier, Lucene, Lupy, Pyndex, Swish-e and + Xapian.""", url='http://swapoff.org/pyndexter', download_url='http://swapoff.org/pyndexter', @@ -16,5 +16,5 @@ author_email='alec@swapoff.org', version='0.4', - #test_suite='pyndexter.test.suite', + test_suite='pyndexter.tests.suite', classifiers=['Development Status :: 3 - Alpha', 'Environment :: Plugins',