Terrier Core

When using two pass indexing, the resulting inverted index contains wrong entries. Single pass indexing is not affected.

The error can be reproduced using this example:

= doc0.txt =
cats dogs horses

= doc1.txt =
chicken cats chicken chicken

= Program =
List<String> files = new ArrayList<String>();
files.add( "doc0.txt" );
files.add( "doc1.txt" );

/* two pass */
Collection col = new SimpleFileCollection( files, false );
Collection[] collections = new Collection[] { col };

Indexer indexer = new BasicIndexer( ApplicationSetup.TERRIER_INDEX_PATH, "test_filecollection" );
indexer.createDirectIndex( collections );
indexer.createInvertedIndex();

Index index = Index.createIndex( "index", "test_filecollection" );
Lexicon<String> lexicon = index.getLexicon();
InvertedIndex invertedIndex = index.getInvertedIndex();

LexiconEntry chickenEntry = lexicon.getLexiconEntry( "chicken" );
int[][] docs = invertedIndex.getDocuments( chickenEntry );

System.out.println( "docs[0]: " + Arrays.toString( docs[0] ) );
System.out.println( "docs[1]: " + Arrays.toString( docs[1] ) );
System.out.println( "docno of docs[0][0]: " + index.getMetaIndex().getItem( "docno", docs[0][0] ) );

/* single pass */
col = new SimpleFileCollection( files, false );
collections = new Collection[] { col };

BasicSinglePassIndexer singlePassIndexer = new BasicSinglePassIndexer(
ApplicationSetup.TERRIER_INDEX_PATH, "test_filecollection_singlepass" );
singlePassIndexer.createInvertedIndex( collections );

index = Index.createIndex( "index", "test_filecollection_singlepass" );
lexicon = index.getLexicon();
invertedIndex = index.getInvertedIndex();

chickenEntry = lexicon.getLexiconEntry( "chicken" );
docs = invertedIndex.getDocuments( chickenEntry );

System.out.println( "docs[0]: " + Arrays.toString( docs[0] ) );
System.out.println( "docs[1]: " + Arrays.toString( docs[1] ) );
System.out.println( "docno of docs[0][0]: " + index.getMetaIndex().getItem( "docno", docs[0][0] ) );

= Output =
docs[0]: [0]
docs[1]: [1]
docno of docs[0][0]: 1

docs[0]: [1]
docs[1]: [3]
docno of docs[0][0]: 2