Index: src/uk/ac/gla/terrier/indexing/hadoop/Hadoop_BasicSinglePassIndexer.java =================================================================== --- src/uk/ac/gla/terrier/indexing/hadoop/Hadoop_BasicSinglePassIndexer.java (revision 2549) +++ src/uk/ac/gla/terrier/indexing/hadoop/Hadoop_BasicSinglePassIndexer.java (working copy) @@ -194,6 +194,9 @@ Path indexDestination = FileOutputFormat.getWorkOutputPath(jc); mapTaskID = TaskAttemptID.forName(jc.get("mapred.task.id")).getTaskID().toString(); currentIndex = Index.createNewIndex(indexDestination.toString(), mapTaskID); + //during reduce, we dont want to load indices into memory, as we only use + //them as streams + currentIndex.setIndexProperty("index.preloadIndices.disabled", "true"); RunData = new DataOutputStream( Files.writeFileStream( new Path(indexDestination, mapTaskID+".runs").toString()) @@ -451,6 +454,7 @@ throws IOException { if (logger.isDebugEnabled()) logger.debug("Reduce for term "+Term.getText()); + reporter.setStatus("Reducer is merging term " + Term.getText()); if (! reduceStarted) { final LinkedList runData = loadRunData(); Index: src/uk/ac/gla/terrier/structures/IndexUtil.java =================================================================== --- src/uk/ac/gla/terrier/structures/IndexUtil.java (revision 2549) +++ src/uk/ac/gla/terrier/structures/IndexUtil.java (working copy) @@ -2,11 +2,13 @@ import java.io.Closeable; import java.io.IOException; +import java.util.Arrays; import java.util.Iterator; -import uk.ac.gla.terrier.utility.ArrayUtils; import uk.ac.gla.terrier.utility.Files; +/** Class with handy utilities for use on an Index. + * @since 3.0 */ public class IndexUtil { /** Move an index from one location to another */ @@ -33,12 +35,12 @@ { if (filename.startsWith(actualPrefix)) { - Files.delete(path + "/" + filename); } } } + /** Print the contents of the document index */ @SuppressWarnings("unchecked") public static void printDocumentIndex(Index index) throws IOException { @@ -53,17 +55,20 @@ close(iterator); } + + /** Print the contents of the meta index */ + @SuppressWarnings("unchecked") public static void printMetaIndex(Index index) throws IOException { - final int numDocs = index.getCollectionStatistics().getNumberOfDocuments(); - final MetaIndex meta = index.getMetaIndex(); - for(int i=0;i inputStream = (Iterator)index.getIndexStructureInputStream("meta"); + while(inputStream.hasNext()) { - String[] values = meta.getAllItems(i); - System.out.println(i+":"+ArrayUtils.join(values, " ")); + System.out.println(Arrays.toString(inputStream.next())); } + IndexUtil.close(inputStream); } + /** Rename a structure within a given index */ public static void renameIndexStructure(Index index, String sourceStructureName, String destinationStructureName) { final String actualSourcePrefix = index.getPrefix() +'.' + sourceStructureName; @@ -78,6 +83,7 @@ index.getPath() + '/' + filename.replaceFirst(actualSourcePrefix, actualDestinationPrefix)); } } + //TODO - should the index properties be updated? } Index: src/uk/ac/gla/terrier/structures/CompressingMetaIndex.java =================================================================== --- src/uk/ac/gla/terrier/structures/CompressingMetaIndex.java (revision 2549) +++ src/uk/ac/gla/terrier/structures/CompressingMetaIndex.java (working copy) @@ -1,12 +1,14 @@ package uk.ac.gla.terrier.structures; import gnu.trove.TObjectIntHashMap; + import java.io.DataInputStream; import java.io.IOException; import java.io.ObjectInputStream; import java.io.RandomAccessFile; import java.nio.MappedByteBuffer; import java.nio.channels.FileChannel; +import java.util.Iterator; import java.util.Map; import java.util.NoSuchElementException; import java.util.zip.DataFormatException; @@ -65,7 +67,7 @@ return out; } - public void close() throws IOException + public final void close() throws IOException { dataSource.close(); } @@ -89,13 +91,116 @@ return out; } - public void close() throws IOException + public final void close() throws IOException { dataSourceChannel.close(); dataSource.close(); } } + + public static class InputStream implements Iterator, java.io.Closeable + { + final DataInputStream zdata; + final ObjectInputStream idx; + final protected int compressionLevel; + final protected int recordLength; + + protected Inflater inflater; + + protected int keyCount; + protected int[] keyOffsets; + protected int[] keyLengths; + + final int numberOfRecords; + int index=0; + + String[] metaValues; + + protected long lastOffset; + protected long fileLength; + + public InputStream(Index index, String structureName) throws IOException + { + final String dataFilename = index.getPath() + ApplicationSetup.FILE_SEPARATOR + index.getPrefix() + "." + structureName + ".zdata"; + final String indxFilename = index.getPath() + ApplicationSetup.FILE_SEPARATOR + index.getPrefix() + "." + structureName + ".idx"; + zdata = new DataInputStream(Files.openFileStream(dataFilename)); + idx = new ObjectInputStream(Files.openFileStream(indxFilename)); + fileLength = Files.length(dataFilename); + + //1. int - how much zlib was used + compressionLevel = idx.readInt(); + //2. int - how big each record was before compression + recordLength = idx.readInt(); + + try{ + //3. key names + //keyNames = (String[]) + idx.readObject(); + //4. lengths of each key + keyLengths = (int[])idx.readObject(); + keyCount = keyLengths.length; + } catch (ClassNotFoundException cnfe) { + throw new WrappedIOException(cnfe); + } + numberOfRecords = idx.readInt(); + inflater = inflaterCache.get(); + lastOffset = idx.readLong(); + keyOffsets = new int[keyCount]; + int cumulativeOffset = 0; + for(int i=0;i " + docid); - String value = m.getItem("docno", docid); - System.out.println(docid + " -> " + value); - System.out.println("Equals check: " + value.equals(args[0])); + if (args[0].equals("print")) + { + IndexUtil.printMetaIndex(index); + } + else + { + MetaIndex m = index.getMetaIndex(); + int docid = m.getDocument("docno", args[0]); + System.out.println(args[0] + " -> " + docid); + String value = m.getItem("docno", docid); + System.out.println(docid + " -> " + value); + System.out.println("Equals check: " + value.equals(args[0])); + } } } Index: src/uk/ac/gla/terrier/structures/indexing/CompressingMetaIndexBuilder.java =================================================================== --- src/uk/ac/gla/terrier/structures/indexing/CompressingMetaIndexBuilder.java (revision 2549) +++ src/uk/ac/gla/terrier/structures/indexing/CompressingMetaIndexBuilder.java (working copy) @@ -147,13 +147,18 @@ w.close(); dataOutput.close(); - indexOutput.writeObject(offsets.toNativeArray()); indexOutput.writeInt(ZIP_COMPRESSION_LEVEL); indexOutput.writeInt(entryLength); indexOutput.writeObject(keyNames); indexOutput.writeObject(valueLens); + final long[] offsetsArray = offsets.toNativeArray(); + indexOutput.writeInt(offsetsArray.length); + for(long v : offsetsArray) + indexOutput.writeLong(v); indexOutput.close(); index.addIndexStructure("meta", "uk.ac.gla.terrier.structures.CompressingMetaIndex", "uk.ac.gla.terrier.structures.Index,java.lang.String", "index,structureName"); + index.addIndexStructureInputStream("meta", "uk.ac.gla.terrier.structures.CompressingMetaIndex$InputStream", "uk.ac.gla.terrier.structures.Index,java.lang.String", "index,structureName"); + //index.setIndexProperty("index.meta.compression-level", ""+ZIP_COMPRESSION_LEVEL); index.setIndexProperty("index.meta.data-source", currentOffset > MAX_MB_IN_MEM_RETRIEVAL * 1024 * 1024 Index: src/uk/ac/gla/terrier/structures/Index.java =================================================================== --- src/uk/ac/gla/terrier/structures/Index.java (revision 2549) +++ src/uk/ac/gla/terrier/structures/Index.java (working copy) @@ -168,7 +168,7 @@ } } - /** for an immultable index, use a normal collection statistics, never changes */ + /** for an immutable index, use a normal collection statistics, never changes */ protected void loadStatistics() { structureCache.put("collectionstatistics", @@ -187,16 +187,22 @@ structureCache.put("collectionstatistics", new UpdatingCollectionStatistics()); } - /** load all index structures */ + /** load all index structures. Is disabled if index property index.preloadIndices.disabled + * is set to true. It is false by default, which means that all non-inputstream indices are loaded + * on initialisation of the index. When the property is true, indices are loaded as required. */ protected void loadIndices() { + final boolean methodDisabled = Boolean.parseBoolean(properties.getProperty("index.preloadIndices.disabled", "false")); + if (methodDisabled) + return; + boolean OK = true; //look for all index structures for (Object oKey: properties.keySet()) { final String sKey = (String)oKey; if (sKey.matches("^index\\..+\\.class$") - && ! (sKey.matches("^index\\..+-inputstream.class$"))) //dont preload input streams + && ! (sKey.matches("^index\\..+-inputstream.class$"))) //don't pre-load input streams { final String structureName = sKey.split("\\.")[1]; Object o = getIndexStructure(structureName); Index: src/uk/ac/gla/terrier/structures/merging/StructureMerger.java =================================================================== --- src/uk/ac/gla/terrier/structures/merging/StructureMerger.java (revision 2549) +++ src/uk/ac/gla/terrier/structures/merging/StructureMerger.java (working copy) @@ -519,26 +519,26 @@ //opening the first set of files. final Iterator docidInput1 = (Iterator)srcIndex1.getIndexStructureInputStream("document"); - final MetaIndex metaInput1 = srcIndex1.getMetaIndex(); + final Iterator metaInput1 = (Iterator)srcIndex1.getIndexStructureInputStream("meta"); //traversing the first set of files, without any change - int sourceDocid = 0; while(docidInput1.hasNext()) { + metaInput1.hasNext(); DocumentIndexEntry die = docidInput1.next(); docidOutput.addEntryToBuffer(die); - metaBuilder.writeDocumentEntry(metaInput1.getAllItems(sourceDocid++)); + metaBuilder.writeDocumentEntry(metaInput1.next()); } - final MetaIndex metaInput2 = srcIndex2.getMetaIndex(); final Iterator docidInput2 = (Iterator)srcIndex2.getIndexStructureInputStream("document"); - sourceDocid = 0; + final Iterator metaInput2 = (Iterator)srcIndex2.getIndexStructureInputStream("meta"); //traversing the 2nd set of files, without any change while(docidInput2.hasNext()) { + metaInput2.hasNext(); DocumentIndexEntry die = docidInput2.next(); docidOutput.addEntryToBuffer(new SimpleDocumentIndexEntry(die)); - metaBuilder.writeDocumentEntry(metaInput2.getAllItems(sourceDocid++)); + metaBuilder.writeDocumentEntry(metaInput2.next()); } docidOutput.finishedCollections();