Index: src/TrecTerrier.java
===================================================================
--- src/TrecTerrier.java	(revision 2563)
+++ src/TrecTerrier.java	(working copy)
@@ -1,575 +0,0 @@
-/*
- * Terrier - Terabyte Retriever 
- * Webpage: http://ir.dcs.gla.ac.uk/terrier 
- * Contact: terrier{a.}dcs.gla.ac.uk
- * University of Glasgow - Department of Computing Science
- * http://www.gla.ac.uk/
- * 
- * The contents of this file are subject to the Mozilla Public License
- * Version 1.1 (the "License"); you may not use this file except in
- * compliance with the License. You may obtain a copy of the License at
- * http://www.mozilla.org/MPL/
- *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
- *
- * The Original Code is TrecTerrier.java.
- *
- * The Original Code is Copyright (C) 2004-2009 the University of Glasgow.
- * All Rights Reserved.
- *
- * Contributor(s):
- *   Vassilis Plachouras <vassilis{a.}dcs.gla.ac.uk> (original author) 
- */
-import java.io.File;
-
-import org.apache.log4j.Logger;
-
-import uk.ac.gla.terrier.applications.HadoopIndexing;
-import uk.ac.gla.terrier.applications.TRECIndexing;
-import uk.ac.gla.terrier.applications.TRECLMIndexing;
-import uk.ac.gla.terrier.applications.TRECLMQuerying;
-import uk.ac.gla.terrier.applications.TRECQuerying;
-import uk.ac.gla.terrier.applications.TRECQueryingExpansion;
-import uk.ac.gla.terrier.evaluation.AdhocEvaluation;
-import uk.ac.gla.terrier.evaluation.Evaluation;
-import uk.ac.gla.terrier.evaluation.NamedPageEvaluation;
-import uk.ac.gla.terrier.structures.DirectIndexInputStream;
-import uk.ac.gla.terrier.structures.Index;
-import uk.ac.gla.terrier.structures.IndexUtil;
-import uk.ac.gla.terrier.structures.InvertedIndexInputStream;
-import uk.ac.gla.terrier.structures.LexiconUtil;
-import uk.ac.gla.terrier.utility.ApplicationSetup;
-/**
- * The text-based application that handles querying 
- * with Terrier, for TREC-like test collections.
- * <code>
-TrecTerrier, indexing TREC collections with Terrier.<br>
-usage: java TrecTerrier [flags in any order]<br>
-<br>
-  -h --help		print this message<br>
-  -V --version	 print version information<br>
-  -i --index	   index a collection<br>
-  -r --retrieve	retrieve from an indexed collection<br>
-  -e --evaluate	evaluates the results in the directory<br>
-				   var/results with the specified qrels file<br>
-				   in the file etc/trec.qrels<br>
-<br>
-If invoked with '-i', then both the direct and<br>
-inverted files are build, unless it is specified which<br>
-of the structures to build.<br>
-  -d --direct	  creates the direct file<br>
-  -v --inverted	creates the inverted file, from an already existing direct<br>
-  -l --langmodel   creates additional structures for language modelling<br>
-<br>
-If invoked with '-r', there are the following options.<br>
-  -c value		 parameter value for term frequency normalisation.<br>
-				   If it is not specified, then the default value for each<br>
-					weighting model is used, eg PL2 =&gt; c=1, BM25 b=&gt; 0.75<br>
-  -q --queryexpand applies query expansion<br>
-  -l --langmodel   applies language modelling<br>
-<br>
-If invoked with '-e', there is the following option.<br>
-  -p --perquery	reports the average precision for each query separately.<br>
-  filename.res	 restrict evaluation to filename.res only.<br>
-<br>
-If invoked with one of the following options, then the contents of the<br>
-corresponding data structure are shown in the standard output.<br>
-  --printdocid	 prints the contents of the document index<br>
-  --printlexicon   prints the contents of the lexicon<br>
-  --printinverted  prints the contents of the inverted file<br>
-  --printdirect	prints the contents of the direct file<br>
-  --printstats	 prints statistics about the indexed collection<br>
-</code
- * 
- * @author Vassilis Plachouras
- * @version $Revision: 1.57 $
- */
-public class TrecTerrier {
-	/** The logger used */
-	private static Logger logger = Logger.getRootLogger();
-	/** The unkown option*/
-	protected String unknownOption;
-	/** The file to evaluation, if any */
-	protected String evaluationFilename = null;
-	
-	/** Specifies whether to apply query expansion*/
-	protected boolean queryexpand;
-	
-	/** Specifies whether language modelling is applied*/
-	protected boolean languagemodel;
-	
-	/** Specifies whether a help message is printed*/
-	protected boolean printHelp;
-	/** Specified whether a version message is printed*/
-	protected boolean printVersion;
-	
-	/** Specifies whether to index a collection*/
-	protected boolean indexing;
-	
-	/**
-	 * Specifies whether to build the inverted file 
-	 * from scrach, sigle pass method	
-	 */
-	protected boolean singlePass = false;
-
-	/** use Hadoop indexing */
-	protected boolean hadoop = false;
-	
-	/** Specifies whether to retrieve from an indexed collection*/
-	protected boolean retrieving;
-	
-	/** Specifies whether to print the document index*/
-	protected boolean printdocid;
-	
-	/** Specifies whether to print the lexicon*/
-	protected boolean printlexicon;
-	
-	/** Specifies whether to print the inverted file*/
-	protected boolean printinverted;
-	
-	/** Specifies whether to print the direct file*/
-	protected boolean printdirect;
-	
-	/** Specifies whether to print the statistics file*/
-	protected boolean printstats;
-	
-	protected boolean printmeta;
-
-	/**
-	  * Specifies whether to perform trec_eval like evaluation,
-	  * reporting only average precision for each query.
-	  */
-	protected boolean evaluation_per_query;
-	/**
-	 * Specifies if the evaluation is done for adhoc or named-page
-	 * finding retrieval task. adhoc by default.
-	 */
-	protected String evaluation_type = "adhoc";
-
-
-	/** 
-	 * Specifies whether to build the inverted file
-	 * from an already created direct file.
-	 */
-	protected boolean inverted;
-	
-	/**
-	 * Specifies whether to build the direct file only.
-	 */
-	protected boolean direct;
-	
-	/** 
-	 * The value of the term frequency 
-	 * normalisation parameter.
-	 */
-	protected double c;
-	
-	/**
-	 * Specifies whether to perform trec_eval like evaluation.
-	 */
-	protected boolean evaluation;
-	
-	/**
-	 * Indicates whether there is a specified 
-	 * value for the term frequency normalisation
-	 * parameter.
-	 */
-	protected boolean isParameterValueSpecified;
-
-	/**
-	 * Prints the version information about Terrier
-	 */
-	protected void version()
-	{
-		System.out.println("TrecTerrier, indexing TREC collections with Terrier. Version "+ApplicationSetup.TERRIER_VERSION);
-		//System.out.println("Built on ");
-	}
-	
-	/**
-	 * Prints a help message that explains the
-	 * possible options.
-	 */
-	protected void usage() {
-		System.out.println("TrecTerrier, indexing TREC collections with Terrier. Version "+ApplicationSetup.TERRIER_VERSION);
-		System.out.println("usage: java TrecTerrier [flags in any order]");
-		System.out.println("");
-		System.out.println("  -h --help		print this message");
-		System.out.println("  -V --version	 print version information");
-		System.out.println("  -i --index	   index a collection");
-		System.out.println("  -r --retrieve	retrieve from an indexed collection");
-		System.out.println("  -e --evaluate	evaluates the results in the directory");
-		System.out.println("				   var/results with the specified qrels file");
-		System.out.println("				   in the file etc/trec.qrels");
-		System.out.println("");
-		System.out.println("If invoked with \'-i\', then both the direct and");
-		System.out.println("inverted files are build, unless it is specified which");
-		System.out.println("of the structures to build.");
-		System.out.println("  -d --direct	  creates the direct file");
-		System.out.println("  -v --inverted	creates the inverted file, from an already existing direct");
-		System.out.println("  -j --ifile	   creates the inverted file, from scratch, single pass");
-		System.out.println("  -H --hadoop	   creates the inverted file, from scratch, using Hadoop MapReduce indexing");
-		System.out.println("  -l --langmodel   creates additional structures for language modelling");
-		System.out.println("");
-		System.out.println("If invoked with \'-r\', there are the following options.");
-		System.out.println("  -c value		 parameter value for term frequency normalisation.");
-		System.out.println("				   If it is not specified, then the default value for each");
-		System.out.println("				   weighting model is used, eg PL2 => c=1, BM25 b=> 0.75");
-		System.out.println("  -q --queryexpand applies query expansion");
-		System.out.println("  -l --langmodel   applies language modelling");
-		System.out.println("");
-		System.out.println("If invoked with \'-e\', there is the following options.");
-		System.out.println("  -p --perquery	reports the average precision for each query separately.");
-		System.out.println("  -n --named		evaluates for the named-page finding task.");
-		System.out.println("  filename.res	 restrict evaluation to filename.res only.");
-		System.out.println("");
-		System.out.println("If invoked with one of the following options, then the contents of the ");
-		System.out.println("corresponding data structure are shown in the standard output.");
-		System.out.println("  --printdocid	 prints the contents of the document index");
-		System.out.println("  --printlexicon   prints the contents of the lexicon");
-		System.out.println("  --printinverted  prints the contents of the inverted file");
-		System.out.println("  --printdirect	prints the contents of the direct file");
-		System.out.println("  --printstats	 prints statistics about the indexed collection");
-		System.out.println("  --printmeta	 prints the contents of the meta structure");
-	}
-	
-	/**
-	 * The main method that starts the application
-	 * @param args the command line arguments
-	 */
-	public static void main(String[] args) {
-		try {
-			TrecTerrier trecTerrier = new TrecTerrier();
-			int status = trecTerrier.processOptions(args);
-			trecTerrier.applyOptions(status);
-		} catch (Exception e) {
-			System.err.println("A problem occurred: "+ e);
-			e.printStackTrace();
-			System.exit(1);
-		} catch (java.lang.OutOfMemoryError oome) {
-			System.err.println(oome);
-			oome.printStackTrace();
-			System.exit(2);
-		}
-	}
-	
-	/**
-	 * Processes the command line arguments and
-	 * sets the corresponding properties accordingly.
-	 * @param args the command line arguments.
-	 * @return int zero if the command line arguments are 
-	 *		 processed successfully, otherwise it returns
-	 *		 an error code.
-	 */
-	protected int processOptions(String[] args) {
-		if (args.length == 0)
-			return ERROR_NO_ARGUMENTS;
-		
-		int pos = 0;
-		while (pos < args.length) {
-			if (args[pos].equals("-h") || args[pos].equals("--help"))
-				printHelp = true;
-			else if (args[pos].equals("-i") || args[pos].equals("--index"))
-				indexing = true;
-			else if (args[pos].equals("-j") || args[pos].equals("--ifile"))
-				singlePass = true;					
-			else if (args[pos].equals("-H") || args[pos].equals("--hadoop"))
-				hadoop = true;
-			else if (args[pos].equals("-r") || args[pos].equals("--retrieve"))
-				retrieving = true;
-			else if (args[pos].equals("-v") || args[pos].equals("--inverted"))
-				inverted = true;
-			else if (args[pos].equals("-d") || args[pos].equals("--direct"))
-				direct = true;
-			else if (args[pos].equals("-q") || args[pos].equals("--queryexpand")) 
-				queryexpand = true;
-			else if (args[pos].equals("-l") || args[pos].equals("--langmodel"))
-				languagemodel = true;
-			else if (args[pos].equals("--printdocid"))
-				printdocid = true;
-			else if (args[pos].equals("-p") || args[pos].equals("--perquery"))
-				evaluation_per_query = true;
-			else if (args[pos].equals("--printlexicon"))
-				printlexicon = true;
-			else if (args[pos].equals("--printinverted"))
-				printinverted = true;
-			else if (args[pos].equals("--printdirect"))
-				printdirect = true;
-			else if (args[pos].equals("--printstats"))
-				printstats = true;
-			else if (args[pos].equals("--printmeta"))
-				printmeta = true;
-			else if (args[pos].equals("-e") || args[pos].equals("--evaluate")){
-				evaluation = true;
-			}
-			else if (args[pos].equals("-n") || args[pos].equals("--named")){
-				evaluation_type = "named";
-			}
-			else if (args[pos].startsWith("-c")) {
-				isParameterValueSpecified = true;
-				if (args[pos].length()==2) { //the next argument is the value
-					if (pos+1<args.length) { //there is another argument
-						pos++;
-						c = Double.parseDouble(args[pos]);
-					} else 
-						return ERROR_NO_C_VALUE;
-				} else { //the value is in the same argument
-					c = Double.parseDouble(args[pos].substring(2));
-				}
-			}
-			else if (evaluation)
-			{
-				evaluationFilename= args[pos];
-			} else {
-				unknownOption = args[pos];
-				return ERROR_UNKNOWN_OPTION;
-			}
-			pos++;
-		}
-		
-		if (isParameterValueSpecified && !retrieving) 
-			return ERROR_GIVEN_C_NOT_RETRIEVING;
-		
-		if ((retrieving || queryexpand || c!=0) && (direct || inverted || indexing))
-			return ERROR_CONFLICTING_ARGUMENTS;		
-
-		if (hadoop && ! indexing)
-			return ERROR_HADOOP_NOT_RETRIEVAL;
-		
-		if (hadoop && (languagemodel || direct || inverted))
-			return ERROR_HADOOP_ONLY_INDEX;
-		
-		if (direct && !indexing) 
-			return ERROR_DIRECT_NOT_INDEXING;
-		
-		if (inverted && !indexing) 
-			return ERROR_INVERTED_NOT_INDEXING;
-		
-		if (queryexpand && !retrieving)
-			return ERROR_EXPAND_NOT_RETRIEVE;
-		
-		return ARGUMENTS_OK;
-	}
-	
-	/**
-	 * Calls the required classes from Terrier.
-	 */
-	public void run() throws Exception {
-		if (printVersion) {
-			version();
-			return;
-		}
-		if (printHelp) { 
-			usage();
-			return;
-		}
-
-		long startTime = System.currentTimeMillis();
-		if (languagemodel && indexing){
-			TRECLMIndexing LMIndexing = new TRECLMIndexing();
-			LMIndexing.createLMIndex();
-		} else if (languagemodel && retrieving) {
-			TRECLMQuerying trecLMQuerying = new TRECLMQuerying();
-			trecLMQuerying.processQueries();
-		}
-		else if (indexing) {
-			if (hadoop) 
-			{
-				try{
-					HadoopIndexing.main(new String[]{});
-				} catch (Exception e) {
-					System.err.println(e);
-					e.printStackTrace();
-					return;
-				}
-			}
-			else
-			{
-				TRECIndexing trecIndexing = new TRECIndexing();
-				if(singlePass)
-					trecIndexing.createSinglePass();	
-				else if (direct)
-					trecIndexing.createDirectFile();
-				else if (inverted) 
-					trecIndexing.createInvertedFile();
-				else { //if none of the options is specified, build both structures
-					trecIndexing.index();
-				}
-			}
-		} else if (retrieving) {
-			//if no value is given, then we use a default value
-			if (queryexpand) {
-				TRECQueryingExpansion trecQueryingExpansion = 
-					new TRECQueryingExpansion();
-				trecQueryingExpansion.processQueries(c, isParameterValueSpecified);
-				trecQueryingExpansion.close();
-			} else {
-				TRECQuerying trecQuerying = new TRECQuerying();
-				trecQuerying.processQueries(c, isParameterValueSpecified);
-				trecQuerying.close();
-			}
-		} else if (printdocid) {
-			Index i = Index.createIndex();
-			IndexUtil.printDocumentIndex(i);
-			i.close();
-		} else if (printmeta) {
-			Index i = Index.createIndex();
-			IndexUtil.printMetaIndex(i);
-			i.close();
-		} else if (printlexicon) {
-			Index i = Index.createIndex();
-			LexiconUtil.printLexicon(i, "lexicon");
-		} else if (printdirect) {
-			Index i = Index.createIndex();
-			if (! i.hasIndexStructureInputStream("direct"))
-			{
-				logger.warn("Sorry, no direct index structure in index");
-			}
-			else
-			{
-			DirectIndexInputStream dirIndex = (DirectIndexInputStream)(i.getIndexStructureInputStream("direct"));
-			dirIndex.print();
-			dirIndex.close();
-			i.close();
-			}
-		} else if (printinverted) {
-			Index i = Index.createIndex();
-			if (i.hasIndexStructureInputStream("inverted"))
-			{
-				InvertedIndexInputStream invIndex = (InvertedIndexInputStream)(i.getIndexStructureInputStream("inverted"));
-				invIndex.print();
-				invIndex.close();
-			}
-			else
-			{
-				logger.warn("Sorry, no inverted index inputstream structure in index");
-			}
-			i.close();
-		} else if (printstats) {
-			Index i = Index.createIndex();
-			if (i == null)
-			{
-				logger.error("No such index : "+ Index.getLastIndexLoadError());
-				
-			}
-			else if(logger.isInfoEnabled()){
-				logger.info("Collection statistics:");
-				logger.info("number of indexed documents: " + i.getCollectionStatistics().getNumberOfDocuments());
-				logger.info("size of vocabulary: " +  i.getCollectionStatistics().getNumberOfUniqueTerms());
-				logger.info("number of tokens: " +  i.getCollectionStatistics().getNumberOfTokens());
-				logger.info("number of pointers: " +  i.getCollectionStatistics().getNumberOfPointers());
-			}
-			i.close();
-		} else if (evaluation) {
-			Evaluation te = null;
-			if (evaluation_type.equals("adhoc"))
-				te = new AdhocEvaluation();
-			else if (evaluation_type.equals("named"))
-				te = new NamedPageEvaluation();
-			String[] nomefile = null;
-			if (evaluationFilename == null)
-			{
-				/* list all the result files and then evaluate them */
-				File fresdirectory = new File(ApplicationSetup.TREC_RESULTS);
-				nomefile = fresdirectory.list();
-			}
-			else
-			{
-				nomefile =new String[]{evaluationFilename}; 
-			}
-			for (int i = 0; i < nomefile.length; i++) {
-				if (nomefile[i].endsWith(".res")) {
-					String resultFilename = ApplicationSetup.TREC_RESULTS+ "/" + nomefile[i];
-					if (nomefile[i].indexOf("/") >= 0)
-						resultFilename = nomefile[i];
-					String evaluationResultFilename =
-						resultFilename.substring(
-							0,
-							resultFilename.lastIndexOf('.'))
-							+ ".eval";
-					te.evaluate(resultFilename);
-					if (evaluation_per_query)
-						te.writeEvaluationResultOfEachQuery(evaluationResultFilename);
-					else
-						te.writeEvaluationResult(evaluationResultFilename);
-				}
-			}
-		}
-		
-		long endTime = System.currentTimeMillis();
-		System.err.println("Time elapsed: " + (endTime-startTime)/1000.0d + " seconds.");
-	}
-	
-	public void applyOptions(int status) throws Exception {
-		switch(status) {
-			case ERROR_NO_ARGUMENTS : 
-				usage();
-				break;
-			case ERROR_NO_C_VALUE :
-				System.err.println("A value for the term frequency normalisation parameter");
-				System.err.println("is required. Please specify it with the option '-c value'");
-				break;
-			case ERROR_CONFLICTING_ARGUMENTS : 
-				System.err.println("There is a conclict between the specified options. For example,");
-				System.err.println("option '-c' is used only in conjuction with option '-r'.");
-				System.err.println("In addition, options '-v' or '-d' are used only in conjuction");
-				System.err.println("with option '-i'");
-				break;
-			case ERROR_PRINT_DOCINDEX_FILE_NOT_EXISTS :
-				System.err.println("The specified document index file does not exist.");
-				break;
-			case ERROR_PRINT_DIRECT_FILE_NOT_EXISTS : 
-				System.err.println("The specified direct index does not exist.");
-				break;
-			case ERROR_UNKNOWN_OPTION : 
-				System.err.println("The option '" +unknownOption+"' is not recognised");
-				break;
-			case ERROR_DIRECT_NOT_INDEXING : 
-				System.err.println("The option '-d' or '--direct' can be used only while indexing with option '-i'.");
-				break;
-			case ERROR_INVERTED_NOT_INDEXING : 
-				System.err.println("The option '-i' or '--inverted' can be used only while indexing with option '-i'.");
-				break;
-			case ERROR_EXPAND_NOT_RETRIEVE : 
-				System.err.println("The option '-q' or '--queryexpand' can be used only while retrieving with option '-r'.");
-				break;
-			case ERROR_LANGUAGEMODEL_NOT_RETRIEVE : 
-				System.err.println("The option '-l' or '--langmodel' can be used only while retrieving with option '-r'.");
-				break;	
-			case ERROR_GIVEN_C_NOT_RETRIEVING : 
-				System.err.println("A value for the parameter c can be specified only while retrieving with option '-r'.");
-				break;
-			case ERROR_HADOOP_NOT_RETRIEVAL : 
-				System.err.println("Hadoop mode '-H' can only be used for indexing");
-				break;
-			case ERROR_HADOOP_ONLY_INDEX :
-				System.err.println("Hadoop mode '-H' can only be used for straightforward indexing");
-				break;
-			case ARGUMENTS_OK : 
-			default :
-				run();			
-		}
-	}
-	
-	protected static final int ARGUMENTS_OK = 0;
-	protected static final int ERROR_NO_ARGUMENTS = 1;
-	protected static final int ERROR_NO_C_VALUE = 2;
-	protected static final int ERROR_CONFLICTING_ARGUMENTS = 3;
-	protected static final int ERROR_DIRECT_FILE_EXISTS = 4;
-	protected static final int ERROR_DIRECT_FILE_NOT_EXISTS = 6;
-	protected static final int ERROR_PRINT_DOCINDEX_FILE_NOT_EXISTS = 7;
-	protected static final int ERROR_PRINT_LEXICON_FILE_NOT_EXISTS = 8;
-	protected static final int ERROR_PRINT_INVERTED_FILE_NOT_EXISTS = 9;
-	protected static final int ERROR_PRINT_STATS_FILE_NOT_EXISTS = 10;
-	protected static final int ERROR_PRINT_DIRECT_FILE_NOT_EXISTS = 11;
-	protected static final int ERROR_UNKNOWN_OPTION = 12;
-	protected static final int ERROR_DIRECT_NOT_INDEXING = 13;
-	protected static final int ERROR_INVERTED_NOT_INDEXING = 14;
-	protected static final int ERROR_EXPAND_NOT_RETRIEVE = 15;
-	protected static final int ERROR_GIVEN_C_NOT_RETRIEVING = 16;
-	protected static final int ERROR_LANGUAGEMODEL_NOT_RETRIEVE = 17;
-	protected static final int ERROR_HADOOP_NOT_RETRIEVAL = 18;
-	protected static final int ERROR_HADOOP_ONLY_INDEX = 19;
-}
Index: src/uk/ac/gla/terrier/indexing/BasicSinglePassIndexer.java
===================================================================
--- src/uk/ac/gla/terrier/indexing/BasicSinglePassIndexer.java	(revision 2571)
+++ src/uk/ac/gla/terrier/indexing/BasicSinglePassIndexer.java	(working copy)
@@ -46,7 +46,10 @@
 import uk.ac.gla.terrier.structures.indexing.singlepass.MemoryPostings;
 import uk.ac.gla.terrier.structures.indexing.singlepass.RunsMerger;
 import uk.ac.gla.terrier.structures.indexing.singlepass.SimplePostingInRun;
+import uk.ac.gla.terrier.structures.postings.BasicIterablePosting;
+import uk.ac.gla.terrier.structures.postings.FieldIterablePosting;
 import uk.ac.gla.terrier.utility.ApplicationSetup;
+import uk.ac.gla.terrier.utility.ArrayUtils;
 import uk.ac.gla.terrier.utility.FieldScore;
 import uk.ac.gla.terrier.utility.Files;
 import uk.ac.gla.terrier.utility.MemoryChecker;
@@ -110,9 +113,11 @@
 	/** Number of pointers indexed */
 	protected long numberOfPointers = 0;
 	/** what class should be used to read the generated inverted index? */
-	protected String invertedIndexClass = "uk.ac.gla.terrier.structures.InvertedIndex";
+	protected String invertedIndexClass = uk.ac.gla.terrier.structures.InvertedIndex.class.getName();
+	protected String basicInvertedIndexPostingIteratorClass = BasicIterablePosting.class.getName();
+	protected String fieldInvertedIndexPostingIteratorClass = FieldIterablePosting.class.getName();
 	/** what class should be used to read the inverted index as a stream? */
-	protected String invertedIndexInputStreamClass = "uk.ac.gla.terrier.structures.InvertedIndexInputStream";
+	protected String invertedIndexInputStreamClass = uk.ac.gla.terrier.structures.InvertedIndexInputStream.class.getName();
 	/**
 	 * Constructs an instance of a BasicSinglePassIndexer, using the given path name
 	 * for storing the data structures.
@@ -357,14 +362,18 @@
 			currentIndex.addIndexStructure(
 					"inverted",
 					invertedIndexClass,
-					"uk.ac.gla.terrier.structures.Index,java.lang.String,uk.ac.gla.terrier.structures.DocumentIndex", 
-					"index,structureName,document");
+					"uk.ac.gla.terrier.structures.Index,java.lang.String,uk.ac.gla.terrier.structures.DocumentIndex,java.lang.Class", 
+					"index,structureName,document,"+ 
+						(FieldScore.FIELDS_COUNT > 0
+							? fieldInvertedIndexPostingIteratorClass
+							: basicInvertedIndexPostingIteratorClass ));
 			currentIndex.addIndexStructureInputStream(
                     "inverted",
                     invertedIndexInputStreamClass,
                     "uk.ac.gla.terrier.structures.Index,java.lang.String,java.util.Iterator",
                     "index,structureName,lexicon-entry-inputstream");
-			currentIndex.setIndexProperty("num.inverted.fields.bits", ""+FieldScore.FIELDS_COUNT );
+			currentIndex.setIndexProperty("index.inverted.fields.count", ""+FieldScore.FIELDS_COUNT );
+			currentIndex.setIndexProperty("index.inverted.fields.names", ArrayUtils.join(FieldScore.FIELD_NAMES, ","));
 		}catch(Exception e){
 			logger.error("Problem in performMultiWayMerge", e);
 		}
@@ -397,7 +406,8 @@
 	 * @throws IOException if an I/O error occurs.
 	 */
 	protected void createRunMerger(String[][] files) throws Exception{
-		merger = new RunsMerger(new FileRunIteratorFactory(files, SimplePostingInRun.class));
+		merger = new RunsMerger(new FileRunIteratorFactory(files, 
+				useFieldInformation ? SimplePostingInRun.class : FieldPostingInRun.class));
 	}
 
 	/**
Index: src/uk/ac/gla/terrier/indexing/BasicIndexer.java
===================================================================
--- src/uk/ac/gla/terrier/indexing/BasicIndexer.java	(revision 2563)
+++ src/uk/ac/gla/terrier/indexing/BasicIndexer.java	(working copy)
@@ -26,19 +26,29 @@
  */
 package uk.ac.gla.terrier.indexing;
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.Map;
 import java.util.Set;
 
+import uk.ac.gla.terrier.compression.BitIn;
 import uk.ac.gla.terrier.structures.BasicDocumentIndexEntry;
+import uk.ac.gla.terrier.structures.BasicLexiconEntry;
 import uk.ac.gla.terrier.structures.BitIndexPointer;
+import uk.ac.gla.terrier.structures.DirectInvertedOutputStream;
 import uk.ac.gla.terrier.structures.DocumentIndexEntry;
+import uk.ac.gla.terrier.structures.FieldDirectInvertedOutputStream;
+import uk.ac.gla.terrier.structures.FieldLexiconEntry;
 import uk.ac.gla.terrier.structures.Index;
-import uk.ac.gla.terrier.structures.indexing.DirectIndexBuilder;
 import uk.ac.gla.terrier.structures.indexing.DocumentIndexBuilder;
 import uk.ac.gla.terrier.structures.indexing.DocumentPostingList;
+import uk.ac.gla.terrier.structures.indexing.FieldDocumentPostingList;
+import uk.ac.gla.terrier.structures.indexing.FieldLexiconMap;
 import uk.ac.gla.terrier.structures.indexing.InvertedIndexBuilder;
 import uk.ac.gla.terrier.structures.indexing.LexiconBuilder;
+import uk.ac.gla.terrier.structures.indexing.LexiconMap;
 import uk.ac.gla.terrier.terms.TermPipeline;
+import uk.ac.gla.terrier.utility.ApplicationSetup;
+import uk.ac.gla.terrier.utility.ArrayUtils;
 import uk.ac.gla.terrier.utility.FieldScore;
 import uk.ac.gla.terrier.utility.TermCodes;
 /** 
@@ -100,18 +110,19 @@
 			{
 				/* add term to Document tree */
 				final int[] fieldIds = new int[numFields];
+				Arrays.fill(fieldIds, -1);
 				int i=0;
 				for (String fieldName: termFields)
 				{
-					fieldIds[i] = FieldNames.get(fieldName);
+					fieldIds[i] = -1 + FieldNames.get(fieldName);
 					i++;
 				}
-				termsInDocument.insert(term,fieldIds);
+				((FieldDocumentPostingList)termsInDocument).insert(term,fieldIds);
 				numOfTokensInDocument++;
 			}
 		}
 	}
-
+	
 	/** 
 	 * A private variable for storing the fields a term appears into.
 	 */
@@ -175,8 +186,17 @@
 	public void createDirectIndex(Collection[] collections)
 	{
 		currentIndex = Index.createNewIndex(path, prefix);
-		lexiconBuilder = new LexiconBuilder(currentIndex, "lexicon");
-		directIndexBuilder = new DirectIndexBuilder(currentIndex, "direct");
+		lexiconBuilder = FieldScore.FIELDS_COUNT > 0
+			? new LexiconBuilder(currentIndex, "lexicon", new FieldLexiconMap(FieldScore.FIELDS_COUNT), FieldLexiconEntry.class.getName())
+			: new LexiconBuilder(currentIndex, "lexicon", new LexiconMap(), BasicLexiconEntry.class.getName());
+		try{
+			directIndexBuilder = FieldScore.FIELDS_COUNT > 0
+				? new FieldDirectInvertedOutputStream(currentIndex.getPath() + ApplicationSetup.FILE_SEPARATOR + currentIndex.getPrefix() + "." + "direct" + BitIn.USUAL_EXTENSION)
+				: new DirectInvertedOutputStream(currentIndex.getPath() + ApplicationSetup.FILE_SEPARATOR + currentIndex.getPrefix() + "." + "direct" + BitIn.USUAL_EXTENSION);
+		} catch (IOException ioe) {
+			logger.error("Cannot make DirectInvertedOutputStream:", ioe);
+		}
+			//	new DirectIndexBuilder(currentIndex, "direct");
 		docIndexBuilder = new DocumentIndexBuilder(currentIndex, "document");
 		metaBuilder = createMetaIndexBuilder();
 				
@@ -264,7 +284,22 @@
 		finishedDirectIndexBuild();
 		/*end of all the collections has been reached */
 		/* flush the index buffers */
-		directIndexBuilder.finishedCollections();
+		currentIndex.addIndexStructure(
+				"direct", 
+				"uk.ac.gla.terrier.structures.DirectIndex", 
+				"uk.ac.gla.terrier.structures.Index,java.lang.String", 
+				"index,structureName");
+		currentIndex.addIndexStructureInputStream(
+				"direct",
+				"uk.ac.gla.terrier.structures.DirectIndexInputStream", 
+				"uk.ac.gla.terrier.structures.Index,java.lang.String,java.lang.Class",
+				"index,structureName,"+ 
+					(FieldScore.FIELDS_COUNT > 0 ? fieldDirectIndexPostingIteratorClass : basicDirectIndexPostingIteratorClass));
+		currentIndex.setIndexProperty("index.direct.fields.count", ""+FieldScore.FIELDS_COUNT );
+		currentIndex.setIndexProperty("index.direct.fields.names", ArrayUtils.join(FieldScore.FIELD_NAMES, ","));
+		
+		//directIndexBuilder.finishedCollections();
+		directIndexBuilder.close();
 		docIndexBuilder.finishedCollections();
 		currentIndex.addIndexStructure("document-factory", BasicDocumentIndexEntry.Factory.class.getName(), "", "");
 		try{
@@ -304,7 +339,8 @@
 			/* add words to lexicontree */
 			lexiconBuilder.addDocumentTerms(termsInDocument);
 			/* add doc postings to the direct index */
-			BitIndexPointer dirIndexPost = directIndexBuilder.addDocument(termsInDocument.getPostings());
+			BitIndexPointer dirIndexPost = directIndexBuilder.writePostings(termsInDocument.getPostings2());
+				//.addDocument(termsInDocument.getPostings());
 			/* add doc to documentindex */
 			DocumentIndexEntry die = termsInDocument.getDocumentStatistics();
 			die.setBitIndexPointer(dirIndexPost);
@@ -367,7 +403,10 @@
 	 * Hook method that creates the right type of DocumentTree class.
 	 */
 	protected void createDocumentPostings(){
-		termsInDocument = new DocumentPostingList(FieldScore.FIELDS_COUNT);
+		if (FieldScore.FIELDS_COUNT > 0)
+			termsInDocument = new FieldDocumentPostingList(FieldScore.FIELDS_COUNT);
+		else
+			termsInDocument = new DocumentPostingList();		
 	}
 
 	/** Hook method, called when the inverted index is finished - ie the lexicon is finished */
Index: src/uk/ac/gla/terrier/indexing/BlockIndexer.java
===================================================================
--- src/uk/ac/gla/terrier/indexing/BlockIndexer.java	(revision 2563)
+++ src/uk/ac/gla/terrier/indexing/BlockIndexer.java	(working copy)
@@ -29,22 +29,31 @@
 import gnu.trove.THashSet;
 
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.Map;
 import java.util.Set;
 
+import uk.ac.gla.terrier.compression.BitIn;
 import uk.ac.gla.terrier.structures.BasicDocumentIndexEntry;
 import uk.ac.gla.terrier.structures.BitIndexPointer;
+import uk.ac.gla.terrier.structures.BlockDirectInvertedOutputStream;
+import uk.ac.gla.terrier.structures.BlockLexiconEntry;
 import uk.ac.gla.terrier.structures.DocumentIndexEntry;
+import uk.ac.gla.terrier.structures.FieldLexiconEntry;
 import uk.ac.gla.terrier.structures.Index;
-import uk.ac.gla.terrier.structures.indexing.BlockDirectIndexBuilder;
 import uk.ac.gla.terrier.structures.indexing.BlockDocumentPostingList;
+import uk.ac.gla.terrier.structures.indexing.BlockFieldDocumentPostingList;
+import uk.ac.gla.terrier.structures.indexing.BlockFieldLexiconMap;
 import uk.ac.gla.terrier.structures.indexing.BlockInvertedIndexBuilder;
-import uk.ac.gla.terrier.structures.indexing.BlockLexiconBuilder;
+import uk.ac.gla.terrier.structures.indexing.BlockLexiconMap;
 import uk.ac.gla.terrier.structures.indexing.DocumentIndexBuilder;
 import uk.ac.gla.terrier.structures.indexing.DocumentPostingList;
 import uk.ac.gla.terrier.structures.indexing.LexiconBuilder;
+import uk.ac.gla.terrier.structures.postings.BlockFieldIterablePosting;
+import uk.ac.gla.terrier.structures.postings.BlockIterablePosting;
 import uk.ac.gla.terrier.terms.TermPipeline;
 import uk.ac.gla.terrier.utility.ApplicationSetup;
+import uk.ac.gla.terrier.utility.ArrayUtils;
 import uk.ac.gla.terrier.utility.FieldScore;
 import uk.ac.gla.terrier.utility.TermCodes;
 /**
@@ -83,7 +92,7 @@
 			//	null means the term has been filtered out (eg stopwords)
 			if (t != null) {
 				//add term to thingy tree
-				termsInDocument.insert(t, blockId);
+				((BlockDocumentPostingList)termsInDocument).insert(t, blockId);
 				numOfTokensInDocument++;
 				if (++numOfTokensInBlock >= BLOCK_SIZE && blockId < MAX_BLOCKS) {
 					numOfTokensInBlock = 0;
@@ -103,13 +112,14 @@
 			if (t != null) {
 				//add term to document posting list
 				final int[] fieldIds = new int[numFields];
+				Arrays.fill(fieldIds, -1);
 				int i=0;
 				for (String fieldName: termFields)
 				{
-					fieldIds[i] = FieldNames.get(fieldName);
+					fieldIds[i] = -1 + FieldNames.get(fieldName);
 					i++;
 				}
-				termsInDocument.insert(t,fieldIds, blockId);
+				((BlockFieldDocumentPostingList)termsInDocument).insert(t,fieldIds, blockId);
 				numOfTokensInDocument++;
 				if (++numOfTokensInBlock >= BLOCK_SIZE && blockId < MAX_BLOCKS) {
 					numOfTokensInBlock = 0;
@@ -145,7 +155,7 @@
 			if (blockDelimiterTerms.contains(t)) {
 				// delimiters should also be indexed
 				if (indexDelimiters) {
-						termsInDocument.insert(t, blockId);
+						((BlockDocumentPostingList)termsInDocument).insert(t, blockId);
 						if (countDelimiters)
 								numOfTokensInDocument++;
 				}
@@ -154,7 +164,7 @@
 			}
 			else {
 				// index non-delimiter term
-				termsInDocument.insert(t, blockId);
+				((BlockDocumentPostingList)termsInDocument).insert(t, blockId);
 				numOfTokensInDocument++;
 			}
 		}
@@ -194,7 +204,7 @@
 						fieldIds[i] = FieldNames.get(fieldName);
 						i++;
 					}
-					termsInDocument.insert(t, fieldIds, blockId);
+					((BlockFieldDocumentPostingList)termsInDocument).insert(t, fieldIds, blockId);
 					if (countDelimiters)
 						numOfTokensInDocument++;
 				}
@@ -210,7 +220,7 @@
 					fieldIds[i] = FieldNames.get(fieldName);
 					i++;
 				}
-				termsInDocument.insert(t, fieldIds, blockId);
+				((BlockFieldDocumentPostingList)termsInDocument).insert(t, fieldIds, blockId);
 				numOfTokensInDocument++;
 			}
 		}
@@ -225,7 +235,7 @@
 	/** The fields that are set for the current term. */
 	protected Set<String> termFields = null;
 	/** The list of terms in this document, and for each, the block occurrences. */
-	protected BlockDocumentPostingList termsInDocument = null;
+	protected DocumentPostingList termsInDocument = null;
 	/** The maximum number of terms allowed in a block. See Property <tt>blocks.size</tt> */
 	protected int BLOCK_SIZE;
 	/** 
@@ -242,6 +252,8 @@
 		super(pathname, prefix);
 		if (this.getClass() == BlockIndexer.class)
 			init();
+		basicDirectIndexPostingIteratorClass = BlockIterablePosting.class.getName();
+		fieldDirectIndexPostingIteratorClass = BlockFieldIterablePosting.class.getName();
 	}
 
 	/** 
@@ -284,8 +296,16 @@
 			(Boolean.parseBoolean(ApplicationSetup.getProperty("block.delimiters.enabled", "false"))
 			? " delimited-block indexing enabled" : ""));
 		currentIndex = Index.createNewIndex(path, prefix);
-		lexiconBuilder = new BlockLexiconBuilder(currentIndex, "lexicon");
-		directIndexBuilder = new BlockDirectIndexBuilder(currentIndex, "direct");
+		lexiconBuilder = FieldScore.FIELDS_COUNT > 0
+			? new LexiconBuilder(currentIndex, "lexicon", new BlockFieldLexiconMap(FieldScore.FIELDS_COUNT), FieldLexiconEntry.class.getName())
+			: new LexiconBuilder(currentIndex, "lexicon", new BlockLexiconMap(), BlockLexiconEntry.class.getName());
+		//lexiconBuilder = new BlockLexiconBuilder(currentIndex, "lexicon");
+		try{
+			directIndexBuilder = new BlockDirectInvertedOutputStream(currentIndex.getPath() + ApplicationSetup.FILE_SEPARATOR + currentIndex.getPrefix() + "." + "direct" + BitIn.USUAL_EXTENSION);
+		} catch (IOException ioe) {
+			logger.error("Cannot make DirectInvertedOutputStream:", ioe);
+		}
+		//directIndexBuilder =  new BlockDirectInvertedOutputStream(); //new BlockDirectIndexBuilder(currentIndex, "direct");
 		docIndexBuilder = new DocumentIndexBuilder(currentIndex, "document");
 		metaBuilder = createMetaIndexBuilder();
 		
@@ -371,8 +391,22 @@
 
 		/* end of the collection has been reached */
 		finishedDirectIndexBuild();
+		currentIndex.addIndexStructure(
+				"direct", 
+				"uk.ac.gla.terrier.structures.BlockDirectIndex", 
+				"uk.ac.gla.terrier.structures.Index,java.lang.String", 
+				"index,structureName");
+		currentIndex.addIndexStructureInputStream(
+				"direct",
+				"uk.ac.gla.terrier.structures.BlockDirectIndexInputStream", 
+				"uk.ac.gla.terrier.structures.Index,java.lang.String,java.lang.Class",
+				"index,structureName,"+ 
+					(FieldScore.FIELDS_COUNT > 0 ? fieldDirectIndexPostingIteratorClass : basicDirectIndexPostingIteratorClass));
+		currentIndex.setIndexProperty("index.direct.fields.count", ""+FieldScore.FIELDS_COUNT );
+		currentIndex.setIndexProperty("index.direct.fields.names", ArrayUtils.join(FieldScore.FIELD_NAMES, ","));
+		
 		/* flush the index buffers */
-		directIndexBuilder.finishedCollections();
+		directIndexBuilder.close();
 		docIndexBuilder.finishedCollections();
 		currentIndex.addIndexStructure("document-factory", BasicDocumentIndexEntry.Factory.class.getName(), "", "");
 		/* and then merge all the temporary lexicons */
@@ -407,7 +441,7 @@
 			/* add words to lexicontree */
 			lexiconBuilder.addDocumentTerms(termsInDocument);
 			/* add doc postings to the direct index */
-			BitIndexPointer dirIndexPost = directIndexBuilder.addDocument(termsInDocument.getPostings());
+			BitIndexPointer dirIndexPost = directIndexBuilder.writePostings(termsInDocument.getPostings2());
 			/* add doc to documentindex */
 			DocumentIndexEntry die = termsInDocument.getDocumentStatistics();
 			die.setBitIndexPointer(dirIndexPost);
@@ -474,7 +508,10 @@
 
 	
 	protected void createDocumentPostings(){
-		termsInDocument = new BlockDocumentPostingList(FieldScore.FIELDS_COUNT);		
+		if (FieldScore.FIELDS_COUNT > 0)
+			termsInDocument = new BlockFieldDocumentPostingList(FieldScore.FIELDS_COUNT);
+		else
+			termsInDocument = new BlockDocumentPostingList();
 		blockId = 0;
 		numOfTokensInBlock = 0;	
 	}
Index: src/uk/ac/gla/terrier/indexing/Indexer.java
===================================================================
--- src/uk/ac/gla/terrier/indexing/Indexer.java	(revision 2563)
+++ src/uk/ac/gla/terrier/indexing/Indexer.java	(working copy)
@@ -34,17 +34,19 @@
 import org.apache.log4j.Logger;
 
 import uk.ac.gla.terrier.structures.BasicDocumentIndexEntry;
+import uk.ac.gla.terrier.structures.DirectInvertedOutputStream;
 import uk.ac.gla.terrier.structures.DocumentIndexEntry;
 import uk.ac.gla.terrier.structures.Index;
 import uk.ac.gla.terrier.structures.IndexUtil;
 import uk.ac.gla.terrier.structures.indexing.CompressingMetaIndexBuilder;
-import uk.ac.gla.terrier.structures.indexing.DirectIndexBuilder;
 import uk.ac.gla.terrier.structures.indexing.DocumentIndexBuilder;
 import uk.ac.gla.terrier.structures.indexing.InvertedIndexBuilder;
 import uk.ac.gla.terrier.structures.indexing.LexiconBuilder;
 import uk.ac.gla.terrier.structures.indexing.MetaIndexBuilder;
 import uk.ac.gla.terrier.structures.merging.BlockStructureMerger;
 import uk.ac.gla.terrier.structures.merging.StructureMerger;
+import uk.ac.gla.terrier.structures.postings.BasicIterablePosting;
+import uk.ac.gla.terrier.structures.postings.FieldIterablePosting;
 import uk.ac.gla.terrier.terms.SkipTermPipeline;
 import uk.ac.gla.terrier.terms.TermPipeline;
 import uk.ac.gla.terrier.utility.ApplicationSetup;
@@ -115,7 +117,7 @@
 	/**
 	 * The builder that creates the direct index.
 	 */
-	protected DirectIndexBuilder directIndexBuilder;
+	protected DirectInvertedOutputStream directIndexBuilder;
 	
 	/**
 	 * The builder that creates the document index.
@@ -148,6 +150,9 @@
 	/** The index being worked on, denoted by path and prefix */
 	protected Index currentIndex = null;
 
+	protected String basicDirectIndexPostingIteratorClass = BasicIterablePosting.class.getName();
+	protected String fieldDirectIndexPostingIteratorClass = FieldIterablePosting.class.getName();
+	
 	public Indexer()
 	{
 		this(ApplicationSetup.TERRIER_INDEX_PATH, ApplicationSetup.TERRIER_INDEX_PREFIX);
@@ -175,6 +180,7 @@
 	  */
 	protected void init()
 	{
+		FieldScore.init();
 		//construct pipeline using list specified in terrier.properties
         //this object should be the last item in the pipeline
 		this.load_indexer_properties();
Index: src/uk/ac/gla/terrier/indexing/BlockSinglePassIndexer.java
===================================================================
--- src/uk/ac/gla/terrier/indexing/BlockSinglePassIndexer.java	(revision 2563)
+++ src/uk/ac/gla/terrier/indexing/BlockSinglePassIndexer.java	(working copy)
@@ -29,15 +29,23 @@
 
 package uk.ac.gla.terrier.indexing;
 
-import java.io.IOException;
 import gnu.trove.THashSet;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+import uk.ac.gla.terrier.structures.BlockInvertedIndex;
+import uk.ac.gla.terrier.structures.BlockInvertedIndexInputStream;
 import uk.ac.gla.terrier.structures.indexing.BlockDocumentPostingList;
+import uk.ac.gla.terrier.structures.indexing.BlockFieldDocumentPostingList;
 import uk.ac.gla.terrier.structures.indexing.singlepass.BlockFieldMemoryPostings;
 import uk.ac.gla.terrier.structures.indexing.singlepass.BlockFieldPostingInRun;
 import uk.ac.gla.terrier.structures.indexing.singlepass.BlockMemoryPostings;
 import uk.ac.gla.terrier.structures.indexing.singlepass.BlockPostingInRun;
 import uk.ac.gla.terrier.structures.indexing.singlepass.FileRunIteratorFactory;
 import uk.ac.gla.terrier.structures.indexing.singlepass.RunsMerger;
+import uk.ac.gla.terrier.structures.postings.BlockFieldIterablePosting;
+import uk.ac.gla.terrier.structures.postings.BlockIterablePosting;
 import uk.ac.gla.terrier.terms.TermPipeline;
 import uk.ac.gla.terrier.utility.ApplicationSetup;
 import uk.ac.gla.terrier.utility.FieldScore;
@@ -80,13 +88,14 @@
 			if (t != null) {
 				//add term to document posting list
 				final int[] fieldIds = new int[numFields];
+				Arrays.fill(fieldIds, -1);
 				int i=0;
 				for (String fieldName: termFields)
 				{
-					fieldIds[i] = FieldNames.get(fieldName);
+					fieldIds[i] = -1 + FieldNames.get(fieldName);
 					i++;
 				}
-				((BlockDocumentPostingList)termsInDocument).insert(t,fieldIds, blockId);
+				((BlockFieldDocumentPostingList)termsInDocument).insert(t,fieldIds, blockId);
 				numOfTokensInDocument++;
 				if (++numOfTokensInBlock >= BLOCK_SIZE && blockId < MAX_BLOCKS) {
 					numOfTokensInBlock = 0;
@@ -171,7 +180,7 @@
 						fieldIds[i] = FieldNames.get(fieldName);
 						i++;
 					}
-					((BlockDocumentPostingList)termsInDocument).insert(t, fieldIds, blockId);
+					((BlockFieldDocumentPostingList)termsInDocument).insert(t, fieldIds, blockId);
 					if (countDelimiters)
 						numOfTokensInDocument++;
 				}
@@ -187,7 +196,7 @@
 					fieldIds[i] = FieldNames.get(fieldName);
 					i++;
 				}
-				((BlockDocumentPostingList)termsInDocument).insert(t, fieldIds, blockId);
+				((BlockFieldDocumentPostingList)termsInDocument).insert(t, fieldIds, blockId);
 				numOfTokensInDocument++;
 			}
 		}
@@ -239,8 +248,10 @@
 		//delay the execution of init() if we are a parent class
         if (this.getClass() == BlockSinglePassIndexer.class) 
             init();
-		invertedIndexClass = "uk.ac.gla.terrier.structures.BlockInvertedIndex";
-		invertedIndexInputStreamClass =  "uk.ac.gla.terrier.structures.BlockInvertedIndexInputStream";
+		invertedIndexClass = BlockInvertedIndex.class.getName();
+		invertedIndexInputStreamClass = BlockInvertedIndexInputStream.class.getName();
+		basicInvertedIndexPostingIteratorClass = BlockIterablePosting.class.getName();
+		fieldInvertedIndexPostingIteratorClass = BlockFieldIterablePosting.class.getName();
 	}
 	
 	protected void createFieldRunMerger(String[][] files) throws IOException{
@@ -260,7 +271,10 @@
 
 
 	protected void createDocumentPostings(){
-		termsInDocument = new BlockDocumentPostingList(FieldScore.FIELDS_COUNT);
+		if (FieldScore.FIELDS_COUNT > 0)
+			termsInDocument = new BlockFieldDocumentPostingList(FieldScore.FIELDS_COUNT);
+		else
+			termsInDocument = new BlockDocumentPostingList();
 		blockId = 0;
 		numOfTokensInBlock = 0;
 	}
Index: src/uk/ac/gla/terrier/indexing/hadoop/Hadoop_BasicSinglePassIndexer.java
===================================================================
--- src/uk/ac/gla/terrier/indexing/hadoop/Hadoop_BasicSinglePassIndexer.java	(revision 2571)
+++ src/uk/ac/gla/terrier/indexing/hadoop/Hadoop_BasicSinglePassIndexer.java	(working copy)
@@ -74,6 +74,7 @@
 import uk.ac.gla.terrier.structures.indexing.singlepass.hadoop.MapEmittedTerm;
 import uk.ac.gla.terrier.structures.indexing.singlepass.hadoop.SplitAwareWrapper;
 import uk.ac.gla.terrier.utility.ApplicationSetup;
+import uk.ac.gla.terrier.utility.ArrayUtils;
 import uk.ac.gla.terrier.utility.FieldScore;
 import uk.ac.gla.terrier.utility.Files;
 import uk.ac.gla.terrier.utility.io.HadoopPlugin;
@@ -585,7 +586,8 @@
 	            invertedIndexInputStreamClass,
                 "uk.ac.gla.terrier.structures.Index,java.lang.String,java.util.Iterator",
                 "index,structureName,lexicon-entry-inputstream");
-		currentIndex.setIndexProperty("num.inverted.fields.bits", ""+FieldScore.FIELDS_COUNT );
+		currentIndex.setIndexProperty("index.inverted.fields.count", ""+FieldScore.FIELDS_COUNT );
+		currentIndex.setIndexProperty("index.inverted.fields.names", ArrayUtils.join(FieldScore.FIELD_NAMES, ","));
 		
 		//3. document index
 		Index[] sourceIndices = new Index[MapIndexPrefixes.length];
Index: src/uk/ac/gla/terrier/indexing/hadoop/Hadoop_BlockSinglePassIndexer.java
===================================================================
--- src/uk/ac/gla/terrier/indexing/hadoop/Hadoop_BlockSinglePassIndexer.java	(revision 2563)
+++ src/uk/ac/gla/terrier/indexing/hadoop/Hadoop_BlockSinglePassIndexer.java	(working copy)
@@ -29,10 +29,13 @@
 
 package uk.ac.gla.terrier.indexing.hadoop;
 
-import java.io.IOException;
 import gnu.trove.THashSet;
+
+import java.io.IOException;
+
 import uk.ac.gla.terrier.compression.BitOutputStream;
 import uk.ac.gla.terrier.structures.indexing.BlockDocumentPostingList;
+import uk.ac.gla.terrier.structures.indexing.BlockFieldDocumentPostingList;
 import uk.ac.gla.terrier.structures.indexing.singlepass.BlockFieldMemoryPostings;
 import uk.ac.gla.terrier.structures.indexing.singlepass.BlockFieldPostingInRun;
 import uk.ac.gla.terrier.structures.indexing.singlepass.BlockMemoryPostings;
@@ -87,7 +90,7 @@
 					fieldIds[i] = FieldNames.get(fieldName);
 					i++;
 				}
-				((BlockDocumentPostingList)termsInDocument).insert(t,fieldIds, blockId);
+				((BlockFieldDocumentPostingList)termsInDocument).insert(t,fieldIds, blockId);
 				numOfTokensInDocument++;
 				if (++numOfTokensInBlock >= BLOCK_SIZE && blockId < MAX_BLOCKS) {
 					numOfTokensInBlock = 0;
@@ -172,7 +175,7 @@
 						fieldIds[i] = FieldNames.get(fieldName);
 						i++;
 					}
-					((BlockDocumentPostingList)termsInDocument).insert(t, fieldIds, blockId);
+					((BlockFieldDocumentPostingList)termsInDocument).insert(t, fieldIds, blockId);
 					if (countDelimiters)
 						numOfTokensInDocument++;
 				}
@@ -188,7 +191,7 @@
 					fieldIds[i] = FieldNames.get(fieldName);
 					i++;
 				}
-				((BlockDocumentPostingList)termsInDocument).insert(t, fieldIds, blockId);
+				((BlockFieldDocumentPostingList)termsInDocument).insert(t, fieldIds, blockId);
 				numOfTokensInDocument++;
 			}
 		}
@@ -250,7 +253,10 @@
 
 
 	protected void createDocumentPostings(){
-		termsInDocument = new BlockDocumentPostingList(FieldScore.FIELDS_COUNT);
+		if (FieldScore.FIELDS_COUNT > 0)
+			termsInDocument = new BlockFieldDocumentPostingList(FieldScore.FIELDS_COUNT);
+		else
+			termsInDocument = new BlockDocumentPostingList();
 		blockId = 0;
 		numOfTokensInBlock = 0;
 	}
Index: src/uk/ac/gla/terrier/structures/BlockDirectInvertedOutputStream.java
===================================================================
--- src/uk/ac/gla/terrier/structures/BlockDirectInvertedOutputStream.java	(revision 2563)
+++ src/uk/ac/gla/terrier/structures/BlockDirectInvertedOutputStream.java	(working copy)
@@ -42,20 +42,19 @@
 	/** Creates a new output stream, writing a BitOutputStream to the specified file. The number of binary bits
 	  * for fields must also be specified.
 	  * @param filename Location of the file to write to
-	  * @param binaryBits the number of fields in this index 
 	  */
-	public BlockDirectInvertedOutputStream(String filename, int binaryBits) throws IOException
+	public BlockDirectInvertedOutputStream(String filename) throws IOException
 	{
-		super(filename, binaryBits);
+		super(filename);
 	}
 	/** Creates a new output stream, writing to the specified BitOut implementation.  The number of binary bits
 	  * for fields must also be specified.
 	  * @param out BitOut implementation to write the file to 
 	  * @param binaryBits the number of fields in this index 
 	  */
-	public BlockDirectInvertedOutputStream(BitOut out, int binaryBits)
+	public BlockDirectInvertedOutputStream(BitOut out)
 	{
-		super(out, binaryBits);
+		super(out);
 	}
 	
 	/**
@@ -68,7 +67,7 @@
 	 *        an id plus one, or the gap of the current id and the previous one.
 	 * @override
 	 */
-	
+	/*
 	protected BitIndexPointer  writeFieldPostings(final int[][] postings, int offset, final int Length, final int firstId)
 		throws IOException {
 		
@@ -127,7 +126,7 @@
 			}
 		}
 		return rtr;
-	}
+	}*/
 
 	/**
 	 * Writes the given block postings to the bit file. This method assumes that
Index: src/uk/ac/gla/terrier/structures/BitPostingIndex.java
===================================================================
--- src/uk/ac/gla/terrier/structures/BitPostingIndex.java	(revision 2563)
+++ src/uk/ac/gla/terrier/structures/BitPostingIndex.java	(working copy)
@@ -6,13 +6,14 @@
 import uk.ac.gla.terrier.compression.BitIn;
 import uk.ac.gla.terrier.compression.BitInSeekable;
 import uk.ac.gla.terrier.structures.postings.IterablePosting;
-import uk.ac.gla.terrier.utility.FieldScore;
 import uk.ac.gla.terrier.utility.io.WrappedIOException;
 
 abstract class BitPostingIndex implements PostingIndex
 {
 	protected BitInSeekable file;
 	protected Class<? extends IterablePosting> postingImplementation;
+	protected Index index = null;
+	protected int fieldCount = 0;
 	
 	protected BitPostingIndex(String filename, Class<? extends IterablePosting> _postingImplementation)
 		throws IOException
@@ -28,21 +29,23 @@
 		throws IOException
 	{
 		this(_index.getPath() + "/" + _index.getPrefix() + "." + _structureName + BitIn.USUAL_EXTENSION, _postingImplementation);
+		index = _index;
+		fieldCount = index.getIntIndexProperty("index."+_structureName+".fields.count", 0);
 	}
 	
 	public IterablePosting getPostings(BitIndexPointer pointer) throws IOException
 	{
 		final BitIn file = this.file.readReset(pointer.getOffset(), pointer.getOffsetBits());
-		final int fieldCount = FieldScore.FIELDS_COUNT;
-		if (fieldCount > 0)
-		{
-			throw new IOException("Fields not support this way");
-		}
 		IterablePosting rtr = null;
 		try{
-			rtr = postingImplementation
-				.getConstructor(BitIn.class, Integer.TYPE, DocumentIndex.class)
-				.newInstance(file, pointer.getNumberOfEntries(), null);
+			if (fieldCount > 0)
+				rtr = postingImplementation
+					.getConstructor(BitIn.class, Integer.TYPE, DocumentIndex.class, Integer.TYPE)
+					.newInstance(file, pointer.getNumberOfEntries(), null, fieldCount);
+			else
+				rtr = postingImplementation
+					.getConstructor(BitIn.class, Integer.TYPE, DocumentIndex.class)
+					.newInstance(file, pointer.getNumberOfEntries(), null);
 		} catch (Exception e) {
 			throw new WrappedIOException(e);
 		}
Index: src/uk/ac/gla/terrier/structures/SingleLineTRECQuery.java
===================================================================
--- src/uk/ac/gla/terrier/structures/SingleLineTRECQuery.java	(revision 2563)
+++ src/uk/ac/gla/terrier/structures/SingleLineTRECQuery.java	(working copy)
@@ -99,6 +99,15 @@
 				String query;
 				if (QueryLineHasQueryID)
 				{
+					final int queryIdEnd = minOver0(new int[]{ line.indexOf(' '), line.indexOf('\t'), line.indexOf(":")});
+					if (queryIdEnd == -1)
+					{
+						//no query Id found
+						continue;
+					}
+					queryID = line.substring(0,queryIdEnd);
+					query = line.substring(queryIdEnd+1);
+					/*if ()
 					String parts[] = line.split("\\s+|:");
 					queryID = parts[0];
 					StringBuilder query_tmp = new StringBuilder();
@@ -107,7 +116,7 @@
 						query_tmp.append(parts[i]);
 						query_tmp.append(' ');
 					}
-					query = query_tmp.toString();
+					query = query_tmp.toString();*/
 					if (! QueryHasPeriods)
 						query = query.replaceAll("\\.", " ");
 				}
@@ -131,4 +140,13 @@
 		logger.info("Extracted "+ vecStringQueries.size() + " queries");
 		return gotSome;
 	}
+	
+	static int minOver0(final int[] a)
+	{
+		int min = Integer.MAX_VALUE;
+		for(int i : a)
+			if (i != -1 && i < min)
+				min = i;
+		return min;
+	}
 }
Index: src/uk/ac/gla/terrier/structures/merging/StructureMerger.java
===================================================================
--- src/uk/ac/gla/terrier/structures/merging/StructureMerger.java	(revision 2563)
+++ src/uk/ac/gla/terrier/structures/merging/StructureMerger.java	(working copy)
@@ -205,10 +205,9 @@
 			try{
 				invOS = 
 					(DirectInvertedOutputStream)invertedFileOutputStreamClass
-					.getConstructor(String.class,Integer.TYPE)
+					.getConstructor(String.class)
 					.newInstance(destIndex.getPath() + ApplicationSetup.FILE_SEPARATOR +  
-								destIndex.getPrefix() + ".inverted"+ BitIn.USUAL_EXTENSION,
-								binaryBits);
+								destIndex.getPrefix() + ".inverted"+ BitIn.USUAL_EXTENSION);
 			} catch (Exception e) {
 				logger.error("Couldn't create specified DirectInvertedOutputStream", e);
 				return;
@@ -367,10 +366,9 @@
 			try{
 				dfOutput = 
 					(DirectInvertedOutputStream)invertedFileOutputStreamClass
-					.getConstructor(String.class,Integer.TYPE)
+					.getConstructor(String.class)
 					.newInstance(destIndex.getPath() + ApplicationSetup.FILE_SEPARATOR +  
-								destIndex.getPrefix() + ".direct" + ".bf",
-								binaryBits);
+								destIndex.getPrefix() + ".direct" + ".bf");
 			} catch (Exception e) {
 				logger.error("Couldn't create specified DirectInvertedOutputStream", e);
 				return;
Index: src/uk/ac/gla/terrier/structures/BlockInvertedIndex.java
===================================================================
--- src/uk/ac/gla/terrier/structures/BlockInvertedIndex.java	(revision 2563)
+++ src/uk/ac/gla/terrier/structures/BlockInvertedIndex.java	(working copy)
@@ -31,6 +31,7 @@
 
 import uk.ac.gla.terrier.compression.BitIn;
 import uk.ac.gla.terrier.structures.postings.BlockIterablePosting;
+import uk.ac.gla.terrier.structures.postings.IterablePosting;
 import uk.ac.gla.terrier.utility.FieldScore;
 /**
  * This class implements the block field inverted 
@@ -41,6 +42,11 @@
 public class BlockInvertedIndex extends InvertedIndex implements IndexConfigurable {
 	protected int DocumentBlockCountDelta = 1;
 		
+	public BlockInvertedIndex(Index index, String structureName, DocumentIndex _doi, Class<? extends IterablePosting> postingClass) throws IOException
+	{
+		super(index, structureName, _doi, postingClass);
+	}
+	
 	public BlockInvertedIndex(Index index, String structureName, DocumentIndex doi) throws IOException {
 		super(index, structureName, doi, BlockIterablePosting.class);
 	}
Index: src/uk/ac/gla/terrier/structures/FieldEntryStatistics.java
===================================================================
--- src/uk/ac/gla/terrier/structures/FieldEntryStatistics.java	(revision 0)
+++ src/uk/ac/gla/terrier/structures/FieldEntryStatistics.java	(revision 0)
@@ -0,0 +1,5 @@
+package uk.ac.gla.terrier.structures;
+
+public interface FieldEntryStatistics extends EntryStatistics {
+	public int[] getFieldFrequencies();
+}
Index: src/uk/ac/gla/terrier/structures/indexing/DirectIndexBuilder.java
===================================================================
--- src/uk/ac/gla/terrier/structures/indexing/DirectIndexBuilder.java	(revision 2563)
+++ src/uk/ac/gla/terrier/structures/indexing/DirectIndexBuilder.java	(working copy)
@@ -37,6 +37,7 @@
 import uk.ac.gla.terrier.structures.Index;
 import uk.ac.gla.terrier.structures.SimpleBitIndexPointer;
 import uk.ac.gla.terrier.utility.ApplicationSetup;
+import uk.ac.gla.terrier.utility.ArrayUtils;
 import uk.ac.gla.terrier.utility.FieldScore;
 /**
  * Builds a direct index, using field information optionally.
@@ -164,7 +165,9 @@
 				"uk.ac.gla.terrier.structures.DirectIndexInputStream", 
 				"uk.ac.gla.terrier.structures.Index,java.lang.String",
 				"index,structureName");
-			index.setIndexProperty("num.direct.fields.bits", ""+fieldTags);
+			index.setIndexProperty("index.direct.fields.count", ""+FieldScore.FIELDS_COUNT );
+			index.setIndexProperty("index.direct.fields.names", ArrayUtils.join(FieldScore.FIELD_NAMES, ","));
+			//index.setIndexProperty("num.direct.fields.bits", ""+fieldTags);
 		}
 	}
 
Index: src/uk/ac/gla/terrier/structures/indexing/LexiconBuilder.java
===================================================================
--- src/uk/ac/gla/terrier/structures/indexing/LexiconBuilder.java	(revision 2563)
+++ src/uk/ac/gla/terrier/structures/indexing/LexiconBuilder.java	(working copy)
@@ -56,7 +56,7 @@
 	@SuppressWarnings("unchecked") //TODO : this is complicated to fix
 	protected Class<? extends LexiconOutputStream> lexiconOutputStream = null;
 
-	protected Class<? extends LexiconMap> LexiconMapClass = null;
+	//protected Class<? extends LexiconMap> LexiconMapClass = null;
 	
 	protected final String lexiconEntryFactoryValueClass;
 	
@@ -82,9 +82,6 @@
 	/** The lexicontree to write the current term stream to */
 	protected LexiconMap TempLex;
 	
-	/** The directory to write temporary lexicons to */
-	//protected String TemporaryLexiconDirectory = null;
-	
 	/** The directory to write the final lexicons to */
 	protected String indexPath = null;
 	/** The filename of the lexicons. */
@@ -95,12 +92,6 @@
 	/** How many temporary lexicons have been generated so far */
 	protected int TempLexCount = 0;
 	
-	/** How many temporary directories have been generated so far */
-	//protected int TempLexDirCount = 0;
-	
-	/** How many temporary lexicons per temporary directory. Set from the property <tt>lexicon.builder.templexperdir</tt>, default 100 */
-	//protected static final int TempLexPerDir = Integer.parseInt(ApplicationSetup.getProperty("lexicon.builder.templexperdir", "100"));
-
 	/** Should we only merge lexicons in pairs (Terrier 1.0.x scheme)? Set by property <tt>lexicon.builder.merge.2lex.attime</tt> */
 	protected static final boolean MERGE2LEXATTIME = Boolean.parseBoolean(ApplicationSetup.getProperty("lexicon.builder.merge.2lex.attime", "false"));
 
@@ -125,11 +116,11 @@
 		}
 		
 		public void count(LexiconEntry value)
-	{
+		{
 			numberOfTokens += value.getFrequency();
 			numberOfPointers += value.getDocumentFrequency();
 			numberOfTerms++;
-	}
+		}
 	
 		public void close()
 		{
@@ -142,28 +133,43 @@
 		}
 	}
 	
+	protected static LexiconMap instantiate(Class<? extends LexiconMap> LexiconMapClass)
+	{
+		LexiconMap TempLex = null;
+		try{ TempLex = (LexiconMap) LexiconMapClass.newInstance(); } catch (Exception e) {logger.error(e);}
+		return TempLex;
+	}
+	
 	protected String defaultStructureName;
 	protected FixedSizeWriteableFactory<LexiconEntry> valueFactory;
 	
 	
 	public LexiconBuilder(Index i, String _structureName) {
 		this(i, _structureName, 
-				LexiconMap.class, "uk.ac.gla.terrier.structures.BasicLexiconEntry");
+				instantiate(LexiconMap.class), "uk.ac.gla.terrier.structures.BasicLexiconEntry");
 	}
 	
-	@SuppressWarnings("unchecked")
-	protected LexiconBuilder(Index i, String _structureName, 
+	public LexiconBuilder(Index i, String _structureName, 
 			Class <? extends LexiconMap> _LexiconMapClass,
 			String _lexiconEntryClass)
 	{
+		this(i, _structureName, instantiate(_LexiconMapClass), _lexiconEntryClass);
+	}
+	
+	@SuppressWarnings("unchecked")
+	public LexiconBuilder(Index i, String _structureName, 
+				LexiconMap lexiconMap,
+				String _lexiconEntryClass)
+	{
 		this.index = i;
 		this.indexPath = index.getPath();
 		this.indexPrefix = index.getPrefix();
 		this.defaultStructureName = _structureName;
+		this.TempLex = lexiconMap;
 		//TemporaryLexiconDirectory = indexPath + ApplicationSetup.FILE_SEPARATOR + indexPrefix + "_";
-		LexiconMapClass = _LexiconMapClass;	
+		//LexiconMapClass = lexiconMap;	
 		lexiconEntryFactoryValueClass = _lexiconEntryClass;
-		try{ TempLex = (LexiconMap) LexiconMapClass.newInstance(); } catch (Exception e) {logger.error(e);}
+		
 	
 		this.index.addIndexStructure(
 				defaultStructureName+"-keyfactory", 
@@ -238,7 +244,8 @@
 				logger.debug("flushing lexicon");
 			writeTemporaryLexicon();
 			TempLexCount++;
-			try{ TempLex = (LexiconMap)LexiconMapClass.newInstance(); } catch (Exception e) {logger.error(e);}
+			TempLex.clear();
+			//try{ TempLex = (LexiconMap)LexiconMapClass.newInstance(); } catch (Exception e) {logger.error(e);}
 		}
 	}
 
@@ -249,7 +256,8 @@
 			logger.debug("flushing lexicon");
 		writeTemporaryLexicon();
 		TempLexCount++;
-		try{ TempLex = (LexiconMap)LexiconMapClass.newInstance(); } catch (Exception e) {logger.error(e);}
+		TempLex.clear();
+		//try{ TempLex = (LexiconMap)LexiconMapClass.newInstance(); } catch (Exception e) {logger.error(e);}
 	}
 	
 	/**
Index: src/uk/ac/gla/terrier/structures/indexing/BlockDocumentPostingList.java
===================================================================
--- src/uk/ac/gla/terrier/structures/indexing/BlockDocumentPostingList.java	(revision 2563)
+++ src/uk/ac/gla/terrier/structures/indexing/BlockDocumentPostingList.java	(working copy)
@@ -26,9 +26,17 @@
  *   
  */
 package uk.ac.gla.terrier.structures.indexing;
-import gnu.trove.*;
+import gnu.trove.THashMap;
+import gnu.trove.TIntHashSet;
+import gnu.trove.TIntObjectHashMap;
+import gnu.trove.TObjectIntProcedure;
+
+import java.io.IOException;
 import java.util.Arrays;
+
 import uk.ac.gla.terrier.sorting.HeapSortInt;
+import uk.ac.gla.terrier.structures.postings.BlockPosting;
+import uk.ac.gla.terrier.structures.postings.IterablePosting;
 import uk.ac.gla.terrier.utility.TermCodes;
 /** Represents the postings of one document, and saves block (term position) information. Uses HashMaps internally.
   * <p>
@@ -45,11 +53,7 @@
 	protected int blockCount = 0;
 	/** Instantiate a new block document posting list. Saves block information, but no fields */
 	public BlockDocumentPostingList() {super();} 
-	/** Instantiate a new block document posting list. Saves block information, with the specified number of fields */
-	public BlockDocumentPostingList(int fieldCount)  {
-		super(fieldCount);
-	}
-
+	
 	/** Insert a term into this document, occurs at given block id */
 	public void insert(String t, int blockId)
 	{
@@ -63,44 +67,7 @@
 		blockCount++;	
 	}
 
-	/** Insert a term into this document, occurs at given block id, and in the given field */
-	public void insert(String t, int fieldId, int blockId)
-	{
-		super.insert(t, fieldId);
-		TIntHashSet blockids = null;
-		if ((blockids = term_blocks.get(t)) == null)
-		{
-			term_blocks.put(t, blockids = new TIntHashSet(/*TODO */));
-		}
-		blockids.add(blockId);
-		blockCount++;
-	}
-
-	/** Insert a term into this document, occurs at given block id, and in the given fields */
-	public void insert(String t,int[] fieldIds, int blockId)
-	{
-		super.insert(t, fieldIds);
-		TIntHashSet blockids = null;
-		if ((blockids = term_blocks.get(t)) == null)
-		{
-			term_blocks.put(t, blockids = new TIntHashSet(/*TODO */));
-		}
-		blockids.add(blockId);
-		blockCount++;
-	}
-
-	/** Insert a term into this document tf times, occurs at given block id, and in the given fields */
-	public void insert(int tf, String t,int[] fieldIds, int blockId)
-	{
-		super.insert(tf, t, fieldIds);
-		TIntHashSet blockids = null;
-		if ((blockids = term_blocks.get(t)) == null)
-		{
-			term_blocks.put(t, blockids = new TIntHashSet(/*TODO */));
-		}
-		blockids.add(blockId);
-		blockCount++;
-	}
+	
 	
 	public int[] getBlocks(String term)
 	{
@@ -112,66 +79,38 @@
 	}
 
 	/** returns the postings suitable to be written into the block direct index */
+	@Override
 	public int[][] getPostings()
 	{
 		final int termCount = occurrences.size();
 		final int[] termids = new int[termCount];
 		final int[] tfs = new int[termCount];
-		final int[] fields = new int[termCount];
+		final int[] fields = null;
 		final int[] blockfreqs = new int[termCount];
 		final TIntObjectHashMap<int[]> term2blockids = new TIntObjectHashMap<int[]>();
 		int blockTotal = 0; //TODO we already have blockTotal as this.blockCount, so no need to count?
-		if (fieldCount > 0)
-		{
-			
-			class PostingVisitor implements TObjectIntProcedure<String> {
-				int blockTotal = 0;
-				int i=0;
-				public boolean execute(final String a, final int b)
-				{
-					termids[i] = TermCodes.getCode(a);
-					tfs[i] = b;
-					fields[i] = term_fields.get(a);
-					final TIntHashSet ids = term_blocks.get(a);
-					blockfreqs[i] = ids.size();
-					this.blockTotal += ids.size();
-					final int[] bids = ids.toArray();
-					Arrays.sort(bids);
-					term2blockids.put(termids[i], bids);
-					i++;
-					return true;
-				}
-			};
-			PostingVisitor proc = new PostingVisitor();
-			occurrences.forEachEntry(proc);
-			blockTotal = proc.blockTotal;
-			HeapSortInt.ascendingHeapSort(termids, tfs, fields, blockfreqs);
-		}
-		else
-		{
-			class PostingVisitor implements TObjectIntProcedure<String> {
-				int i=0;
-				int blockTotal = 0;
-				public boolean execute(final String a, final int b)
-				{
-					termids[i] = TermCodes.getCode(a);
-					tfs[i] = b;
-					final TIntHashSet ids = term_blocks.get(a);
-					blockfreqs[i] = ids.size();
-					blockTotal += ids.size();
-					final int[] bids = ids.toArray();
-					Arrays.sort(bids);
-					term2blockids.put(termids[i], bids);
-					//System.err.println(a+": tid="+termids[i]+" tf="+tfs[i]+" bf="+blockfreqs[i] +" blocks="+Arrays.toString(bids));
-					i++;
-					return true;
-				}
-			};
-			PostingVisitor proc = new PostingVisitor();
-			occurrences.forEachEntry(proc);
-			blockTotal = proc.blockTotal;
-			HeapSortInt.ascendingHeapSort(termids, tfs,  blockfreqs);
-		}
+		class PostingVisitor implements TObjectIntProcedure<String> {
+			int i=0;
+			int blockTotal = 0;
+			public boolean execute(final String a, final int b)
+			{
+				termids[i] = TermCodes.getCode(a);
+				tfs[i] = b;
+				final TIntHashSet ids = term_blocks.get(a);
+				blockfreqs[i] = ids.size();
+				blockTotal += ids.size();
+				final int[] bids = ids.toArray();
+				Arrays.sort(bids);
+				term2blockids.put(termids[i], bids);
+				//System.err.println(a+": tid="+termids[i]+" tf="+tfs[i]+" bf="+blockfreqs[i] +" blocks="+Arrays.toString(bids));
+				i++;
+				return true;
+			}
+		};
+		PostingVisitor proc = new PostingVisitor();
+		occurrences.forEachEntry(proc);
+		blockTotal = proc.blockTotal;
+		HeapSortInt.ascendingHeapSort(termids, tfs,  blockfreqs);
 		final int[] blockids = new int[blockTotal];
 		int offset = 0;
 		for (int termid : termids)
@@ -183,4 +122,39 @@
 		}
 		return new int[][]{termids, tfs, fields, blockfreqs, blockids};
 	}
+	
+	class BlockPostingIterator extends postingIterator implements BlockPosting
+	{
+		int blockOffset = 0;
+		int blockLength = 0;
+		public BlockPostingIterator(int[][] postings)
+		{
+			super(postings);
+		}
+		
+		public int[] getPositions() {
+			int[] rtr = new int[blockLength];
+			if (blockLength == 0)
+				return rtr;
+			System.arraycopy(allPostings[4], blockOffset, rtr, 0, blockLength);
+			return rtr;
+		}
+
+		@Override
+		public boolean next() throws IOException {
+			if (! super.next())
+				return false;
+			blockOffset += blockLength;
+			blockLength = allPostings[3][i];
+			return true;
+		}
+		
+				
+	}
+	
+	@Override
+	public IterablePosting getPostings2()
+	{
+		return new BlockPostingIterator(this.getPostings());		
+	}
 }
Index: src/uk/ac/gla/terrier/structures/indexing/BlockLexiconBuilder.java
===================================================================
--- src/uk/ac/gla/terrier/structures/indexing/BlockLexiconBuilder.java	(revision 2563)
+++ src/uk/ac/gla/terrier/structures/indexing/BlockLexiconBuilder.java	(working copy)
@@ -31,6 +31,7 @@
  * Builds a block lexicon using block frequencies.
  * @author Craig Macdonald
  * @version $Revision: 1.32 $
+ * @deprecated
  */
 public class BlockLexiconBuilder extends LexiconBuilder
 {
Index: src/uk/ac/gla/terrier/structures/indexing/BlockInvertedIndexBuilder.java
===================================================================
--- src/uk/ac/gla/terrier/structures/indexing/BlockInvertedIndexBuilder.java	(revision 2563)
+++ src/uk/ac/gla/terrier/structures/indexing/BlockInvertedIndexBuilder.java	(working copy)
@@ -46,6 +46,7 @@
 import uk.ac.gla.terrier.structures.LexiconEntry;
 import uk.ac.gla.terrier.structures.LexiconOutputStream;
 import uk.ac.gla.terrier.utility.ApplicationSetup;
+import uk.ac.gla.terrier.utility.ArrayUtils;
 import uk.ac.gla.terrier.utility.FieldScore;
 import uk.ac.gla.terrier.utility.Files;
 
@@ -309,7 +310,10 @@
 					"uk.ac.gla.terrier.structures.BlockInvertedIndexInputStream",
                     "uk.ac.gla.terrier.structures.Index,java.lang.String,java.util.Iterator",
                     "index,structureName,lexicon-inputstream");
-			index.setIndexProperty("num.inverted.fields.bits", ""+FieldScore.FIELDS_COUNT );
+			 index.setIndexProperty("index.inverted.fields.count", ""+FieldScore.FIELDS_COUNT );
+			 index.setIndexProperty("index.inverted.fields.names", ArrayUtils.join(FieldScore.FIELD_NAMES, ","));
+				
+			//index.setIndexProperty("num.inverted.fields.bits", ""+FieldScore.FIELDS_COUNT );
 			//these should be already set, but in case their not
 			index.setIndexProperty("num.Terms", ""+numberOfUniqueTerms);
 			index.setIndexProperty("num.Tokens", ""+numberOfTokens);
Index: src/uk/ac/gla/terrier/structures/indexing/FieldLexiconMap.java
===================================================================
--- src/uk/ac/gla/terrier/structures/indexing/FieldLexiconMap.java	(revision 0)
+++ src/uk/ac/gla/terrier/structures/indexing/FieldLexiconMap.java	(revision 0)
@@ -0,0 +1,81 @@
+package uk.ac.gla.terrier.structures.indexing;
+
+import gnu.trove.TObjectIntHashMap;
+import gnu.trove.TObjectIntProcedure;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+import uk.ac.gla.terrier.structures.FieldLexiconEntry;
+import uk.ac.gla.terrier.structures.LexiconOutputStream;
+import uk.ac.gla.terrier.utility.TermCodes;
+
+public class FieldLexiconMap extends LexiconMap {
+
+	protected final int fieldCount;
+	protected final TObjectIntHashMap<String>field_tfs[]; 
+	
+	@SuppressWarnings("unchecked")
+	public FieldLexiconMap(int _fieldCount)
+	{
+		super();
+		fieldCount = _fieldCount;
+		field_tfs = new TObjectIntHashMap[fieldCount];
+		for(int fi=0;fi<fieldCount;fi++)
+			field_tfs[fi] = new TObjectIntHashMap<String>(BUNDLE_AVG_UNIQUE_TERMS);
+	}
+	
+	protected int[] getFieldFrequency(String term)
+	{
+		int[] fieldFrequencies = new int[fieldCount];
+		for(int fi=0;fi<fieldCount;fi++)
+			fieldFrequencies[fi] = field_tfs[fi].get(term);
+		return fieldFrequencies;
+	}
+	
+	/** Inserts all the terms from a document posting
+	  * into the lexicon map
+	  * @param doc The postinglist for that document
+	  */
+	public void insert(DocumentPostingList _doc)
+	{
+		super.insert(_doc);
+		FieldDocumentPostingList doc = (FieldDocumentPostingList)_doc;
+		int fi = 0;
+		for(TObjectIntHashMap<String> docField : doc.field_occurrences)
+		{
+			final TObjectIntHashMap<String> thisField = field_tfs[fi];
+			docField.forEachEntry(new TObjectIntProcedure<String>() {
+				public boolean execute(String arg0, int arg1) {
+					thisField.adjustOrPutValue(arg0, arg1, arg1);
+					return true;
+				}
+			});
+			fi++;
+		}
+	}
+	
+	/** Stores the lexicon tree to a lexicon stream as a sequence of entries.
+	  * The binary tree is traversed in order, by called the method
+	  * traverseAndStoreToStream.
+	  * @param lexiconStream The lexicon output stream to store to. */
+	public void storeToStream(LexiconOutputStream<String> lexiconStream) throws IOException
+	{
+		final String[] terms = tfs.keys(new String[0]);
+		Arrays.sort(terms);
+		for (String t : terms)
+		{
+			FieldLexiconEntry fle = new FieldLexiconEntry(getFieldFrequency(t));
+			fle.setTermId(TermCodes.getCode(t));
+			fle.setStatistics(nts.get(t), tfs.get(t));	
+			lexiconStream.writeNextEntry(t, fle);
+		}
+	}
+
+	@Override
+	public void clear() {
+		super.clear();
+		for(int fi=0;fi<fieldCount;fi++)
+			field_tfs[fi].clear();
+	}
+}
Index: src/uk/ac/gla/terrier/structures/indexing/BlockFieldDocumentPostingList.java
===================================================================
--- src/uk/ac/gla/terrier/structures/indexing/BlockFieldDocumentPostingList.java	(revision 0)
+++ src/uk/ac/gla/terrier/structures/indexing/BlockFieldDocumentPostingList.java	(revision 0)
@@ -0,0 +1,171 @@
+package uk.ac.gla.terrier.structures.indexing;
+
+import gnu.trove.THashMap;
+import gnu.trove.TIntHashSet;
+import gnu.trove.TIntObjectHashMap;
+import gnu.trove.TObjectIntProcedure;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+import uk.ac.gla.terrier.sorting.HeapSortInt;
+import uk.ac.gla.terrier.structures.postings.BlockFieldPostingImpl;
+import uk.ac.gla.terrier.structures.postings.BlockPosting;
+import uk.ac.gla.terrier.structures.postings.IterablePosting;
+import uk.ac.gla.terrier.structures.postings.WritablePosting;
+import uk.ac.gla.terrier.utility.TermCodes;
+
+public class BlockFieldDocumentPostingList extends FieldDocumentPostingList {
+	/** mapping term to blockids in this document */
+	protected final THashMap<String, TIntHashSet> term_blocks = new THashMap<String, TIntHashSet>(AVG_DOCUMENT_UNIQUE_TERMS);
+	/** number of blocks in this document. usually equal to document length, but perhaps less */
+	protected int blockCount = 0;
+		
+	public BlockFieldDocumentPostingList(int NUM_FIELDS) {
+		super(NUM_FIELDS);
+	}
+	
+	public int[] getBlocks(String term)
+	{
+		int[] rtr = term_blocks.get(term).toArray();
+		if (rtr ==  null)
+			return new int[0];
+		Arrays.sort(rtr);
+		return rtr;
+	}
+
+	/** Insert a term into this document, occurs at given block id, and in the given field */
+	public void insert(String t, int fieldId, int blockId)
+	{
+		super.insert(t, fieldId);
+		TIntHashSet blockids = null;
+		if ((blockids = term_blocks.get(t)) == null)
+		{
+			term_blocks.put(t, blockids = new TIntHashSet(/*TODO */));
+		}
+		blockids.add(blockId);
+		blockCount++;
+	}
+
+	/** Insert a term into this document, occurs at given block id, and in the given fields */
+	public void insert(String t,int[] fieldIds, int blockId)
+	{
+		super.insert(t, fieldIds);
+		TIntHashSet blockids = null;
+		if ((blockids = term_blocks.get(t)) == null)
+		{
+			term_blocks.put(t, blockids = new TIntHashSet(/*TODO */));
+		}
+		blockids.add(blockId);
+		blockCount++;
+	}
+
+	/** Insert a term into this document tf times, occurs at given block id, and in the given fields */
+	public void insert(int tf, String t,int[] fieldIds, int blockId)
+	{
+		super.insert(tf, t, fieldIds);
+		TIntHashSet blockids = null;
+		if ((blockids = term_blocks.get(t)) == null)
+		{
+			term_blocks.put(t, blockids = new TIntHashSet(/*TODO */));
+		}
+		blockids.add(blockId);
+		blockCount++;
+	}
+	
+	class blockFieldPostings extends fieldPostingIterator implements BlockPosting
+	{
+		int blockOffset = 0;
+		int blockLength = 0;
+		public blockFieldPostings(int[][] postings) {
+			super(postings);
+		}
+		
+		public int[] getPositions() {
+			int[] rtr = new int[blockLength];
+			if (blockLength == 0)
+				return rtr;
+			System.arraycopy(allPostings[4], blockOffset, rtr, 0, blockLength);
+			return rtr;
+		}
+
+		@Override
+		public boolean next() throws IOException {
+			if (! super.next())
+				return false;
+			blockOffset += blockLength;
+			blockLength = allPostings[3][i];
+			return true;
+		}
+
+		@Override
+		public WritablePosting asWritablePosting() {
+			BlockFieldPostingImpl fbp = new BlockFieldPostingImpl(termids[i], tfs[i], getPositions(), fieldCount);
+			System.arraycopy(getFieldFrequencies(), 0, fbp.getFieldFrequencies(), 0, fieldCount);
+			return fbp;
+		}		
+	}
+	
+	/** returns the postings suitable to be written into the block direct index */
+	public int[][] getPostings()
+	{
+		final int termCount = occurrences.size();
+		final int[] termids = new int[termCount];
+		final int[] tfs = new int[termCount];
+		
+		final int[][] fields = new int[fieldCount][termCount];
+		final int[] blockfreqs = new int[termCount];
+		final TIntObjectHashMap<int[]> term2blockids = new TIntObjectHashMap<int[]>();
+		int blockTotal = 0; //TODO we already have blockTotal as this.blockCount, so no need to count?
+		
+		class PostingVisitor implements TObjectIntProcedure<String> {
+			int blockTotal = 0;
+			int i=0;
+			public boolean execute(final String a, final int b)
+			{
+				termids[i] = TermCodes.getCode(a);
+				tfs[i] = b;
+				for(int fi=0;fi<fieldCount;fi++)
+					fields[fi][i] = field_occurrences[fi].get(a);
+				final TIntHashSet ids = term_blocks.get(a);
+				blockfreqs[i] = ids.size();
+				this.blockTotal += ids.size();
+				final int[] bids = ids.toArray();
+				Arrays.sort(bids);
+				term2blockids.put(termids[i], bids);
+				i++;
+				return true;
+			}
+		};
+		PostingVisitor proc = new PostingVisitor();
+		occurrences.forEachEntry(proc);
+		blockTotal = proc.blockTotal;
+		int[][] tmppostings = new int[3+fieldCount][];
+		tmppostings[0] = termids;
+		tmppostings[1] = tfs;
+		for(int fi=0;fi<fieldCount;fi++)
+			tmppostings[fi+2] = fields[fi];
+		tmppostings[fieldCount+2] = blockfreqs;
+		HeapSortInt.ascendingHeapSort(tmppostings);
+		final int[] blockids = new int[blockTotal];
+		int offset = 0;
+		for (int termid : termids)
+		{
+			final int[] src = term2blockids.get(termid);
+			final int src_l = src.length;
+			System.arraycopy(src, 0, blockids, offset, src_l);
+			offset+= src_l;
+		}
+		int[][] postings = new int[4+fieldCount][];
+		for(int fi=0;fi<fieldCount;fi++)
+			postings[fi+2] = fields[fi];
+		postings[fieldCount+2] = blockfreqs;
+		postings[fieldCount+3] = blockids;
+		return postings;
+	}
+
+	@Override
+	public IterablePosting getPostings2() {
+		return new blockFieldPostings(this.getPostings());
+	}
+}
\ No newline at end of file
Index: src/uk/ac/gla/terrier/structures/indexing/InvertedIndexBuilder.java
===================================================================
--- src/uk/ac/gla/terrier/structures/indexing/InvertedIndexBuilder.java	(revision 2563)
+++ src/uk/ac/gla/terrier/structures/indexing/InvertedIndexBuilder.java	(working copy)
@@ -51,6 +51,7 @@
 import uk.ac.gla.terrier.structures.LexiconOutputStream;
 import uk.ac.gla.terrier.structures.seralization.FixedSizeWriteableFactory;
 import uk.ac.gla.terrier.utility.ApplicationSetup;
+import uk.ac.gla.terrier.utility.ArrayUtils;
 import uk.ac.gla.terrier.utility.FieldScore;
 import uk.ac.gla.terrier.utility.Files;
 import uk.ac.gla.terrier.utility.Rounding;
@@ -109,6 +110,8 @@
 			Pointers = b;
 		}
 	}
+	
+	protected int fieldCount = 0;
 
 	/** The number of unique terms in the vocabulary.*/
 	public int numberOfUniqueTerms;
@@ -180,6 +183,9 @@
 			long numberOfTokens = 0;
 			long numberOfPointers = 0;
 			int numberOfUniqueTerms = index.getLexicon().numberOfEntries();
+			
+			fieldCount = index.getIntIndexProperty("index.direct.fields.count", 0);
+			
 			Iterator<Map.Entry<String,LexiconEntry>> lexiconStream = 
 				(Iterator<Map.Entry<String,LexiconEntry>>)index.getIndexStructureInputStream("lexicon");
 		
@@ -378,7 +384,8 @@
                     "uk.ac.gla.terrier.structures.InvertedIndexInputStream",
                     "uk.ac.gla.terrier.structures.Index,java.lang.String,java.util.Iterator",
                     "index,structureName,lexicon-entry-inputstream");
-			index.setIndexProperty("num.inverted.fields.bits", ""+FieldScore.FIELDS_COUNT );
+			index.setIndexProperty("index.inverted.fields.count", ""+FieldScore.FIELDS_COUNT );
+			index.setIndexProperty("index.inverted.fields.names", ArrayUtils.join(FieldScore.FIELD_NAMES, ","));
 			//should be already set, but in case their not
 			index.setIndexProperty("num.Terms", ""+numberOfUniqueTerms);
 			index.setIndexProperty("num.Tokens", ""+numberOfTokens);
@@ -393,11 +400,10 @@
 	
 	protected TIntArrayList[] createPointerForTerm(LexiconEntry le)
 	{
-		TIntArrayList[] tmpArray = new TIntArrayList[3];
+		TIntArrayList[] tmpArray = new TIntArrayList[2 + fieldCount];
 		final int tmpNT = le.getDocumentFrequency();
-		tmpArray[0] = new TIntArrayList(tmpNT);
-		tmpArray[1] = new TIntArrayList(tmpNT);
-		tmpArray[2] = new TIntArrayList(tmpNT);
+		for(int i = 0; i < fieldCount+2; i++)
+			tmpArray[i] = new TIntArrayList(tmpNT);
 		return tmpArray;
 	}
 	
@@ -519,9 +525,9 @@
 			//the two next vectors are used for reducing the number of references
 			final int[] documentTerms0 = documentTerms[0];
 			final int[] termfreqs = documentTerms[1];
-			int[] htmlscores = null;
-			if (useFieldInformation)
-				htmlscores = documentTerms[2];
+			//int[] htmlscores = null;
+			//if (useFieldInformation)
+			//	htmlscores = documentTerms[2];
 
 			//scan the list of the j-th document's terms
 			final int length = documentTerms0.length;
@@ -539,7 +545,11 @@
 					tmpMatrix[0].add(p);
 					tmpMatrix[1].add(termfreqs[k]);
 					if (useFieldInformation)
-						tmpMatrix[2].add(htmlscores[k]);
+					{
+						for(int fi = 0; fi < fieldCount; fi++)
+							tmpMatrix[2+fi].add(documentTerms[fi+2][k]);
+						//tmpMatrix[2].add(htmlscores[k]);
+					}
 				}
 			}
 			p++;
Index: src/uk/ac/gla/terrier/structures/indexing/LexiconMap.java
===================================================================
--- src/uk/ac/gla/terrier/structures/indexing/LexiconMap.java	(revision 2563)
+++ src/uk/ac/gla/terrier/structures/indexing/LexiconMap.java	(working copy)
@@ -59,8 +59,8 @@
 	/** Clear the lexicon map */
 	public void clear()	
 	{
-		tfs.clear();
-		nts.clear();
+		tfs.clear(); tfs.compact();
+		nts.clear(); nts.compact();
 	}
 
 	/**
Index: src/uk/ac/gla/terrier/structures/indexing/BlockDirectIndexBuilder.java
===================================================================
--- src/uk/ac/gla/terrier/structures/indexing/BlockDirectIndexBuilder.java	(revision 2563)
+++ src/uk/ac/gla/terrier/structures/indexing/BlockDirectIndexBuilder.java	(working copy)
@@ -29,6 +29,8 @@
 import java.io.IOException;
 
 import uk.ac.gla.terrier.structures.Index;
+import uk.ac.gla.terrier.utility.ArrayUtils;
+import uk.ac.gla.terrier.utility.FieldScore;
 /**
  * Builds a direct index using block and possibly field information.
  * @author Douglas Johnson &amp; Vassilis Plachouras &amp; Craig Macdonald
@@ -62,12 +64,15 @@
 					"uk.ac.gla.terrier.structures.BlockDirectIndex", 
 					"uk.ac.gla.terrier.structures.Index,java.lang.String", 
 					"index,structureName");
-				index.addIndexStructureInputStream(
+			index.addIndexStructureInputStream(
 					"direct", 
 					"uk.ac.gla.terrier.structures.BlockDirectIndexInputStream", 
 					"uk.ac.gla.terrier.structures.Index,java.lang.String",
 					"index,structureName");
-			index.setIndexProperty("num.direct.fields.bits", ""+fieldTags);
+			index.setIndexProperty("index.direct.fields.count", ""+FieldScore.FIELDS_COUNT );
+			index.setIndexProperty("index.direct.fields.names", ArrayUtils.join(FieldScore.FIELD_NAMES, ","));
+				
+			//index.setIndexProperty("num.direct.fields.bits", ""+fieldTags);
 		}
 	}
 	
Index: src/uk/ac/gla/terrier/structures/indexing/DocumentPostingList.java
===================================================================
--- src/uk/ac/gla/terrier/structures/indexing/DocumentPostingList.java	(revision 2563)
+++ src/uk/ac/gla/terrier/structures/indexing/DocumentPostingList.java	(working copy)
@@ -27,11 +27,16 @@
 
 package uk.ac.gla.terrier.structures.indexing;
 
+import java.io.IOException;
+
 import gnu.trove.TObjectIntHashMap;
 import gnu.trove.TObjectIntProcedure;
 import uk.ac.gla.terrier.sorting.HeapSortInt;
 import uk.ac.gla.terrier.structures.BasicDocumentIndexEntry;
 import uk.ac.gla.terrier.structures.DocumentIndexEntry;
+import uk.ac.gla.terrier.structures.postings.BasicPosting;
+import uk.ac.gla.terrier.structures.postings.IterablePosting;
+import uk.ac.gla.terrier.structures.postings.WritablePosting;
 import uk.ac.gla.terrier.utility.ApplicationSetup;
 import uk.ac.gla.terrier.utility.TermCodes;
 /** Represents the postings of one document. Uses HashMaps internally.
@@ -41,7 +46,7 @@
   * size of the haashmaps used in this class.</li></ul>
   */
 public class DocumentPostingList {
-	/** number of unique terms per doc on average, used to tune the initial size of the haashmaps used in this class. */
+	/** number of unique terms per doc on average, used to tune the initial size of the hashmaps used in this class. */
 	protected static final int AVG_DOCUMENT_UNIQUE_TERMS =
 		Integer.parseInt(ApplicationSetup.getProperty("indexing.avg.unique.terms.per.doc", "120"));
 
@@ -50,23 +55,11 @@
 
 	/** mapping term to tf mapping */	
 	protected final TObjectIntHashMap<String> occurrences = new TObjectIntHashMap<String>(AVG_DOCUMENT_UNIQUE_TERMS);
-	/** mapping term to field bitset */
-	protected final TObjectIntHashMap<String> term_fields = new TObjectIntHashMap<String>(AVG_DOCUMENT_UNIQUE_TERMS);
-	/** number of fields in this index */
-	protected final int fieldCount;
-
-	/** Make a new postings list for a document. No fields */	
+	
+	
 	public DocumentPostingList()
-	{
-		this.fieldCount = 0;
-	}
+	{}
 	
-	/** Make a new postings list for a document, with the specified number of fields
-	  * @param fieldCount number of fields marked in this index */
-	public DocumentPostingList(int fieldCount)
-	{
-		this.fieldCount = fieldCount;
-	}
 	
 	public String[] termSet()
 	{
@@ -78,16 +71,12 @@
 		return occurrences.get(term);
 	}
 	
-	public int getFields(String term)
-	{
-		return term_fields.get(term);
-	}
 
 	/** Removes all postings from this document */
 	public void clear()
 	{
 		occurrences.clear();
-		term_fields.clear();
+		documentLength = 0;
 	}
 
 	/** Returns the total number of tokens in this document */	
@@ -118,72 +107,7 @@
         documentLength++;
     }
 
-	/** Insert a term into the posting list of this document, in the given field, with the given frequency
-	  * @param tf frequency of the term in this document
-	  * @param term String form of term
-	  * @param fieldNum fieldNumber it occurrs in */
-	public void insert(final int tf, final String term, final int fieldNum)
-	{
-		occurrences.adjustOrPutValue(term,tf,tf);
-		if (fieldNum > 0)
-		{
-			final int InScore = 1<<(fieldCount - fieldNum);
-			final int ExScore = term_fields.get(term);
-			term_fields.put(term, InScore|ExScore);
-		}
-		documentLength+=tf;
-	}
-	/**  Insert a term into the posting list of this document, in the given field
-	  * @param term the Term being inserted
-	  * @param fieldNum the id of the field that the term was found in */
-	public void insert(final String term, final int fieldNum)
-	{
-		occurrences.adjustOrPutValue(term,1,1);
-		if (fieldNum > 0)
-		{
-			final int InScore = 1<<(fieldCount - fieldNum);
-			final int ExScore = term_fields.get(term);
-			term_fields.put(term, InScore|ExScore);
-		}
-		documentLength++;
-	}
-
-	/**  Insert a term into the posting list of this document, in the given field
-	  * @param term the Term being inserted
-	  * @param fieldNums the ids of the fields that the term was found in */
-	public void insert(final String term, final int[] fieldNums)
-	{
-		occurrences.adjustOrPutValue(term,1,1);
-		final int l = fieldNums.length; 
-		int InScore = term_fields.get(term);
-		for(int i=0;i<l;i++)
-		{
-			if (fieldNums[i] > 0)
-				InScore |= 1<<(fieldCount - fieldNums[i]);
-		}
-		term_fields.put(term, InScore);
-		documentLength++;
-	}
-
-	/**  Insert a term into the posting list of this document, in the given field
-	  * @param tf the frequency of the term
-	  * @param term the Term being inserted
-	  * @param fieldNums the ids of the fields that the term was found in */
-	public void insert(final int tf, final String term, final int[] fieldNums)
-	{
-		occurrences.adjustOrPutValue(term,tf,tf);
-		final int l = fieldNums.length;
-		int InScore = term_fields.get(term);
-		for(int i=0;i<l;i++)
-		{
-			if (fieldNums[i] > 0)
-				InScore |= 1<<(fieldCount - fieldNums[i]);
-		}
-		term_fields.put(term, InScore);
-		documentLength+=tf;
-	}
-	
-	public DocumentIndexEntry getDocumentStatistics()
+    public DocumentIndexEntry getDocumentStatistics()
 	{
 		DocumentIndexEntry die = new BasicDocumentIndexEntry();
 		die.setDocumentLength(documentLength);
@@ -197,35 +121,87 @@
 		final int termCount = occurrences.size();
 		final int[] termids = new int[termCount];
 		final int[] tfs = new int[termCount];
-		final int[] fields = new int[termCount];
-		if (fieldCount > 0)
+		occurrences.forEachEntry( new TObjectIntProcedure<String>() { 
+			int i=0;
+			public boolean execute(final String a, final int b)
+			{
+				termids[i] = TermCodes.getCode(a);
+				tfs[i++] = b;
+				return true;
+			}
+		});
+		HeapSortInt.ascendingHeapSort(termids, tfs);
+		return new int[][]{termids, tfs};
+	}
+	
+	public IterablePosting getPostings2()
+	{
+		//obtain and sort termids by id
+		final int termCount = occurrences.size();
+		final int[] termids = new int[termCount];
+		final int[] tfs = new int[termCount];
+		occurrences.forEachEntry( new TObjectIntProcedure<String>() { 
+			int i=0;
+			public boolean execute(final String a, final int b)
+			{
+				termids[i] = TermCodes.getCode(a);
+				tfs[i++] = b;
+				return true;
+			}
+		});
+		HeapSortInt.ascendingHeapSort(termids, tfs);
+		return new postingIterator(termids, tfs);		
+	}
+	
+	class postingIterator implements IterablePosting
+	{
+		int i=-1;
+		final int termCount;
+		final int[] termids;
+		final int[] tfs;
+		final int[][] allPostings;
+		
+		public postingIterator(int[][] postings)
 		{
-			occurrences.forEachEntry( new TObjectIntProcedure<String>() {
-				int i=0;
-				public boolean execute(final String a, final int b)
-				{
-					termids[i] = TermCodes.getCode(a);
-					tfs[i] = b;
-					fields[i++] = term_fields.get(a);
-					return true;
-				}
-			});	
-			HeapSortInt.ascendingHeapSort(termids, tfs, fields);
-			return new int[][]{termids, tfs, fields};
+			termids = postings[0];
+			tfs = postings[1];
+			allPostings = postings;
+			termCount = termids.length;
 		}
-		else
+		
+		public postingIterator(int[] _termids, int[] _tfs)
 		{
-			occurrences.forEachEntry( new TObjectIntProcedure<String>() { 
-				int i=0;
-				public boolean execute(final String a, final int b)
-				{
-					termids[i] = TermCodes.getCode(a);
-					tfs[i++] = b;
-					return true;
-				}
-			});
-			HeapSortInt.ascendingHeapSort(termids, tfs);
-			return new int[][]{termids, tfs};	
+			termids = _termids;
+			tfs = _tfs;
+			allPostings = null;
+			termCount = termids.length;
+		}
+		
+		public boolean next() throws IOException {
+			if (i == termCount -1)
+				return false;
+			i++;
+			//System.err.println("next OK: i="+i + " termCount="+termCount);
+			return true;
+		}
+
+		public int getDocumentLength() {
+			return documentLength;
+		}
+
+		public int getFrequency() {
+			return tfs[i];
+		}
+
+		public int getId() {
+			return termids[i];
+		}
+
+		public void setId(int id) {}
+		public void close() throws IOException {	}
+		
+		public WritablePosting asWritablePosting() {
+			return new BasicPosting(termids[i], tfs[i]);
 		}
 	}
 	
Index: src/uk/ac/gla/terrier/structures/indexing/singlepass/BlockFieldPosting.java
===================================================================
--- src/uk/ac/gla/terrier/structures/indexing/singlepass/BlockFieldPosting.java	(revision 2563)
+++ src/uk/ac/gla/terrier/structures/indexing/singlepass/BlockFieldPosting.java	(working copy)
@@ -28,8 +28,6 @@
 package uk.ac.gla.terrier.structures.indexing.singlepass;
 
 import java.io.IOException;
-
-import uk.ac.gla.terrier.utility.FieldScore;
 /**
  * Class representing a posting list in memory containing fields and block iformation.
  * It keeps the information for <code>tf, df, field</code> and the sequence <code>[doc, idf, bockNo [blockId]]</code>
@@ -38,15 +36,16 @@
  */
 public class BlockFieldPosting extends BlockPosting{
 	/** The number of different fields that are used for indexing field information.*/	
-	protected static final int fieldTags = FieldScore.FIELDS_COUNT;
+	//protected static final int fieldTags = FieldScore.FIELDS_COUNT;
 	
-	public void  writeFirstDoc(final int doc, final int frequency, int fieldScore, int[] blockids) throws IOException{
+	public void  writeFirstDoc(final int doc, final int frequency, int[] fieldFreqs, int[] blockids) throws IOException{
 		super.writeFirstDoc(doc, frequency);
-		if (fieldTags> 0)
-			docIds.writeBinary(fieldTags, fieldScore);
+		for(int tff : fieldFreqs)
+			docIds.writeGamma(tff+1);
 		final int blockCount = blockids.length;
-		
 		docIds.writeUnary(blockCount+1);
+		
+			
 		if (blockCount > 0)
 		{
 			docIds.writeGamma(blockids[0]+1);
@@ -56,10 +55,10 @@
 		}
 	}
 	
-	public int insert(final int doc, final int freq, final int fieldScore, final int[] blockids) throws IOException{
+	public int insert(final int doc, final int freq, final int[] fieldFreqs, final int[] blockids) throws IOException{
 		int c = insert(doc, freq);
-		if (fieldTags> 0)
-			docIds.writeBinary(fieldTags, fieldScore);
+		for(int tff : fieldFreqs)
+			docIds.writeUnary(tff+1);
 		final int blockCount = blockids.length;
 		
 		docIds.writeUnary(blockCount+1);
Index: src/uk/ac/gla/terrier/structures/indexing/singlepass/FieldPostingInRun.java
===================================================================
--- src/uk/ac/gla/terrier/structures/indexing/singlepass/FieldPostingInRun.java	(revision 2563)
+++ src/uk/ac/gla/terrier/structures/indexing/singlepass/FieldPostingInRun.java	(working copy)
@@ -61,7 +61,8 @@
 			int docid = postingSource.readGamma() + current;
 			bos.writeGamma(docid - last);
 			bos.writeUnary(postingSource.readGamma());
-			bos.writeBinary(fieldTags, postingSource.readBinary(fieldTags));
+			for(int f=0;f<fieldTags;f++)
+				bos.writeUnary(postingSource.readGamma());
 			current = last = docid;		
 		}
 		try{
Index: src/uk/ac/gla/terrier/structures/indexing/singlepass/FieldPosting.java
===================================================================
--- src/uk/ac/gla/terrier/structures/indexing/singlepass/FieldPosting.java	(revision 2563)
+++ src/uk/ac/gla/terrier/structures/indexing/singlepass/FieldPosting.java	(working copy)
@@ -28,18 +28,14 @@
 
 import java.io.IOException;
 
-import uk.ac.gla.terrier.utility.FieldScore;
-
 /** Class holding the information for a posting list read
  * from a previously written run at disk. Used in the merging phase of the Single pass inversion method.
  * This class knows how to append itself to a {@link uk.ac.gla.terrier.compression.BitOutputStream} and it
- * represents a posting with field information <code>(tf, df, [docid, idf, fieldScore])</code>
- * @author Roi Blanco
+ * represents a posting with field information <code>(tf, df, [docid, idf, field_f, field_f, field_f])</code>
+ * @author Roi Blanco and Craig Macdonald
  *
  */
 public class FieldPosting extends Posting{
-	/** The number of different fields that are used for indexing field information.*/	
-	protected static final int fieldTags = FieldScore.FIELDS_COUNT;
 	/**
 	 * Writes the first document in the posting list.
 	 * @param doc the document identifier.
@@ -47,9 +43,10 @@
 	 * @param fieldScore field score for the term in the document.
 	 * @throws IOException if an I/O error ocurrs.
 	 */	
-	public void writeFirstDoc(final int doc, final int frequency, final int fieldScore) throws IOException{
+	public void writeFirstDoc(final int doc, final int frequency, final int[] fieldFrequencies) throws IOException{
 		writeFirstDoc(doc, frequency);
-		docIds.writeBinary(fieldTags, fieldScore);
+		for(int field_f : fieldFrequencies)
+			docIds.writeGamma(field_f+1);
 	}
 	
 	/**
@@ -61,9 +58,10 @@
 	 * @return the updated term frequency.
 	 * @throws IOException if and I/O error occurs.
 	 */
-	public int insert(final int doc, final int freq, final int fieldScore) throws IOException{		
+	public int insert(final int doc, final int freq, final int[] fieldFrequencies) throws IOException{		
 	  int c = insert(doc, freq);
-	  docIds.writeBinary(fieldTags, fieldScore);
+	  for(int field_f : fieldFrequencies)
+			docIds.writeGamma(field_f+1);
 	  return c;
 	}
 }
\ No newline at end of file
Index: src/uk/ac/gla/terrier/structures/indexing/singlepass/FieldsMemoryPostings.java
===================================================================
--- src/uk/ac/gla/terrier/structures/indexing/singlepass/FieldsMemoryPostings.java	(revision 2563)
+++ src/uk/ac/gla/terrier/structures/indexing/singlepass/FieldsMemoryPostings.java	(working copy)
@@ -29,6 +29,7 @@
 import java.io.IOException;
 
 import uk.ac.gla.terrier.structures.indexing.DocumentPostingList;
+import uk.ac.gla.terrier.structures.indexing.FieldDocumentPostingList;
 /**
  * Class for handling posting lists containing field information in memory while indexing.
  * @author Roi Blanco
@@ -38,7 +39,7 @@
 	
 	public void addTerms(DocumentPostingList docPostings, int docid) throws IOException{
 		for (String term : docPostings.termSet())
-			add(term, docid, docPostings.getFrequency(term), docPostings.getFields(term));
+			add(term, docid, docPostings.getFrequency(term), ((FieldDocumentPostingList)docPostings).getFieldFrequencies(term));
 	}
 	
 	/**
@@ -49,17 +50,17 @@
 	 * @param fieldScore int containing the field score for the term in the document.
 	 * @throws IOException if an I/O error occurs.
 	 */
-	public void add(String term, int doc, int frequency, int fieldScore) throws IOException{
+	public void add(String term, int doc, int frequency, int[] fieldFrequencies) throws IOException{
 		FieldPosting post;
 		if((post = (FieldPosting) postings.get(term)) != null) {						
-			post.insert(doc, frequency, fieldScore);
+			post.insert(doc, frequency, fieldFrequencies);
 			int tf = post.getTF();
 			// Update the max size
 			if(maxSize < tf) maxSize = tf; 
 		}
 		else{
 			post = new FieldPosting();
-			post.writeFirstDoc(doc, frequency, fieldScore);			
+			post.writeFirstDoc(doc, frequency, fieldFrequencies);			
 			postings.put(term,post);
 		}
 		numPointers++;
Index: src/uk/ac/gla/terrier/structures/indexing/singlepass/BlockInverted2DirectIndexBuilder.java
===================================================================
--- src/uk/ac/gla/terrier/structures/indexing/singlepass/BlockInverted2DirectIndexBuilder.java	(revision 2563)
+++ src/uk/ac/gla/terrier/structures/indexing/singlepass/BlockInverted2DirectIndexBuilder.java	(working copy)
@@ -132,17 +132,17 @@
 					System.arraycopy(postings4, blockIndex, blocks, 0, postings3[offset]);
                     if (prevUse[writerOffset])
                     {
-                        if (saveTagInformation)
-                            ((BlockFieldPosting)directPostings[writerOffset]).insert(termId, postings1[offset],  postings2[offset], blocks);
-                        else
+                        //if (saveTagInformation)
+                         //   ((BlockFieldPosting)directPostings[writerOffset]).insert(termId, postings1[offset],  postings2[offset], blocks);
+                        //else
                             ((BlockPosting)directPostings[writerOffset]).insert(termId, postings1[offset], blocks);
                     }
                     else
                     {
                         prevUse[writerOffset] = true;
-                        if (saveTagInformation)
-                            ((BlockFieldPosting)directPostings[writerOffset]).writeFirstDoc(termId, postings1[offset],  postings2[offset], blocks);
-                        else
+                        //if (saveTagInformation)
+                         //   ((BlockFieldPosting)directPostings[writerOffset]).writeFirstDoc(termId, postings1[offset],  postings2[offset], blocks);
+                        //else
                             ((BlockPosting)directPostings[writerOffset]).writeFirstDoc(termId, postings1[offset], blocks);
                     }
                 }
Index: src/uk/ac/gla/terrier/structures/indexing/singlepass/SimplePostingInRun.java
===================================================================
--- src/uk/ac/gla/terrier/structures/indexing/singlepass/SimplePostingInRun.java	(revision 2563)
+++ src/uk/ac/gla/terrier/structures/indexing/singlepass/SimplePostingInRun.java	(working copy)
@@ -29,7 +29,9 @@
 import java.io.IOException;
 
 import uk.ac.gla.terrier.compression.BitOut;
+import uk.ac.gla.terrier.structures.postings.BasicPosting;
 import uk.ac.gla.terrier.structures.postings.IterablePosting;
+import uk.ac.gla.terrier.structures.postings.WritablePosting;
 
 /** Class holding the information for a posting list read
  * from a previously written run at disk. Used in the merging phase of the Single pass inversion method.
@@ -114,6 +116,12 @@
 
 		public void setId(int id) {}	
 		public void close() throws IOException {	}
+
+		public WritablePosting asWritablePosting() {
+			BasicPosting bp = new BasicPosting(docid, frequency);
+			return bp;
+		}
+		
 	}
 
 	@Override
Index: src/uk/ac/gla/terrier/structures/indexing/singlepass/Inverted2DirectIndexBuilder.java
===================================================================
--- src/uk/ac/gla/terrier/structures/indexing/singlepass/Inverted2DirectIndexBuilder.java	(revision 2563)
+++ src/uk/ac/gla/terrier/structures/indexing/singlepass/Inverted2DirectIndexBuilder.java	(working copy)
@@ -46,6 +46,7 @@
 import uk.ac.gla.terrier.structures.InvertedIndexInputStream;
 import uk.ac.gla.terrier.structures.indexing.DocumentIndexBuilder;
 import uk.ac.gla.terrier.utility.ApplicationSetup;
+import uk.ac.gla.terrier.utility.ArrayUtils;
 import uk.ac.gla.terrier.utility.FieldScore;
 import uk.ac.gla.terrier.utility.Files;
 
@@ -94,6 +95,8 @@
 	/** Construct a new instance of this builder class */
 	public Inverted2DirectIndexBuilder(Index i)
 	{
+		if (saveTagInformation)
+			throw new Error("Fields are not yet supported by this class");
 		this.index =  i;
 	}
 	
@@ -230,7 +233,9 @@
 				directIndexInputStreamClass,
 				"uk.ac.gla.terrier.structures.Index,java.lang.String",
 				"index,structureName");
-			index.setIndexProperty("num.direct.fields.bits", ""+fieldTags);
+			index.setIndexProperty("index.direct.fields.count", ""+FieldScore.FIELDS_COUNT );
+			index.setIndexProperty("index.direct.fields.names", ArrayUtils.join(FieldScore.FIELD_NAMES, ","));
+			//index.setIndexProperty("num.direct.fields.bits", ""+fieldTags);
 			index.flush();//save changes
 			
 			logger.info("Finished generating a direct index from an inverted index. Time elapsed: "+((System.currentTimeMillis() - startTime)/1000) + " seconds");
@@ -289,7 +294,7 @@
 			termId++;
 			final int[] postings0 = postings[0];
 			final int[] postings1 = postings[1];
-			final int[] postings2 = saveTagInformation ? postings[2] : null;
+			//final int[] postings2 = saveTagInformation ? postings[2] : null;
 			int startOffset = Arrays.binarySearch(postings0, firstDocid);
 			int endOffset = Arrays.binarySearch(postings0, lastDocid+1);
 			if (startOffset < 0)
@@ -312,17 +317,17 @@
 					numPostings++;
 					if (prevUse[writerOffset])
 					{
-						if (saveTagInformation)
-							((FieldPosting)directPostings[writerOffset]).insert(termId, postings1[offset],  postings2[offset]);
-						else
+						//if (saveTagInformation)
+						//	((FieldPosting)directPostings[writerOffset]).insert(termId, postings1[offset],  postings2[offset]);
+						//else
 							directPostings[writerOffset].insert(termId, postings1[offset]);
 					}
 					else
 					{
 						prevUse[writerOffset] = true;
-						if (saveTagInformation)
-							((FieldPosting)directPostings[writerOffset]).writeFirstDoc(termId, postings1[offset],  postings2[offset]);
-						else
+						//if (saveTagInformation)
+						//	((FieldPosting)directPostings[writerOffset]).writeFirstDoc(termId, postings1[offset],  postings2[offset]);
+						//else
 							directPostings[writerOffset].writeFirstDoc(termId, postings1[offset]);
 					}
 				}
Index: src/uk/ac/gla/terrier/structures/indexing/singlepass/BlockFieldMemoryPostings.java
===================================================================
--- src/uk/ac/gla/terrier/structures/indexing/singlepass/BlockFieldMemoryPostings.java	(revision 2563)
+++ src/uk/ac/gla/terrier/structures/indexing/singlepass/BlockFieldMemoryPostings.java	(working copy)
@@ -29,7 +29,7 @@
 
 import java.io.IOException;
 
-import uk.ac.gla.terrier.structures.indexing.BlockDocumentPostingList;
+import uk.ac.gla.terrier.structures.indexing.BlockFieldDocumentPostingList;
 import uk.ac.gla.terrier.structures.indexing.DocumentPostingList;
 /**
  * Class for handling posting lists containing block and field information in memory while indexing.
@@ -39,22 +39,22 @@
 public class BlockFieldMemoryPostings extends BlockMemoryPostings{
 	
 	public void addTerms(DocumentPostingList _docPostings, int docid) throws IOException {
-		BlockDocumentPostingList docPostings = (BlockDocumentPostingList)  _docPostings;
+		BlockFieldDocumentPostingList docPostings = (BlockFieldDocumentPostingList) _docPostings;
 		for (String term : docPostings.termSet())
-			add(term, docid, docPostings.getFrequency(term) , docPostings.getFields(term), docPostings.getBlocks(term));
+			add(term, docid, docPostings.getFrequency(term) , docPostings.getFieldFrequencies(term), docPostings.getBlocks(term));
 	}
 	
 	
-	public void add(String term, int doc, int frequency, int fieldScore, int[] blockids)  throws IOException{
+	public void add(String term, int doc, int frequency, int[] fieldFrequencies, int[] blockids)  throws IOException{
 		BlockFieldPosting post;	
 		if((post =(BlockFieldPosting) postings.get(term)) != null) {						
-			post.insert(doc, frequency, fieldScore, blockids);
+			post.insert(doc, frequency, fieldFrequencies, blockids);
 			int tf = post.getTF();			
 			if(maxSize < tf) maxSize = tf; 
 		}
 		else{
 			post = new BlockFieldPosting();
-			post.writeFirstDoc(doc, frequency, fieldScore, blockids);			
+			post.writeFirstDoc(doc, frequency, fieldFrequencies, blockids);			
 			postings.put(term,post);
 		}
 		numPointers++;
Index: src/uk/ac/gla/terrier/structures/indexing/FieldDocumentPostingList.java
===================================================================
--- src/uk/ac/gla/terrier/structures/indexing/FieldDocumentPostingList.java	(revision 0)
+++ src/uk/ac/gla/terrier/structures/indexing/FieldDocumentPostingList.java	(revision 0)
@@ -0,0 +1,179 @@
+package uk.ac.gla.terrier.structures.indexing;
+
+import gnu.trove.TObjectIntHashMap;
+import gnu.trove.TObjectIntProcedure;
+
+import java.util.Arrays;
+
+import uk.ac.gla.terrier.sorting.HeapSortInt;
+import uk.ac.gla.terrier.structures.DocumentIndexEntry;
+import uk.ac.gla.terrier.structures.FieldDocumentIndexEntry;
+import uk.ac.gla.terrier.structures.postings.FieldPosting;
+import uk.ac.gla.terrier.structures.postings.FieldPostingImpl;
+import uk.ac.gla.terrier.structures.postings.IterablePosting;
+import uk.ac.gla.terrier.structures.postings.WritablePosting;
+import uk.ac.gla.terrier.utility.TermCodes;
+
+public class FieldDocumentPostingList extends DocumentPostingList {
+	
+	/** number of fields */
+	protected final int fieldCount;	
+	/** length of each field */
+	protected final int[] fieldLengths;
+	/** occurrences of terms in fields */
+	protected final TObjectIntHashMap<String>[] field_occurrences;
+	
+	@SuppressWarnings("unchecked")
+	public FieldDocumentPostingList(final int NUM_FIELDS)
+	{
+		super();
+		this.fieldCount = NUM_FIELDS;
+		fieldLengths = new int[fieldCount];
+		field_occurrences = new TObjectIntHashMap[fieldCount];
+		for(int i=0;i<fieldCount;i++)
+		{
+			field_occurrences[i] = new TObjectIntHashMap<String>(AVG_DOCUMENT_UNIQUE_TERMS);
+		}
+	}
+	
+	/** Insert a term into the posting list of this document, in the given field, with the given frequency
+	  * @param tf frequency of the term in this document
+	  * @param term String form of term
+	  * @param fieldNum fieldNumber it occurs in */
+	public void insert(final int tf, final String term, final int fieldNum)
+	{
+		occurrences.adjustOrPutValue(term,tf,tf);
+		field_occurrences[fieldNum].adjustOrPutValue(term, tf, tf);
+		fieldLengths[fieldNum]+=tf;
+		documentLength+=tf;
+	}
+	/**  Insert a term into the posting list of this document, in the given field
+	  * @param term the Term being inserted
+	  * @param fieldNum the id of the field that the term was found in */
+	public void insert(final String term, final int fieldNum)
+	{
+		occurrences.adjustOrPutValue(term,1,1);
+		field_occurrences[fieldNum].adjustOrPutValue(term, 1, 1);
+		fieldLengths[fieldNum]++;
+		documentLength++;
+	}
+
+	/**  Insert a term into the posting list of this document, in the given field
+	  * @param term the Term being inserted
+	  * @param fieldNums the ids of the fields that the term was found in */
+	public void insert(final String term, final int[] fieldNums)
+	{
+		occurrences.adjustOrPutValue(term,1,1);
+		for(int fieldId : fieldNums)
+		{
+			if (fieldId == -1)
+				continue;
+			field_occurrences[fieldId].adjustOrPutValue(term, 1, 1);
+			fieldLengths[fieldId]++;
+		}
+		documentLength++;
+	}
+
+	/**  Insert a term into the posting list of this document, in the given field
+	  * @param tf the frequency of the term
+	  * @param term the Term being inserted
+	  * @param fieldNums the ids of the fields that the term was found in */
+	public void insert(final int tf, final String term, final int[] fieldNums)
+	{
+		occurrences.adjustOrPutValue(term,tf,tf);
+		for(int fieldId : fieldNums)
+		{
+			field_occurrences[fieldId].adjustOrPutValue(term, tf, tf);
+			fieldLengths[fieldId]+=tf;
+		}
+		documentLength+=tf;
+	}
+	
+	/** Return the frequencies of the specified term in all of the fields */
+	public int[] getFieldFrequencies(final String term)
+	{
+		final int[] rtr = new int[fieldCount];
+		for(int i=0;i<fieldCount;i++)
+			rtr[i] = field_occurrences[i].get(term);
+		return rtr;
+	}
+	
+	public DocumentIndexEntry getDocumentStatistics()
+	{
+		FieldDocumentIndexEntry fdie = new FieldDocumentIndexEntry(this.fieldCount);
+		fdie.setDocumentLength(documentLength);
+		fdie.setNumberOfEntries(occurrences.size());
+		fdie.setFieldLengths(fieldLengths);
+		return fdie;
+	}
+
+	@Override
+	public void clear() {
+		super.clear();
+		Arrays.fill(fieldLengths, 0);
+	}
+
+	//TODO FIX
+	@Override
+	public int[][] getPostings() {
+		//throw new Error("Unsupported")
+		
+		final int termCount = occurrences.size();
+		final int[][] postings = new int[fieldCount + 2][termCount];
+		//final int[] termids = new int[termCount];
+		//final int[] tfs = new int[termCount];
+		
+		occurrences.forEachEntry( new TObjectIntProcedure<String>() {
+			int i=0;
+			public boolean execute(final String a, final int b)
+			{
+				postings[0][i] = TermCodes.getCode(a);
+				postings[1][i] = b;
+				for(int fi=0;fi< fieldCount;fi++)
+					postings[2+fi][i] = field_occurrences[fi].get(a);
+				//fields[i++] = term_fields.get(a);
+				i++;
+				return true;
+			}
+		});	
+		HeapSortInt.ascendingHeapSort(postings);
+		return postings;
+	}
+
+	class fieldPostingIterator 
+		extends postingIterator
+		implements FieldPosting
+	{
+		int[] fieldFrequencies = new int[fieldCount];
+		public fieldPostingIterator(int[][] postings)
+		{
+			super(postings);
+		}
+		
+		public int[] getFieldFrequencies()
+		{
+			for(int fi=0;fi<fieldCount;fi++)
+			{
+				fieldFrequencies[fi] = allPostings[fi+2][i];
+			}
+			return fieldFrequencies;
+		}
+
+		public int[] getFieldLengths() {
+			return null;
+		}
+		
+		@Override
+		public WritablePosting asWritablePosting() {
+			FieldPostingImpl fbp = new FieldPostingImpl(termids[i],tfs[i], fieldCount);
+			System.arraycopy(getFieldFrequencies(), 0, fbp.getFieldFrequencies(), 0, fieldCount);
+			return fbp;
+		}
+		
+	}
+	
+	@Override
+	public IterablePosting getPostings2() {
+		return new fieldPostingIterator(getPostings());
+	}
+}
Index: src/uk/ac/gla/terrier/structures/indexing/BlockFieldLexiconMap.java
===================================================================
--- src/uk/ac/gla/terrier/structures/indexing/BlockFieldLexiconMap.java	(revision 0)
+++ src/uk/ac/gla/terrier/structures/indexing/BlockFieldLexiconMap.java	(revision 0)
@@ -0,0 +1,81 @@
+package uk.ac.gla.terrier.structures.indexing;
+
+import gnu.trove.TObjectIntHashMap;
+import gnu.trove.TObjectIntProcedure;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+import uk.ac.gla.terrier.structures.BlockFieldLexiconEntry;
+import uk.ac.gla.terrier.structures.LexiconOutputStream;
+import uk.ac.gla.terrier.utility.TermCodes;
+
+public class BlockFieldLexiconMap extends BlockLexiconMap {
+	protected final int fieldCount;
+	protected final TObjectIntHashMap<String>field_tfs[]; 
+	
+	@SuppressWarnings("unchecked")
+	public BlockFieldLexiconMap(int _fieldCount)
+	{
+		super();
+		fieldCount = _fieldCount;
+		field_tfs = new TObjectIntHashMap[fieldCount];
+		for(int fi=0;fi<fieldCount;fi++)
+			field_tfs[fi] = new TObjectIntHashMap<String>(BUNDLE_AVG_UNIQUE_TERMS);
+	}
+	
+	protected int[] getFieldFrequency(String term)
+	{
+		int[] fieldFrequencies = new int[fieldCount];
+		for(int fi=0;fi<fieldCount;fi++)
+			fieldFrequencies[fi] = field_tfs[fi].get(term);
+		return fieldFrequencies;
+	}
+	
+	/** Inserts all the terms from a document posting
+	  * into the lexicon map
+	  * @param doc The postinglist for that document
+	  */
+	public void insert(DocumentPostingList _doc)
+	{
+		super.insert(_doc);
+		BlockFieldDocumentPostingList doc = (BlockFieldDocumentPostingList)_doc;
+		int fi = 0;
+		for(TObjectIntHashMap<String> docField : doc.field_occurrences)
+		{
+			final TObjectIntHashMap<String> thisField = field_tfs[fi];
+			docField.forEachEntry(new TObjectIntProcedure<String>() {
+				public boolean execute(String arg0, int arg1) {
+					thisField.adjustOrPutValue(arg0, arg1, arg1);
+					return true;
+				}
+			});
+			fi++;
+		}
+	}
+	
+	/** Stores the lexicon tree to a lexicon stream as a sequence of entries.
+	  * The binary tree is traversed in order, by called the method
+	  * traverseAndStoreToStream.
+	  * @param lexiconStream The lexicon output stream to store to. */
+	public void storeToStream(LexiconOutputStream<String> lexiconStream) throws IOException
+	{
+		final String[] terms = tfs.keys(new String[0]);
+		Arrays.sort(terms);
+		for (String t : terms)
+		{
+			BlockFieldLexiconEntry fle = new BlockFieldLexiconEntry(getFieldFrequency(t), blockFreqs.get(t));
+			fle.setOffset((long)0, (byte)0);
+			fle.setTermId(TermCodes.getCode(t));
+			fle.setStatistics(nts.get(t), tfs.get(t));	
+			lexiconStream.writeNextEntry(t, fle);
+		}
+	}
+
+	@Override
+	public void clear() {
+		super.clear();
+		for(int fi=0;fi<fieldCount;fi++)
+			field_tfs[fi].clear();
+	}
+}
Index: src/uk/ac/gla/terrier/structures/FieldDocumentIndexEntry.java
===================================================================
--- src/uk/ac/gla/terrier/structures/FieldDocumentIndexEntry.java	(revision 0)
+++ src/uk/ac/gla/terrier/structures/FieldDocumentIndexEntry.java	(revision 0)
@@ -0,0 +1,67 @@
+package uk.ac.gla.terrier.structures;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+/** A document index entry for use with fields */
+public class FieldDocumentIndexEntry extends BasicDocumentIndexEntry {
+
+	public static class Factory extends BasicDocumentIndexEntry.Factory
+	{	
+		protected int fieldCount;
+		public Factory(int fieldCount)
+		{
+			this.fieldCount = fieldCount;
+		}
+		
+		public Factory(String _fieldCount)
+		{
+			this(Integer.parseInt(_fieldCount));
+		}
+
+		@Override
+		public int getSize() {
+			return super.getSize() + fieldCount * 4;
+		}
+
+		@Override
+		public DocumentIndexEntry newInstance() {
+			return new FieldDocumentIndexEntry(fieldCount);
+		}
+		
+	}
+	
+	protected int[] fieldLengths;
+	
+	public FieldDocumentIndexEntry(int fieldCount) 
+	{
+		this.fieldLengths = new int[fieldCount];
+	}
+	
+	public int[] getFieldLengths()
+	{
+		return this.fieldLengths;
+	}
+	
+	public void setFieldLengths(int[] f_lens)
+	{
+		this.fieldLengths = f_lens;
+	}
+	
+	@Override
+	public void readFields(DataInput in) throws IOException {
+		super.readFields(in);
+		final int l = fieldLengths.length;
+		for(int i=0;i<l;i++)
+			fieldLengths[i] = in.readInt();
+	}
+
+	@Override
+	public void write(DataOutput out) throws IOException {
+		super.write(out);
+		for(int field_l : fieldLengths)
+			out.writeInt(field_l);
+	}
+
+}
Index: src/uk/ac/gla/terrier/structures/BlockFieldLexiconEntry.java
===================================================================
--- src/uk/ac/gla/terrier/structures/BlockFieldLexiconEntry.java	(revision 0)
+++ src/uk/ac/gla/terrier/structures/BlockFieldLexiconEntry.java	(revision 0)
@@ -0,0 +1,79 @@
+package uk.ac.gla.terrier.structures;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import uk.ac.gla.terrier.structures.seralization.FixedSizeWriteableFactory;
+
+/** A LexiconEntry with field support */
+public class BlockFieldLexiconEntry extends BasicLexiconEntry implements
+		FieldEntryStatistics, BlockEntryStatistics {
+
+	public static class Factory implements FixedSizeWriteableFactory<LexiconEntry>
+	{	
+		protected int fieldCount;
+		public Factory(int fieldCount)
+		{
+			this.fieldCount = fieldCount;
+		}
+		
+		public Factory(String _fieldCount)
+		{
+			this(Integer.parseInt(_fieldCount));
+		}
+		
+		public int getSize() {
+			return (3*4) + 8 + 1 + this.fieldCount * 4;
+		}
+		
+		public LexiconEntry newInstance() {
+			return new BlockFieldLexiconEntry(fieldCount);
+		}
+	}	
+	
+	protected final int[] fieldFrequencies;
+	protected int blockCount;
+	
+	public BlockFieldLexiconEntry(int fieldCount)
+	{
+		fieldFrequencies = new int[fieldCount];
+	}
+	
+	public BlockFieldLexiconEntry(int[] _fieldFrequencies, int _blockCount)
+	{
+		fieldFrequencies = _fieldFrequencies;
+		blockCount = _blockCount;
+	}
+	
+	/** {@inheritDoc} */
+	public int[] getFieldFrequencies() {
+		return fieldFrequencies;
+	}
+	
+	public void setFieldFrequencies(int[] _fieldFrequencices) {
+		System.arraycopy(_fieldFrequencices, 0, fieldFrequencies, 0, fieldFrequencies.length);
+	}
+
+	@Override
+	public void readFields(DataInput in) throws IOException {
+		super.readFields(in);
+		final int l = fieldFrequencies.length;
+		for(int i=0;i<l;i++)
+			fieldFrequencies[i] = in.readInt();
+	}
+
+	@Override
+	public void write(DataOutput out) throws IOException {
+		super.write(out);
+		for(int field_f : fieldFrequencies)
+			out.writeInt(field_f);
+	}
+
+	/** {@inheritDoc} */
+	public int getBlockCount()
+	{
+		return blockCount;
+	}	
+
+}
Index: src/uk/ac/gla/terrier/structures/FieldLexiconEntry.java
===================================================================
--- src/uk/ac/gla/terrier/structures/FieldLexiconEntry.java	(revision 0)
+++ src/uk/ac/gla/terrier/structures/FieldLexiconEntry.java	(revision 0)
@@ -0,0 +1,79 @@
+package uk.ac.gla.terrier.structures;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import uk.ac.gla.terrier.structures.seralization.FixedSizeWriteableFactory;
+import uk.ac.gla.terrier.utility.FieldScore;
+
+/** A LexiconEntry with field support */
+public class FieldLexiconEntry extends BasicLexiconEntry implements
+		FieldEntryStatistics {
+
+	public static class Factory implements FixedSizeWriteableFactory<LexiconEntry>
+	{	
+		protected int fieldCount;
+		
+		public Factory()
+		{
+			this(FieldScore.FIELDS_COUNT); //TODO this is a hack
+			System.err.println(this.getClass().getName() + "- default constructor should not be used");
+		}
+		
+		public Factory(int fieldCount)
+		{
+			this.fieldCount = fieldCount;
+		}
+		
+		public Factory(String _fieldCount)
+		{
+			this(Integer.parseInt(_fieldCount));
+		}
+		
+		public int getSize() {
+			return (3*4) + 8 + 1 + this.fieldCount * 4;
+		}
+		
+		public LexiconEntry newInstance() {
+			return new FieldLexiconEntry(fieldCount);
+		}
+	}	
+	
+	protected final int[] fieldFrequencies;
+	
+	public FieldLexiconEntry(int fieldCount)
+	{
+		fieldFrequencies = new int[fieldCount];
+	}
+	
+	public FieldLexiconEntry(int[] _fieldFrequencies)
+	{
+		fieldFrequencies = _fieldFrequencies;
+	}
+	
+	/** {@inheritDoc} */
+	public int[] getFieldFrequencies() {
+		return fieldFrequencies;
+	}
+	
+	public void setFieldFrequencies(int[] _fieldFrequencices) {
+		System.arraycopy(_fieldFrequencices, 0, fieldFrequencies, 0, fieldFrequencies.length);
+	}
+
+	@Override
+	public void readFields(DataInput in) throws IOException {
+		super.readFields(in);
+		final int l = fieldFrequencies.length;
+		for(int i=0;i<l;i++)
+			fieldFrequencies[i] = in.readInt();
+	}
+
+	@Override
+	public void write(DataOutput out) throws IOException {
+		super.write(out);
+		for(int field_f : fieldFrequencies)
+			out.writeInt(field_f);
+	}	
+
+}
Index: src/uk/ac/gla/terrier/structures/FieldDirectInvertedOutputStream.java
===================================================================
--- src/uk/ac/gla/terrier/structures/FieldDirectInvertedOutputStream.java	(revision 0)
+++ src/uk/ac/gla/terrier/structures/FieldDirectInvertedOutputStream.java	(revision 0)
@@ -0,0 +1,30 @@
+package uk.ac.gla.terrier.structures;
+
+import java.io.IOException;
+
+import uk.ac.gla.terrier.compression.BitOut;
+import uk.ac.gla.terrier.structures.postings.FieldPosting;
+import uk.ac.gla.terrier.structures.postings.Posting;
+/** Bit out class for writing a posting list with fields */
+public class FieldDirectInvertedOutputStream extends DirectInvertedOutputStream {
+
+	public FieldDirectInvertedOutputStream(BitOut out) {
+		super(out);
+	}
+
+	public FieldDirectInvertedOutputStream(String filename)
+			throws IOException {
+		super(filename);
+	}
+
+	@Override
+	protected void writePostingNotDocid(Posting _p) throws IOException {
+		super.writePostingNotDocid(_p);
+		final FieldPosting p = (FieldPosting)_p;
+		for(int fieldFrequency : p.getFieldFrequencies())
+		{
+			super.output.writeUnary(fieldFrequency +1);
+		}
+	}
+
+}
Index: src/uk/ac/gla/terrier/structures/DirectIndex.java
===================================================================
--- src/uk/ac/gla/terrier/structures/DirectIndex.java	(revision 2563)
+++ src/uk/ac/gla/terrier/structures/DirectIndex.java	(working copy)
@@ -43,8 +43,8 @@
 public class DirectIndex extends BitPostingIndex {
 	
 	/** Indicates whether field information is indexed. */
-	protected static final boolean saveTagInformation = 
-		FieldScore.USE_FIELD_INFORMATION;
+	//protected static final boolean saveTagInformation = 
+	//	FieldScore.USE_FIELD_INFORMATION;
 	
 	
 	/** The document index employed for retrieving the document offsets.*/
Index: src/uk/ac/gla/terrier/structures/InvertedIndex.java
===================================================================
--- src/uk/ac/gla/terrier/structures/InvertedIndex.java	(revision 2563)
+++ src/uk/ac/gla/terrier/structures/InvertedIndex.java	(working copy)
@@ -92,16 +92,16 @@
 	@Override
 	public IterablePosting getPostings(BitIndexPointer pointer) throws IOException {
 		final BitIn file = this.file.readReset(pointer.getOffset(), pointer.getOffsetBits());
-		final int fieldCount = FieldScore.FIELDS_COUNT;
-		if (fieldCount > 0)
-		{
-			throw new IOException("Fields not support this way yet");
-		}
 		IterablePosting rtr = null;
 		try{
-			rtr = postingImplementation
-				.getConstructor(BitIn.class, Integer.TYPE, DocumentIndex.class)
-				.newInstance(file, pointer.getNumberOfEntries(), doi);
+			if (fieldCount > 0)
+				rtr = postingImplementation
+					.getConstructor(BitIn.class, Integer.TYPE, DocumentIndex.class, Integer.TYPE)
+					.newInstance(file, pointer.getNumberOfEntries(), doi, fieldCount);
+			else
+				rtr = postingImplementation
+					.getConstructor(BitIn.class, Integer.TYPE, DocumentIndex.class)
+					.newInstance(file, pointer.getNumberOfEntries(), doi);
 		} catch (Exception e) {
 			throw new WrappedIOException(e);
 		}
@@ -112,7 +112,6 @@
 	public int[][] getDocuments(BitIndexPointer pointer) {
 		if (pointer==null)
 			return null;
-		final int fieldCount = FieldScore.FIELDS_COUNT;
 		final boolean loadTagInformation = FieldScore.USE_FIELD_INFORMATION;
 		final int count = pointer.getNumberOfEntries();
 		try{
Index: src/uk/ac/gla/terrier/structures/BitPostingIndexInputStream.java
===================================================================
--- src/uk/ac/gla/terrier/structures/BitPostingIndexInputStream.java	(revision 2563)
+++ src/uk/ac/gla/terrier/structures/BitPostingIndexInputStream.java	(working copy)
@@ -6,7 +6,7 @@
 import uk.ac.gla.terrier.compression.BitIn;
 import uk.ac.gla.terrier.compression.BitInputStream;
 import uk.ac.gla.terrier.structures.postings.IterablePosting;
-import uk.ac.gla.terrier.utility.FieldScore;
+import uk.ac.gla.terrier.utility.Files;
 import uk.ac.gla.terrier.utility.io.WrappedIOException;
 
 public class BitPostingIndexInputStream implements PostingIndexInputStream {
@@ -18,14 +18,18 @@
 
 	protected Class<? extends IterablePosting> postingIteratorClass;
 	
+	protected int fieldCount;
+	
 	public BitPostingIndexInputStream(
 			Index _index, String structureName, 
 			Iterator<? extends BitIndexPointer> _pointerList,
 			Class<? extends IterablePosting> _postingIteratorClass) throws IOException
 	{
+		System.err.println(Files.length(_index.getPath() + "/" + _index.getPrefix() +"."+ structureName + BitIn.USUAL_EXTENSION));
 		file = new BitInputStream(_index.getPath() + "/" + _index.getPrefix() +"."+ structureName + BitIn.USUAL_EXTENSION);
 		pointerList = _pointerList;
 		postingIteratorClass = _postingIteratorClass;
+		fieldCount = _index.getIntIndexProperty("index."+structureName+".fields.count", 0);
 	}
 	
 	/** {@inheritDoc} */
@@ -37,16 +41,16 @@
 	
 	protected IterablePosting loadPostingIterator(BitIndexPointer pointer) throws IOException
 	{
-		final int fieldCount = FieldScore.FIELDS_COUNT;
-		if (fieldCount > 0)
-		{
-			throw new IOException("Fields not support this way");
-		}
 		IterablePosting rtr = null;
 		try{
-			rtr = postingIteratorClass
-				.getConstructor(BitIn.class, Integer.TYPE, DocumentIndex.class)
-				.newInstance(file, pointer.getNumberOfEntries(), null);
+			if (fieldCount > 0)
+				rtr = postingIteratorClass
+					.getConstructor(BitIn.class, Integer.TYPE, DocumentIndex.class, Integer.TYPE)
+					.newInstance(file, pointer.getNumberOfEntries(), null, fieldCount);
+			else
+				rtr = postingIteratorClass
+					.getConstructor(BitIn.class, Integer.TYPE, DocumentIndex.class)
+					.newInstance(file, pointer.getNumberOfEntries(), null);
 		} catch (Exception e) {
 			throw new WrappedIOException(e);
 		}
Index: src/uk/ac/gla/terrier/structures/postings/BlockFieldIterablePosting.java
===================================================================
--- src/uk/ac/gla/terrier/structures/postings/BlockFieldIterablePosting.java	(revision 0)
+++ src/uk/ac/gla/terrier/structures/postings/BlockFieldIterablePosting.java	(revision 0)
@@ -0,0 +1,96 @@
+/**
+ * 
+ */
+package uk.ac.gla.terrier.structures.postings;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.hadoop.io.WritableUtils;
+
+import uk.ac.gla.terrier.compression.BitIn;
+import uk.ac.gla.terrier.structures.DocumentIndex;
+import uk.ac.gla.terrier.structures.FieldDocumentIndexEntry;
+
+public class BlockFieldIterablePosting extends BasicIterablePosting implements BlockPosting, FieldPosting
+{
+	int[] positions;
+	final int fieldCount;
+	final int[] fieldFrequencies;
+	
+	public BlockFieldIterablePosting(int fieldCount){
+		super();
+		this.fieldCount = fieldCount;
+		this.fieldFrequencies = new int[fieldCount];
+	}
+	
+	public BlockFieldIterablePosting(BitIn _bitFileReader, int _numEntries, DocumentIndex doi, int fieldCount) throws IOException {
+		super(_bitFileReader, _numEntries, doi);
+		this.fieldCount = fieldCount;
+		this.fieldFrequencies = new int[fieldCount];
+	}
+	
+	public boolean next() throws IOException {
+		if (numEntries-- == 0)
+			return false;
+		id = bitFileReader.readGamma() + id;
+		tf = bitFileReader.readUnary();
+		for(int i = 0;i<fieldCount;i++)
+		{
+			fieldFrequencies[i] = bitFileReader.readUnary()-1;
+		}
+		//TODO: this has a memory allocation for every posting in the posting list. can we reuse an array?
+		positions = new int[bitFileReader.readUnary() -1];
+		positions[0] = bitFileReader.readGamma() -1;
+		for(int i=1;i<positions.length;i++)
+			positions[i] = positions[i-1] + bitFileReader.readGamma();
+		return true;
+	}
+	
+	public int[] getPositions() {
+		return positions;
+	}
+	
+	public int[] getFieldFrequencies() {
+		return fieldFrequencies;
+	}
+	
+	public int[] getFieldLengths() {
+		FieldDocumentIndexEntry fdie = null;
+		try{
+			fdie = ((FieldDocumentIndexEntry)doi.getDocumentEntry(id));
+		} catch (IOException ioe) {
+			//TODO log?
+			System.err.println("Problem looking for doclength for document "+ id);
+			ioe.printStackTrace();
+			return new int[0];
+		}
+		return fdie.getFieldLengths();
+	}
+
+	@Override
+	public void readFields(DataInput in) throws IOException {
+		super.readFields(in);
+		final int blockCount = WritableUtils.readVInt(in);
+		final int l = in.readInt();
+		for(int i=0;i<l;i++)
+			fieldFrequencies[i] = in.readInt();
+		positions = new int[blockCount]; 
+		for(int i=0;i<blockCount;i++)
+			positions[i] = WritableUtils.readVInt(in);
+	}
+
+	@Override
+	public void write(DataOutput out) throws IOException {
+		super.write(out);
+		out.writeInt(fieldFrequencies.length);
+		for(int field_f : fieldFrequencies)
+			out.writeInt(field_f);
+		WritableUtils.writeVInt(out, positions.length);
+		for(int pos : positions)
+			WritableUtils.writeVInt(out, pos);
+	}
+
+	
+}
\ No newline at end of file
Index: src/uk/ac/gla/terrier/structures/postings/BasicPosting.java
===================================================================
--- src/uk/ac/gla/terrier/structures/postings/BasicPosting.java	(revision 2563)
+++ src/uk/ac/gla/terrier/structures/postings/BasicPosting.java	(working copy)
@@ -15,6 +15,12 @@
 		super();
 	}
 
+	public BasicPosting(int docid, int frequency) {
+		super();
+		id = docid;
+		tf = frequency;
+	}
+
 	public int getId() {
 		return id;
 	}
Index: src/uk/ac/gla/terrier/structures/postings/BlockFieldPostingImpl.java
===================================================================
--- src/uk/ac/gla/terrier/structures/postings/BlockFieldPostingImpl.java	(revision 0)
+++ src/uk/ac/gla/terrier/structures/postings/BlockFieldPostingImpl.java	(revision 0)
@@ -0,0 +1,45 @@
+package uk.ac.gla.terrier.structures.postings;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+public class BlockFieldPostingImpl extends BlockPostingImpl implements FieldPosting {
+
+	int[] fieldFrequencies;
+	
+	public BlockFieldPostingImpl() {
+		// TODO Auto-generated constructor stub
+	}
+
+	public BlockFieldPostingImpl(int docid, int frequency, int[] _positions, int fieldCount) {
+		super(docid, frequency, _positions);
+		fieldFrequencies = new int[fieldCount]; 
+	}
+
+	public int[] getFieldFrequencies() {
+		return fieldFrequencies;
+	}
+
+	public int[] getFieldLengths() {
+		return null;
+	}
+	
+	@Override
+	public void readFields(DataInput in) throws IOException {
+		super.readFields(in);
+		final int l = in.readInt();
+		fieldFrequencies = new int[l];
+		for(int i=0;i<l;i++)
+			fieldFrequencies[i] = in.readInt();
+	}
+
+	@Override
+	public void write(DataOutput out) throws IOException {
+		super.write(out);
+		out.writeInt(fieldFrequencies.length);
+		for(int field_f : fieldFrequencies)
+			out.writeInt(field_f);
+	}
+
+}
Index: src/uk/ac/gla/terrier/structures/postings/IterablePosting.java
===================================================================
--- src/uk/ac/gla/terrier/structures/postings/IterablePosting.java	(revision 2563)
+++ src/uk/ac/gla/terrier/structures/postings/IterablePosting.java	(working copy)
@@ -7,4 +7,6 @@
 	/** move to the next document.
      * return false iff end of posting list has been met */
 	public boolean next() throws IOException;
+	
+	public WritablePosting asWritablePosting();
 }
Index: src/uk/ac/gla/terrier/structures/postings/FieldPostingImpl.java
===================================================================
--- src/uk/ac/gla/terrier/structures/postings/FieldPostingImpl.java	(revision 0)
+++ src/uk/ac/gla/terrier/structures/postings/FieldPostingImpl.java	(revision 0)
@@ -0,0 +1,53 @@
+package uk.ac.gla.terrier.structures.postings;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+public class FieldPostingImpl extends BasicPosting implements FieldPosting {
+	int[] fieldFrequencies;	
+	
+	public FieldPostingImpl() {}
+	
+	public FieldPostingImpl(int id, int tf, int _fieldCount)
+	{
+		super(id,tf);
+		fieldFrequencies = new int[_fieldCount];
+	}
+	
+	public FieldPostingImpl(int[] _fieldFrequencies)
+	{
+		fieldFrequencies = _fieldFrequencies;
+	}
+	
+	public FieldPostingImpl(int _fieldCount)
+	{
+		fieldFrequencies = new int[_fieldCount];
+	}
+	
+	public int[] getFieldFrequencies() {
+		return fieldFrequencies;
+	}
+	
+	public int[] getFieldLengths() {
+		return null;
+	}
+	
+	@Override
+	public void readFields(DataInput in) throws IOException {
+		super.readFields(in);
+		final int l = in.readInt();
+		fieldFrequencies = new int[l];
+		for(int i=0;i<l;i++)
+			fieldFrequencies[i] = in.readInt();
+	}
+
+	@Override
+	public void write(DataOutput out) throws IOException {
+		super.write(out);
+		out.writeInt(fieldFrequencies.length);
+		for(int field_f : fieldFrequencies)
+			out.writeInt(field_f);
+	}
+	
+}
Index: src/uk/ac/gla/terrier/structures/postings/FieldIterablePosting.java
===================================================================
--- src/uk/ac/gla/terrier/structures/postings/FieldIterablePosting.java	(revision 0)
+++ src/uk/ac/gla/terrier/structures/postings/FieldIterablePosting.java	(revision 0)
@@ -0,0 +1,88 @@
+package uk.ac.gla.terrier.structures.postings;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import uk.ac.gla.terrier.compression.BitIn;
+import uk.ac.gla.terrier.structures.DocumentIndex;
+import uk.ac.gla.terrier.structures.FieldDocumentIndexEntry;
+
+/** A posting iterator for field postings */
+public class FieldIterablePosting extends BasicIterablePosting implements FieldPosting {
+
+	final int fieldCount;
+	final int[] fieldFrequencies;	
+	
+	public FieldIterablePosting(int fieldCount) {
+		super();
+		this.fieldCount = fieldCount;
+		this.fieldFrequencies = new int[fieldCount];
+	}
+
+	public FieldIterablePosting(BitIn fileReader, int entries, DocumentIndex _doi, int fieldCount) throws IOException {
+		super(fileReader, entries, _doi);
+		this.fieldCount = fieldCount;
+		this.fieldFrequencies = new int[fieldCount];
+	}
+
+	public int[] getFieldFrequencies() {
+		return fieldFrequencies;
+	}
+
+	@Override
+	public boolean next() throws IOException {
+		if (numEntries-- == 0)
+			return false;
+		id = bitFileReader.readGamma() + id;
+		tf = bitFileReader.readUnary();
+		for(int i = 0;i<fieldCount;i++)
+		{
+			fieldFrequencies[i] = bitFileReader.readUnary()-1;
+		}
+		return true;
+	}
+
+	public int[] getFieldLengths() {
+		FieldDocumentIndexEntry fdie = null;
+		try{
+			fdie = ((FieldDocumentIndexEntry)doi.getDocumentEntry(id));
+		} catch (IOException ioe) {
+			//TODO log?
+			System.err.println("Problem looking for doclength for document "+ id);
+			ioe.printStackTrace();
+			return new int[0];
+		}
+		return fdie.getFieldLengths();
+	}
+
+	@Override
+	public void readFields(DataInput in) throws IOException {
+		super.readFields(in);
+		final int l = in.readInt();
+		//TODO array missize?
+		for(int i=0;i<l;i++)
+			fieldFrequencies[i] = in.readInt();
+	}
+
+	@Override
+	public void write(DataOutput out) throws IOException {
+		super.write(out);
+		out.writeInt(fieldFrequencies.length);
+		for(int field_f : fieldFrequencies)
+			out.writeInt(field_f);
+	}
+	
+
+	@Override
+	public WritablePosting asWritablePosting()
+	{	
+		FieldPostingImpl fbp = new FieldPostingImpl(fieldCount);
+		fbp.id = id;
+		fbp.tf = tf;
+		System.arraycopy(fieldFrequencies, 0, fbp.fieldFrequencies, 0, fieldCount);
+		return fbp;
+	}
+
+	
+}
Index: src/uk/ac/gla/terrier/structures/postings/FieldPosting.java
===================================================================
--- src/uk/ac/gla/terrier/structures/postings/FieldPosting.java	(revision 2563)
+++ src/uk/ac/gla/terrier/structures/postings/FieldPosting.java	(working copy)
@@ -1,6 +1,12 @@
 package uk.ac.gla.terrier.structures.postings;
 
+
 public interface FieldPosting extends Posting {
 	/** Returns the frequencies of the term in each field of the document */
 	public int[] getFieldFrequencies();
+	
+	/** Returns the lengths of the each fields in the current document */
+	public int[] getFieldLengths();
+	
+	
 }
Index: src/uk/ac/gla/terrier/structures/postings/BasicIterablePosting.java
===================================================================
--- src/uk/ac/gla/terrier/structures/postings/BasicIterablePosting.java	(revision 2563)
+++ src/uk/ac/gla/terrier/structures/postings/BasicIterablePosting.java	(working copy)
@@ -28,10 +28,12 @@
 			return false;
 		id = bitFileReader.readGamma() + id;
 		tf = bitFileReader.readUnary();
+		//System.err.println("Read posting for doc "+ id + " tf "+ tf);
 		return true;
 	}
 
-	public int getDocumentLength() {
+	public int getDocumentLength()
+	{
 		try{
 			return doi.getDocumentLength(id);
 		} catch (Exception e) {
@@ -48,4 +50,13 @@
 	}
 
 	
+	
+	public WritablePosting asWritablePosting() {
+		BasicPosting bp = new BasicPosting();
+		bp.id = id;
+		bp.tf = tf;
+		return bp;
+	}
+
+	
 }
\ No newline at end of file
Index: src/uk/ac/gla/terrier/structures/postings/BlockIterablePosting.java
===================================================================
--- src/uk/ac/gla/terrier/structures/postings/BlockIterablePosting.java	(revision 2563)
+++ src/uk/ac/gla/terrier/structures/postings/BlockIterablePosting.java	(working copy)
@@ -56,5 +56,12 @@
 			WritableUtils.writeVInt(out, pos);
 	}
 
+	@Override
+	public WritablePosting asWritablePosting() {
+		int[] newPositions = new int[positions.length];
+		System.arraycopy(positions, 0, newPositions, 0, positions.length);
+		return new BlockPostingImpl(getId(), getFrequency(), newPositions);
+	}
+
 	
 }
\ No newline at end of file
Index: src/uk/ac/gla/terrier/structures/postings/BlockPostingImpl.java
===================================================================
--- src/uk/ac/gla/terrier/structures/postings/BlockPostingImpl.java	(revision 0)
+++ src/uk/ac/gla/terrier/structures/postings/BlockPostingImpl.java	(revision 0)
@@ -0,0 +1,40 @@
+package uk.ac.gla.terrier.structures.postings;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.hadoop.io.WritableUtils;
+
+public class BlockPostingImpl extends BasicPosting implements BlockPosting {
+	int[] positions;
+	public BlockPostingImpl() {
+	}
+
+	public BlockPostingImpl(int docid, int frequency, int[] _positions) {
+		super(docid, frequency);
+		positions = _positions;
+	}
+	
+	public int[] getPositions() {
+		return positions;
+	}
+
+	@Override
+	public void readFields(DataInput in) throws IOException {
+		super.readFields(in);
+		final int blockCount = WritableUtils.readVInt(in);
+		positions = new int[blockCount]; 
+		for(int i=0;i<blockCount;i++)
+			positions[i] = WritableUtils.readVInt(in);
+	}
+
+	@Override
+	public void write(DataOutput out) throws IOException {
+		super.write(out);
+		WritableUtils.writeVInt(out, positions.length);
+		for(int pos : positions)
+			WritableUtils.writeVInt(out, pos);
+	}
+
+}
Index: src/uk/ac/gla/terrier/structures/DirectIndexInputStream.java
===================================================================
--- src/uk/ac/gla/terrier/structures/DirectIndexInputStream.java	(revision 2563)
+++ src/uk/ac/gla/terrier/structures/DirectIndexInputStream.java	(working copy)
@@ -33,7 +33,6 @@
 
 import uk.ac.gla.terrier.structures.postings.BasicIterablePosting;
 import uk.ac.gla.terrier.structures.postings.IterablePosting;
-import uk.ac.gla.terrier.utility.FieldScore;
 /**
  * This class reads the direct index structure, sequentially,
  * as an input stream.
@@ -57,6 +56,13 @@
 		super(_index, structureName, _pointerList, _postingIteratorClass);
 	}
 	
+	
+	@SuppressWarnings("unchecked")
+	public DirectIndexInputStream(Index index, String structureName, Class<? extends IterablePosting> postingIterator) throws IOException
+	{
+		super(index, structureName, (Iterator<DocumentIndexEntry>)index.getIndexStructureInputStream("document"), postingIterator);
+	}
+	
 	@SuppressWarnings("unchecked")
 	public DirectIndexInputStream(Index index, String structureName) throws IOException
 	{
@@ -88,8 +94,8 @@
 
 	public int[][] getNextTerms(BitIndexPointer pointer) throws IOException
 	{
-		final int htmlTags = FieldScore.FIELDS_COUNT;
-		final boolean loadTagInformation = FieldScore.USE_FIELD_INFORMATION;
+		final boolean loadTagInformation = fieldCount > 0;
+		System.err.println("Fields = " + fieldCount);
 		ArrayList<int[]> temporaryTerms = new ArrayList<int[]>();
 		int[][] documentTerms = null;
 		//boolean hasMore = false;
@@ -100,27 +106,30 @@
 			//while ((endByteOffset > gammaInputStream.getByteOffset())
 			//		|| (endByteOffset == gammaInputStream.getByteOffset()
 			//			&& endBitOffset > gammaInputStream.getBitOffset())) {
-				int[] tmp = new int[3];
+				int[] tmp = new int[2+fieldCount];
 				tmp[0] = file.readGamma();
 				tmp[1] = file.readUnary();
-				tmp[2] = file.readBinary(htmlTags);
+				for(int fi = 0; fi < fieldCount; fi++)
+					tmp[fi+2] = file.readUnary()-1;
+				//tmp[2] = file.readBinary(htmlTags);
 				temporaryTerms.add(tmp);
 			}
-			documentTerms = new int[3][temporaryTerms.size()];
+			documentTerms = new int[2+fieldCount][temporaryTerms.size()];
 			int[] documentTerms0 = documentTerms[0];
 			int[] documentTerms1 = documentTerms[1];
-			int[] documentTerms2 = documentTerms[2];
+			//int[] documentTerms2 = documentTerms[2];
 			int[] tmpMatrix = (int[]) temporaryTerms.get(0);
 			documentTerms0[0] = tmpMatrix[0] - 1;
 			documentTerms1[0] = tmpMatrix[1];
-			documentTerms2[0] = tmpMatrix[2];
+			//documentTerms2[0] = tmpMatrix[2];
 			if (documentTerms[0].length > 1) {
 				final int documentTerms0Length = documentTerms[0].length;
 				for (int i = 1; i < documentTerms0Length; i++) {
 					tmpMatrix = (int[]) temporaryTerms.get(i);
 					documentTerms0[i] = tmpMatrix[0] + documentTerms[0][i - 1];
 					documentTerms1[i] = tmpMatrix[1];
-					documentTerms2[i] = tmpMatrix[2];
+					for(int fi = 0; fi < fieldCount; fi++)
+						documentTerms[fi+2][i] = tmpMatrix[fi+2];
 				}
 			}		
 		} else {
Index: src/uk/ac/gla/terrier/structures/Index.java
===================================================================
--- src/uk/ac/gla/terrier/structures/Index.java	(revision 2563)
+++ src/uk/ac/gla/terrier/structures/Index.java	(working copy)
@@ -424,6 +424,7 @@
 				int i=0;
 				for (String p: params)
 				{
+					//System.err.println("looking for parameter value called "+ p + " with type '" + param_types[i]+ "'");
 					if (p.equals("path"))
 						objs[i] = path;
 					else if (p.equals("prefix"))
@@ -435,6 +436,11 @@
 						final String tmp = structureName;
 						objs[i] = tmp.replaceAll("-inputstream$", "");
 					}
+					else if (param_types[i].equals(java.lang.Class.class))
+					{
+						//System.err.println("loading class called "+p);
+						objs[i] = Class.forName(p);
+					}
 					else if (p.endsWith("-inputstream"))//no caching for input streams
 						 objs[i] = loadIndexStructure(p);
 					else if (p.matches("^\\$\\{.+\\}$"))
Index: src/uk/ac/gla/terrier/structures/BlockDirectIndexInputStream.java
===================================================================
--- src/uk/ac/gla/terrier/structures/BlockDirectIndexInputStream.java	(revision 2563)
+++ src/uk/ac/gla/terrier/structures/BlockDirectIndexInputStream.java	(working copy)
@@ -32,6 +32,7 @@
 import org.apache.log4j.Logger;
 
 import uk.ac.gla.terrier.structures.postings.BlockIterablePosting;
+import uk.ac.gla.terrier.structures.postings.IterablePosting;
 import uk.ac.gla.terrier.utility.FieldScore;
 /**
  * This class reads the block field direct index structure
@@ -52,6 +53,11 @@
 		super(index, structureName, (Iterator<DocumentIndexEntry>)index.getIndexStructureInputStream("document"), BlockIterablePosting.class);
 	}
 	
+	@SuppressWarnings("unchecked")
+	public BlockDirectIndexInputStream(Index index, String structureName, Class<? extends IterablePosting> postingClass) throws IOException
+	{
+		super(index, structureName, (Iterator<DocumentIndexEntry>)index.getIndexStructureInputStream("document"), postingClass);
+	}
 	
 	/**
 	 * Prints out the block field direct index file.
Index: src/uk/ac/gla/terrier/structures/DirectInvertedOutputStream.java
===================================================================
--- src/uk/ac/gla/terrier/structures/DirectInvertedOutputStream.java	(revision 2563)
+++ src/uk/ac/gla/terrier/structures/DirectInvertedOutputStream.java	(working copy)
@@ -29,6 +29,8 @@
 import java.io.IOException;
 import java.util.Iterator;
 
+import org.apache.log4j.Logger;
+
 import uk.ac.gla.terrier.compression.BitOut;
 import uk.ac.gla.terrier.compression.BitOutputStream;
 import uk.ac.gla.terrier.structures.postings.IterablePosting;
@@ -42,24 +44,24 @@
 public class DirectInvertedOutputStream implements Closeable {
 	/** what to write to */
 	protected BitOut output;
-	/** the number of field bits to write */
-	protected int binaryBits; 
+	/** The logger used */
+	protected static final Logger logger = Logger.getLogger(DirectInvertedOutputStream.class);
+ 
 	/** Creates a new output stream, writing a BitOutputStream to the specified file. The number of binary bits
 	  * for fields must also be specified.
 	  * @param filename Location of the file to write to
-	  * @param binaryBits the number of fields in this index 
 	  */
-	public DirectInvertedOutputStream(String filename, int binaryBits) throws IOException
+	public DirectInvertedOutputStream(String filename) throws IOException
 	{
 		this.output = new BitOutputStream(filename);
-		this.binaryBits = binaryBits;
+		//this.binaryBits = binaryBits;
 	}
 	/** Creates a new output stream, writing to the specified BitOut implementation.  The number of binary bits
 	  * for fields must also be specified.
 	  * @param out BitOut implementation to write the file to 
 	  * @param binaryBits the number of fields in this index 
 	  */
-	public DirectInvertedOutputStream(BitOut out, int binaryBits)
+	public DirectInvertedOutputStream(BitOut out)
 	{
 		this.output = out;
 	}
@@ -70,8 +72,6 @@
 	  */
 	public BitIndexPointer writePostings(int[][] postings, int firstId) throws IOException
 	{
-		if (binaryBits>0) 
-			return writeFieldPostings(postings, 0, postings[0].length, firstId);
 		 return writeNoFieldPostings(postings, 0, postings[0].length, firstId);
 	}
 	
@@ -135,10 +135,7 @@
 	  */
 	public BitIndexPointer writePostings(int[][] postings, int startOffset, int Length, int firstId) throws IOException
 	{
-		if (binaryBits>0) 
-			return writeFieldPostings(postings, startOffset, Length, firstId);
-		else 
-			return writeNoFieldPostings(postings, startOffset, Length, firstId);
+		return writeNoFieldPostings(postings, startOffset, Length, firstId);
 	}
 	
 	/**
@@ -150,7 +147,7 @@
 	 * @param firstId the first identifier to write. This can be 
 	 *        an id plus one, or the gap of the current id and the previous one.
 	 */
-	protected BitIndexPointer writeFieldPostings(int[][] postings, int offset, final int Length, int firstId) 
+	/*protected BitIndexPointer writeFieldPostings(int[][] postings, int offset, final int Length, int firstId) 
 			throws IOException {
 		
 		BitIndexPointer rtr = new SimpleBitIndexPointer(output.getByteOffset(), output.getBitOffset(), Length);
@@ -173,7 +170,7 @@
 			output.writeBinary(binaryBits, postings2[offset]);
 		}
 		return rtr;
-	}
+	}*/
 	
 	/**
 	 * Writes the given postings to the bit file. This method assumes that
@@ -215,7 +212,7 @@
 		try{ 
 			output.close();
 		} catch (IOException ioe) {
-			
+			logger.error("Problem closing DirectInvOutputStream", ioe);
 		}
 	}
 	
Index: src/uk/ac/gla/terrier/structures/BlockFieldDirectInvertedOutputStream.java
===================================================================
--- src/uk/ac/gla/terrier/structures/BlockFieldDirectInvertedOutputStream.java	(revision 0)
+++ src/uk/ac/gla/terrier/structures/BlockFieldDirectInvertedOutputStream.java	(revision 0)
@@ -0,0 +1,38 @@
+package uk.ac.gla.terrier.structures;
+
+import java.io.IOException;
+
+import uk.ac.gla.terrier.compression.BitOut;
+import uk.ac.gla.terrier.structures.postings.BlockPosting;
+import uk.ac.gla.terrier.structures.postings.Posting;
+
+public class BlockFieldDirectInvertedOutputStream extends
+		FieldDirectInvertedOutputStream {
+
+	public BlockFieldDirectInvertedOutputStream(BitOut out) {
+		super(out);
+	}
+
+	public BlockFieldDirectInvertedOutputStream(String filename)
+			throws IOException {
+		super(filename);
+	}
+	
+	@Override
+	protected void writePostingNotDocid(Posting _p) throws IOException
+	{
+		super.writePostingNotDocid(_p);
+		final BlockPosting p = (BlockPosting)_p;
+		final int positions[] = p.getPositions();
+		final int l = positions.length;
+		if (l == 0)
+			return;
+		output.writeUnary(l+1);
+		output.writeGamma(positions[0]+1);
+		for(int i=1;i<l;i++)
+		{
+			output.writeGamma(positions[i] - positions[i-1]);
+		}
+	}
+
+}
Index: src/uk/ac/gla/terrier/utility/ApplicationSetup.java
===================================================================
--- src/uk/ac/gla/terrier/utility/ApplicationSetup.java	(revision 2563)
+++ src/uk/ac/gla/terrier/utility/ApplicationSetup.java	(working copy)
@@ -441,6 +441,10 @@
 	private static Context envCtx = null;
 	
 	static {
+		bootstrapInitialisation();
+	}
+	public static void bootstrapInitialisation()
+	{
 		useContext = Boolean.parseBoolean(System.getProperty("terrier.usecontext", "false"));
 
 		String propertiesFile = null;
@@ -511,7 +515,7 @@
 			System.err.println("is specified in the file terrier.properties,");
 			System.err.println("or as a system property in the command line.");
 		}
-		
+		loadCommonProperties();
 	}
 	
 	public static void loadCommonProperties()
@@ -690,6 +694,7 @@
 	/**
 	 * Sets up the names of the inverted file, the direct file, 
 	 * the document index file and the lexicon file.
+	 * @deprecated
 	 */
 	public static void setupFilenames() {
 		//String filenameTemplate = TERRIER_INDEX_PATH + FILE_SEPARATOR + TERRIER_INDEX_PREFIX;
@@ -756,4 +761,10 @@
 		}
 		return DefaultPath+filename;
 	}
+	
+	public static void clearAllProperties()
+	{
+		appProperties.clear();
+		UsedAppProperties.clear();
+	}
 }
Index: src/uk/ac/gla/terrier/tests/BasicShakespeareEndToEndTest.java
===================================================================
--- src/uk/ac/gla/terrier/tests/BasicShakespeareEndToEndTest.java	(revision 0)
+++ src/uk/ac/gla/terrier/tests/BasicShakespeareEndToEndTest.java	(revision 0)
@@ -0,0 +1,32 @@
+package uk.ac.gla.terrier.tests;
+import org.junit.Test;
+
+public class BasicShakespeareEndToEndTest extends ShakespeareEndToEndTest {
+
+	String testQrels = "share/tests/test.shakespeare-merchant.all.qrels";
+		
+	public BasicShakespeareEndToEndTest()
+	{}
+	
+	@Test public void testBasicClassical() throws Exception {
+		doTrecTerrierRunAndEvaluate(
+				new String[]{"-i"}, 
+				new String[]{"share/tests/test.shakespeare-merchant.basic.topics"},
+				testQrels, 7);
+	}
+	
+	@Test public void testBasicClassicalUTFCollection() throws Exception {
+		doTrecTerrierRunAndEvaluate(
+				new String[]{"-i", "-Dtrec.collection.class=TRECUTFCollection"}, 
+				new String[]{"share/tests/test.shakespeare-merchant.basic.topics"},
+				testQrels, 7);
+	}
+	
+	@Test public void testBasicClassicalFields() throws Exception {
+		return;
+		/*doTrecTerrierRunAndEvaluate(
+				new String[]{"-i", "-DFieldTags.process=TITLE,SPEAKER"}, 
+				new String[]{"share/tests/test.shakespeare-merchant.basic.topics", "share/tests/test.shakespeare-merchant.field.topics"},
+				testQrels, 10);*/
+	}
+}
Index: src/uk/ac/gla/terrier/tests/BlockShakespeareEndToEndTest.java
===================================================================
--- src/uk/ac/gla/terrier/tests/BlockShakespeareEndToEndTest.java	(revision 0)
+++ src/uk/ac/gla/terrier/tests/BlockShakespeareEndToEndTest.java	(revision 0)
@@ -0,0 +1,8 @@
+package uk.ac.gla.terrier.tests;
+
+public class BlockShakespeareEndToEndTest extends BasicShakespeareEndToEndTest {
+
+	public BlockShakespeareEndToEndTest() {
+		indexingOptions.add("-Dblock.indexing=true");
+	}
+}
Index: src/uk/ac/gla/terrier/tests/HadoopShakespeareEndToEndTest.java
===================================================================
--- src/uk/ac/gla/terrier/tests/HadoopShakespeareEndToEndTest.java	(revision 0)
+++ src/uk/ac/gla/terrier/tests/HadoopShakespeareEndToEndTest.java	(revision 0)
@@ -0,0 +1,20 @@
+package uk.ac.gla.terrier.tests;
+
+public class HadoopShakespeareEndToEndTest 
+{
+	static public class BasicHadoopShakespeareEndToEndTest extends BasicShakespeareEndToEndTest
+	{
+		public BasicHadoopShakespeareEndToEndTest()
+		{
+			indexingOptions.add("-H");
+		}
+	}
+	
+	static public class BlockHadoopShakespeareEndToEndTest extends BlockShakespeareEndToEndTest
+	{
+		public BlockHadoopShakespeareEndToEndTest()
+		{
+			indexingOptions.add("-H");
+		}
+	}
+}
Index: src/uk/ac/gla/terrier/tests/ShakespeareEndToEndTest.java
===================================================================
--- src/uk/ac/gla/terrier/tests/ShakespeareEndToEndTest.java	(revision 0)
+++ src/uk/ac/gla/terrier/tests/ShakespeareEndToEndTest.java	(revision 0)
@@ -0,0 +1,146 @@
+package uk.ac.gla.terrier.tests;
+
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.IOException;
+import java.io.Writer;
+import java.util.ArrayList;
+import java.util.List;
+
+import junit.framework.TestCase;
+import uk.ac.gla.terrier.applications.TRECSetup;
+import uk.ac.gla.terrier.applications.TrecTerrier;
+import uk.ac.gla.terrier.structures.Index;
+import uk.ac.gla.terrier.structures.IndexUtil;
+import uk.ac.gla.terrier.utility.ApplicationSetup;
+import uk.ac.gla.terrier.utility.Files;
+
+
+public abstract class ShakespeareEndToEndTest extends TestCase {
+	protected List<String> indexingOptions = new ArrayList<String>();
+	protected String[] getIndexingOptions(String [] specified)
+	{
+		List<String> theseIndexingOptions = new ArrayList<String>();
+		for(String speccedArg: specified)
+			theseIndexingOptions.add(speccedArg);
+		theseIndexingOptions.addAll(indexingOptions);
+		return theseIndexingOptions.toArray(new String[0]);
+	}
+	
+	protected void doIndexing(String... trec_terrier_args) throws Exception
+	{
+		TrecTerrier.main(getIndexingOptions(trec_terrier_args));
+		assertTrue(Index.existsIndex(ApplicationSetup.TERRIER_INDEX_PATH, ApplicationSetup.TERRIER_INDEX_PREFIX));
+		Index i = Index.createIndex();
+		assertNotNull(Index.getLastIndexLoadError(), i);
+		assertTrue(i.hasIndexStructure("inverted"));
+		assertTrue(i.hasIndexStructure("lexicon"));
+		assertTrue(i.hasIndexStructure("document"));
+		assertTrue(i.hasIndexStructure("meta"));
+		i.close();
+	}
+	
+	protected void doRetrieval(String[] topicSet) throws Exception
+	{
+		Writer w = Files.writeFileWriter(ApplicationSetup.TREC_TOPICS_LIST);
+		System.err.println("Writing topics files to" + ApplicationSetup.TREC_TOPICS_LIST);
+		for(String topicFile : topicSet)
+		{	
+			//System.err.println("\t"+topicFile);
+			w.write(topicFile + "\n");
+		}
+		w.close();
+		TrecTerrier.main(new String[]{"-r"});
+	}
+	
+	protected void doEvaluation(int expectedQueryCount, String qrels) throws Exception
+	{
+		Writer w = Files.writeFileWriter(ApplicationSetup.TREC_QRELS);
+		System.err.println("Writing qrel files files to " + ApplicationSetup.TREC_QRELS);
+		w.write(qrels + "\n");
+		w.close();
+		TrecTerrier.main(new String[]{"-e"});
+		float MAP = -1.0f;
+		int queryCount = 0;
+		for (File f : new File(ApplicationSetup.TREC_RESULTS).listFiles())
+		{
+			if (f.getName().endsWith(".eval"))
+			{
+				BufferedReader br = Files.openFileReader(f);
+				String line = null;
+				while((line = br.readLine()) != null )
+				{
+					if (line.startsWith("Average Precision:"))
+					{
+						MAP = Float.parseFloat(line.split(":")[1].trim());	
+					} 
+					else if (line.startsWith("Number of queries  ="))
+					{
+						queryCount = Integer.parseInt(line.split("=")[1].trim());
+					}
+				}
+				br.close();
+				break;
+			}
+		}
+		assertEquals("Query count was "+ queryCount + " instead of "+ expectedQueryCount, expectedQueryCount, queryCount);
+		assertEquals("MAP was "+MAP + " instead of 1.0", MAP, 1.0f);
+	}
+	
+	protected void doTrecTerrierRunAndEvaluate(
+			String[] indexingArgs, String[] topics, String qrels, int queryCount)
+		throws Exception
+	{
+		doIndexing(indexingArgs);
+		doRetrieval(topics);
+		doEvaluation(queryCount, qrels);
+		try{
+			IndexUtil.deleteIndex(ApplicationSetup.TERRIER_INDEX_PATH, ApplicationSetup.TERRIER_INDEX_PREFIX);
+		} catch (IOException ioe) {
+			
+		}
+	}
+
+	@Override
+	protected void setUp() throws Exception {
+		super.setUp();
+		
+		System.setProperty("terrier.home", System.getProperty("user.dir"));
+		System.err.println("terrier.home assumed to be "+ System.getProperty("user.dir"));
+		String terrier_etc = System.getProperty("user.dir") + "/var/test";
+		for(File f : new File(terrier_etc).listFiles())
+			f.delete();
+		new File(terrier_etc).mkdirs();
+		System.setProperty("terrier.etc", terrier_etc);
+		System.setProperty("terrier.setup", terrier_etc + "/terrier.properties");
+		TRECSetup.main(new String[]{System.getProperty("user.dir")});
+		
+		BufferedReader br = Files.openFileReader(terrier_etc + "/terrier.properties");
+		Writer w = Files.writeFileWriter(terrier_etc+ "/terrier.properties.test");
+		String line = null;
+		while((line = br.readLine()) != null)
+		{
+			w.write(line+"\n");
+		}
+		w.write("terrier.index.path="+terrier_etc+ "\n");
+		w.write("trec.results="+terrier_etc+ "\n");
+		w.write("ignore.low.idf.terms=false\n");
+		w.write("trec.topics.parser=SingleLineTRECQuery\n");
+		w.close();
+		Files.delete(terrier_etc + "/terrier.properties");
+		Files.rename(terrier_etc+ "/terrier.properties.test", terrier_etc + "/terrier.properties");		
+		
+		uk.ac.gla.terrier.utility.ApplicationSetup.bootstrapInitialisation();
+		w = Files.writeFileWriter("var/test/collection.spec");
+		w.write("share/shakespeare-merchant.trec");
+		w.close();
+	}
+
+	@Override
+	protected void tearDown() throws Exception {
+		// TODO Auto-generated method stub
+		super.tearDown();
+		ApplicationSetup.clearAllProperties();
+	}
+}
Index: src/uk/ac/gla/terrier/tests/ShakespeareEndToEndTestSuite.java
===================================================================
--- src/uk/ac/gla/terrier/tests/ShakespeareEndToEndTestSuite.java	(revision 0)
+++ src/uk/ac/gla/terrier/tests/ShakespeareEndToEndTestSuite.java	(revision 0)
@@ -0,0 +1,19 @@
+package uk.ac.gla.terrier.tests;
+
+import junit.framework.Test;
+import junit.framework.TestSuite;
+
+public class ShakespeareEndToEndTestSuite {
+
+	public static Test suite()
+	{
+		 TestSuite suite = new TestSuite();
+		 suite.addTestSuite(BasicShakespeareEndToEndTest.class);
+		 suite.addTestSuite(BlockShakespeareEndToEndTest.class);
+		 suite.addTestSuite(SinglePassShakespeareEndToEndTest.BasicSinglePassShakespeareEndToEndTest.class);
+		 suite.addTestSuite(SinglePassShakespeareEndToEndTest.BlockSinglePassShakespeareEndToEndTest.class);
+		 suite.addTestSuite(HadoopShakespeareEndToEndTest.BasicHadoopShakespeareEndToEndTest.class);
+		 suite.addTestSuite(HadoopShakespeareEndToEndTest.BlockHadoopShakespeareEndToEndTest.class);
+		 return suite;
+	}
+}
Index: src/uk/ac/gla/terrier/tests/SinglePassShakespeareEndToEndTest.java
===================================================================
--- src/uk/ac/gla/terrier/tests/SinglePassShakespeareEndToEndTest.java	(revision 0)
+++ src/uk/ac/gla/terrier/tests/SinglePassShakespeareEndToEndTest.java	(revision 0)
@@ -0,0 +1,20 @@
+package uk.ac.gla.terrier.tests;
+
+public class SinglePassShakespeareEndToEndTest 
+{
+	static public class BasicSinglePassShakespeareEndToEndTest extends BasicShakespeareEndToEndTest
+	{
+		public BasicSinglePassShakespeareEndToEndTest()
+		{
+			indexingOptions.add("-j");
+		}
+	}
+	
+	static public class BlockSinglePassShakespeareEndToEndTest extends BlockShakespeareEndToEndTest
+	{
+		public BlockSinglePassShakespeareEndToEndTest()
+		{
+			indexingOptions.add("-j");
+		}
+	}
+}
Index: src/uk/ac/gla/terrier/matching/Matching.java
===================================================================
--- src/uk/ac/gla/terrier/matching/Matching.java	(revision 2570)
+++ src/uk/ac/gla/terrier/matching/Matching.java	(working copy)
@@ -35,6 +35,7 @@
 import uk.ac.gla.terrier.matching.tsms.TermScoreModifier;
 import uk.ac.gla.terrier.structures.CollectionStatistics;
 import uk.ac.gla.terrier.structures.Index;
+import uk.ac.gla.terrier.structures.IndexUtil;
 import uk.ac.gla.terrier.structures.Lexicon;
 import uk.ac.gla.terrier.structures.LexiconEntry;
 import uk.ac.gla.terrier.structures.PostingIndex;
@@ -322,6 +323,7 @@
 				wmodel.setRequest(queryTerms.getRequest());
 				wmodel.setKeyFrequency(queryTerms.getTermWeight(queryTermStrings[i]));
 				wmodel.setEntryStatistics(lEntry);
+				IndexUtil.configure(index, wmodel);
 				//this requests any pre-calculations to be made
 				wmodel.prepare();
 			}
Index: src/uk/ac/gla/terrier/matching/tsms/TermInFieldModifier.java
===================================================================
--- src/uk/ac/gla/terrier/matching/tsms/TermInFieldModifier.java	(revision 2563)
+++ src/uk/ac/gla/terrier/matching/tsms/TermInFieldModifier.java	(working copy)
@@ -25,6 +25,8 @@
  */
 package uk.ac.gla.terrier.matching.tsms;
 import uk.ac.gla.terrier.matching.models.WeightingModel;
+import uk.ac.gla.terrier.structures.Index;
+import uk.ac.gla.terrier.structures.IndexConfigurable;
 import uk.ac.gla.terrier.structures.postings.FieldPosting;
 import uk.ac.gla.terrier.structures.postings.Posting;
 import uk.ac.gla.terrier.utility.FieldScore;
@@ -34,8 +36,11 @@
  * @author Vassilis Plachouras
  * @version $Revision: 1.11 $
  */
-public class TermInFieldModifier extends WeightingModel implements TermScoreModifier  {
-	
+public class TermInFieldModifier 
+	extends WeightingModel 
+	implements TermScoreModifier, IndexConfigurable
+{
+	protected Index index = null;
 	/**
 	 * The fields that a query term should appear in.
 	 */
@@ -119,10 +124,20 @@
 	}
 	
 	
-	int fieldIndex = 0;
+	int fieldIndex = -1;
 	public void prepare()
 	{
-		//TODO populate fieldIndex if not already done
+		String[] indexFieldNames = index.getIndexProperty("index.inverted.fields.names", "").split("\\s*,\\s*");
+		int i=0;
+		for(String f : indexFieldNames)
+		{
+			if (f.equals(this.field))
+			{
+				fieldIndex = i;
+				break;
+			}
+			i++;
+		}
 	}
 	
 	//implementation assumes scores are additive
@@ -171,4 +186,8 @@
 		// TODO Auto-generated method stub
 		return 0;
 	}
+
+	public void setIndex(Index i) {
+		this.index = i;
+	}
 }
Index: src/uk/ac/gla/terrier/applications/TRECQuerying.java
===================================================================
--- src/uk/ac/gla/terrier/applications/TRECQuerying.java	(revision 2563)
+++ src/uk/ac/gla/terrier/applications/TRECQuerying.java	(working copy)
@@ -415,7 +415,7 @@
 		srq.addMatchingModel(mModel, wModel);
 		preQueryingSearchRequestModification(queryId, srq);
 		if (logger.isInfoEnabled()) 
-			logger.info("Processing query: " + queryId);
+			logger.info("Processing query: " + queryId + ": '"+query+"'");
 		matchingCount++;
 		queryingManager.runPreProcessing(srq);
 		queryingManager.runMatching(srq);
Index: src/uk/ac/gla/terrier/applications/TrecTerrier.java
===================================================================
--- src/uk/ac/gla/terrier/applications/TrecTerrier.java	(revision 2563)
+++ src/uk/ac/gla/terrier/applications/TrecTerrier.java	(working copy)
@@ -1,3 +1,4 @@
+package uk.ac.gla.terrier.applications;
 /*
  * Terrier - Terabyte Retriever 
  * Webpage: http://ir.dcs.gla.ac.uk/terrier 
@@ -27,12 +28,6 @@
 
 import org.apache.log4j.Logger;
 
-import uk.ac.gla.terrier.applications.HadoopIndexing;
-import uk.ac.gla.terrier.applications.TRECIndexing;
-import uk.ac.gla.terrier.applications.TRECLMIndexing;
-import uk.ac.gla.terrier.applications.TRECLMQuerying;
-import uk.ac.gla.terrier.applications.TRECQuerying;
-import uk.ac.gla.terrier.applications.TRECQueryingExpansion;
 import uk.ac.gla.terrier.evaluation.AdhocEvaluation;
 import uk.ac.gla.terrier.evaluation.Evaluation;
 import uk.ac.gla.terrier.evaluation.NamedPageEvaluation;
@@ -268,8 +263,15 @@
 			return ERROR_NO_ARGUMENTS;
 		
 		int pos = 0;
+		boolean applicationSetupUpdated = false;
 		while (pos < args.length) {
-			if (args[pos].equals("-h") || args[pos].equals("--help"))
+			if (args[pos].startsWith("-D"))
+			{
+				String[] propertyKV = args[pos].replaceFirst("^-D", "").split("=");
+				ApplicationSetup.setProperty(propertyKV[0], propertyKV[1]);
+				applicationSetupUpdated = true;
+			}
+			else if (args[pos].equals("-h") || args[pos].equals("--help"))
 				printHelp = true;
 			else if (args[pos].equals("-i") || args[pos].equals("--index"))
 				indexing = true;
@@ -329,6 +331,10 @@
 			pos++;
 		}
 		
+		if (applicationSetupUpdated)
+			ApplicationSetup.loadCommonProperties();
+		
+		
 		if (isParameterValueSpecified && !retrieving) 
 			return ERROR_GIVEN_C_NOT_RETRIEVING;
 		
Index: src/uk/ac/gla/terrier/applications/HadoopIndexing.java
===================================================================
--- src/uk/ac/gla/terrier/applications/HadoopIndexing.java	(revision 2566)
+++ src/uk/ac/gla/terrier/applications/HadoopIndexing.java	(working copy)
@@ -40,6 +40,7 @@
 import org.apache.hadoop.mapred.JobID;
 import org.apache.hadoop.mapred.RunningJob;
 import org.apache.hadoop.mapred.TaskID;
+import org.apache.hadoop.mapred.lib.HashPartitioner;
 import org.apache.hadoop.mapred.lib.NullOutputFormat;
 import org.apache.log4j.Logger;
 
@@ -142,7 +143,12 @@
 		if (numberOfReducers> 1)
 		{
 			conf.setPartitionerClass(ByMapPartitioner.class);
-		}		
+		}
+		else
+		{
+			//for JUnit tests, we seem to need to restore the original partitioner class
+			conf.setPartitionerClass(HashPartitioner.class);
+		}
 		
 		JobID jobId = null;
 		try{
Index: src/uk/ac/gla/terrier/applications/TRECSetup.java
===================================================================
--- src/uk/ac/gla/terrier/applications/TRECSetup.java	(revision 2563)
+++ src/uk/ac/gla/terrier/applications/TRECSetup.java	(working copy)
@@ -191,7 +191,7 @@
 		} catch(IOException ioe) {
 			System.err.println("Exception while creating the default configuration files for Terrier: "+ioe);
 			System.err.println("Exiting ...");
-			System.exit(1);
+			ioe.printStackTrace();	
 		}
 			
 	}
Index: src/uk/ac/gla/terrier/sorting/HeapSortInt.java
===================================================================
--- src/uk/ac/gla/terrier/sorting/HeapSortInt.java	(revision 2563)
+++ src/uk/ac/gla/terrier/sorting/HeapSortInt.java	(working copy)
@@ -67,6 +67,12 @@
         return heapSize;
     }
 
+    private static int buildMaxHeap(int[][] A) {
+        final int heapSize = A[0].length;
+        for (int i = heapSize/2; i > 0; i--)
+            maxHeapify(A, i, heapSize);
+        return heapSize;
+    }
 
    	/**
 	 * Builds a maximum heap.
@@ -175,6 +181,28 @@
 			maxHeapify(A, B, 1, heapSize);
 		}
 	}
+	
+	public static void ascendingHeapSort(int[][] A) {
+		int heapSize = buildMaxHeap(A);
+
+		//temporary variables for swaps
+		
+		int tmpInt;
+		//int tmpShort;
+
+		for (int i = A[0].length; i > 0; i--) {
+			//swap elements in i-1 with 0
+			for(int j=0;j<A.length;j++)
+			{
+				tmpInt = A[j][i-1];
+				A[j][i-1] = A[j][0];
+				A[j][0] = tmpInt;
+			}
+
+			heapSize--;
+			maxHeapify(A, 1, heapSize);
+		}
+	}
 
 	/**
 	 * Sorts the given arrays in descending order, using heap-sort.
@@ -426,7 +454,36 @@
 	}
 
 
+	private static void maxHeapify(final int[][] A, final int i, final int heapSize) {
+		final int l = 2 * i;
+		final int r = 2 * i + 1;
+
+		int largest = 
+			(l <= heapSize && A[0][l - 1] > A[0][i - 1])
+				? l
+				: i;
+		//if (l <= heapSize && A[l - 1] > A[i - 1])
+		//	largest = l;
+		//else
+		//	largest = i;
+		if (r <= heapSize && A[0][r - 1] > A[0][largest - 1])
+			largest = r;
+
+		//temporary variables for swaps
+		int tmpInt;
+		//int tmpShort;
+
 
+		if (largest != i) {
+			for(int j=0;j<A.length;j++)
+			{
+				tmpInt = A[j][largest - 1];
+				A[j][largest - 1] = A[j][i - 1];
+				A[j][i - 1] = tmpInt;
+			}
+			maxHeapify(A, largest, heapSize);
+		}
+	}
 
 
 
Index: .antlr-eclipse
===================================================================
--- .antlr-eclipse	(revision 0)
+++ .antlr-eclipse	(revision 0)
@@ -0,0 +1,6 @@
+<?xml version='1.0' ?> 
+<settings>
+  <resource name='**ANTLR-ECLIPSE-PLUGIN**'>
+    <property name='pluginVersion' value='2.7.6' />
+  </resource>
+</settings>
Index: .settings/org.eclipse.jdt.core.prefs
===================================================================
--- .settings/org.eclipse.jdt.core.prefs	(revision 0)
+++ .settings/org.eclipse.jdt.core.prefs	(revision 0)
@@ -0,0 +1,60 @@
+#Thu Jun 18 14:35:23 BST 2009
+eclipse.preferences.version=1
+org.eclipse.jdt.core.compiler.problem.annotationSuperInterface=warning
+org.eclipse.jdt.core.compiler.problem.autoboxing=ignore
+org.eclipse.jdt.core.compiler.problem.deprecation=warning
+org.eclipse.jdt.core.compiler.problem.deprecationInDeprecatedCode=disabled
+org.eclipse.jdt.core.compiler.problem.deprecationWhenOverridingDeprecatedMethod=disabled
+org.eclipse.jdt.core.compiler.problem.discouragedReference=warning
+org.eclipse.jdt.core.compiler.problem.emptyStatement=ignore
+org.eclipse.jdt.core.compiler.problem.fallthroughCase=ignore
+org.eclipse.jdt.core.compiler.problem.fatalOptionalError=enabled
+org.eclipse.jdt.core.compiler.problem.fieldHiding=ignore
+org.eclipse.jdt.core.compiler.problem.finalParameterBound=warning
+org.eclipse.jdt.core.compiler.problem.finallyBlockNotCompletingNormally=warning
+org.eclipse.jdt.core.compiler.problem.forbiddenReference=error
+org.eclipse.jdt.core.compiler.problem.hiddenCatchBlock=warning
+org.eclipse.jdt.core.compiler.problem.incompatibleNonInheritedInterfaceMethod=warning
+org.eclipse.jdt.core.compiler.problem.incompleteEnumSwitch=ignore
+org.eclipse.jdt.core.compiler.problem.indirectStaticAccess=ignore
+org.eclipse.jdt.core.compiler.problem.localVariableHiding=ignore
+org.eclipse.jdt.core.compiler.problem.methodWithConstructorName=warning
+org.eclipse.jdt.core.compiler.problem.missingDeprecatedAnnotation=ignore
+org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotation=ignore
+org.eclipse.jdt.core.compiler.problem.missingSerialVersion=ignore
+org.eclipse.jdt.core.compiler.problem.noEffectAssignment=warning
+org.eclipse.jdt.core.compiler.problem.noImplicitStringConversion=warning
+org.eclipse.jdt.core.compiler.problem.nonExternalizedStringLiteral=ignore
+org.eclipse.jdt.core.compiler.problem.nullReference=warning
+org.eclipse.jdt.core.compiler.problem.overridingPackageDefaultMethod=warning
+org.eclipse.jdt.core.compiler.problem.parameterAssignment=ignore
+org.eclipse.jdt.core.compiler.problem.possibleAccidentalBooleanAssignment=ignore
+org.eclipse.jdt.core.compiler.problem.potentialNullReference=ignore
+org.eclipse.jdt.core.compiler.problem.rawTypeReference=warning
+org.eclipse.jdt.core.compiler.problem.redundantNullCheck=ignore
+org.eclipse.jdt.core.compiler.problem.redundantSuperinterface=ignore
+org.eclipse.jdt.core.compiler.problem.specialParameterHidingField=disabled
+org.eclipse.jdt.core.compiler.problem.staticAccessReceiver=warning
+org.eclipse.jdt.core.compiler.problem.suppressWarnings=enabled
+org.eclipse.jdt.core.compiler.problem.syntheticAccessEmulation=ignore
+org.eclipse.jdt.core.compiler.problem.typeParameterHiding=warning
+org.eclipse.jdt.core.compiler.problem.uncheckedTypeOperation=warning
+org.eclipse.jdt.core.compiler.problem.undocumentedEmptyBlock=ignore
+org.eclipse.jdt.core.compiler.problem.unhandledWarningToken=warning
+org.eclipse.jdt.core.compiler.problem.unnecessaryElse=ignore
+org.eclipse.jdt.core.compiler.problem.unnecessaryTypeCheck=ignore
+org.eclipse.jdt.core.compiler.problem.unqualifiedFieldAccess=ignore
+org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownException=ignore
+org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionExemptExceptionAndThrowable=enabled
+org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionIncludeDocCommentReference=enabled
+org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionWhenOverriding=disabled
+org.eclipse.jdt.core.compiler.problem.unusedImport=warning
+org.eclipse.jdt.core.compiler.problem.unusedLabel=warning
+org.eclipse.jdt.core.compiler.problem.unusedLocal=warning
+org.eclipse.jdt.core.compiler.problem.unusedParameter=ignore
+org.eclipse.jdt.core.compiler.problem.unusedParameterIncludeDocCommentReference=enabled
+org.eclipse.jdt.core.compiler.problem.unusedParameterWhenImplementingAbstract=disabled
+org.eclipse.jdt.core.compiler.problem.unusedParameterWhenOverridingConcrete=disabled
+org.eclipse.jdt.core.compiler.problem.unusedPrivateMember=warning
+org.eclipse.jdt.core.compiler.problem.unusedWarningToken=warning
+org.eclipse.jdt.core.compiler.problem.varargsArgumentNeedCast=warning