Index: uk/ac/gla/terrier/indexing/BasicSinglePassIndexer.java =================================================================== --- uk/ac/gla/terrier/indexing/BasicSinglePassIndexer.java (revision 2534) +++ uk/ac/gla/terrier/indexing/BasicSinglePassIndexer.java (working copy) @@ -335,8 +335,8 @@ currentIndex.addIndexStructure( "inverted", invertedIndexClass, - "uk.ac.gla.terrier.structures.Index,java.lang.String", - "index,structureName"); + "uk.ac.gla.terrier.structures.Index,java.lang.String,uk.ac.gla.terrier.structures.DocumentIndex", + "index,structureName,document"); currentIndex.addIndexStructureInputStream( "inverted", invertedIndexInputStreamClass, Index: uk/ac/gla/terrier/structures/postings/BlockPosting.java =================================================================== --- uk/ac/gla/terrier/structures/postings/BlockPosting.java (revision 0) +++ uk/ac/gla/terrier/structures/postings/BlockPosting.java (revision 0) @@ -0,0 +1,6 @@ +package uk.ac.gla.terrier.structures.postings; + +public interface BlockPosting extends Posting { + /** Return positions of occurrences */ + public int[] getPositions(); +} Index: uk/ac/gla/terrier/structures/postings/FieldPosting.java =================================================================== --- uk/ac/gla/terrier/structures/postings/FieldPosting.java (revision 0) +++ uk/ac/gla/terrier/structures/postings/FieldPosting.java (revision 0) @@ -0,0 +1,6 @@ +package uk.ac.gla.terrier.structures.postings; + +public interface FieldPosting extends Posting { + /** Returns the frequencies of the term in each field of the document */ + public int[] getFieldFrequencies(); +} Index: uk/ac/gla/terrier/structures/postings/IterablePosting.java =================================================================== --- uk/ac/gla/terrier/structures/postings/IterablePosting.java (revision 0) +++ uk/ac/gla/terrier/structures/postings/IterablePosting.java (revision 0) @@ -0,0 +1,10 @@ +package uk.ac.gla.terrier.structures.postings; + +import java.io.Closeable; +import java.io.IOException; + +public interface IterablePosting extends Posting, Closeable { + /** move to the next document. + * return false iff end of posting list has been met */ + public boolean next() throws IOException; +} Index: uk/ac/gla/terrier/structures/postings/Posting.java =================================================================== --- uk/ac/gla/terrier/structures/postings/Posting.java (revision 0) +++ uk/ac/gla/terrier/structures/postings/Posting.java (revision 0) @@ -0,0 +1,13 @@ +package uk.ac.gla.terrier.structures.postings; + +public interface Posting { + /** Return the document id of the current posting */ + public int getDocId(); + /** Return the frequency of the term in the current document */ + public int getFrequency(); + /** Return the length of the document for this posting. + * Usually uses the DocumentIndex, may do otherwise if + * doclength in posting list + */ + public int getDocumentLength(); +} Index: uk/ac/gla/terrier/structures/InvertedIndex.java =================================================================== --- uk/ac/gla/terrier/structures/InvertedIndex.java (revision 2532) +++ uk/ac/gla/terrier/structures/InvertedIndex.java (working copy) @@ -34,6 +34,7 @@ import uk.ac.gla.terrier.compression.BitIn; import uk.ac.gla.terrier.compression.BitInSeekable; import uk.ac.gla.terrier.compression.OldBitFile; +import uk.ac.gla.terrier.structures.postings.IterablePosting; import uk.ac.gla.terrier.utility.FieldScore; /** * This class implements the inverted index @@ -65,6 +66,7 @@ protected BitInSeekable file; /** Filename of the open file */ protected String filename; + protected DocumentIndex doi; /** A constructor for child classes that doesnt open the file */ protected InvertedIndex(long a, long b, long c) { } @@ -79,7 +81,12 @@ public InvertedIndex(Index index, String structureName) { - this(index.getPath() + "/" + index.getPrefix() + "." + structureName + ".bf"); + this(index, structureName, index.getDocumentIndex()); + } + + public InvertedIndex(Index index, String structureName, DocumentIndex doi) + { + this(index.getPath() + "/" + index.getPrefix() + "." + structureName + ".bf", doi); } /** @@ -87,8 +94,9 @@ * lexicon. * @param filename The name of the inverted file */ - public InvertedIndex(String filename) { + public InvertedIndex(String filename, DocumentIndex _doi) { file = new BitFileBuffered(this.filename = filename); + doi = _doi; } /** forces the data structure to reopen the underlying bitfile * using the legacy implementation of BitFile (OldBitFile) @@ -106,6 +114,64 @@ throw new UnsupportedOperationException("InvIndex.print() is missing"); } + static class BasicIterablePosting implements IterablePosting + { + int numEntries; + int docid; + int tf; + BitIn bitFileReader; + DocumentIndex doi; + + protected void readFirst() throws IOException + { + docid = bitFileReader.readGamma() -1; + tf = bitFileReader.readUnary(); + numEntries--; + } + + BasicIterablePosting(BitIn _bitFileReader, int _numEntries, DocumentIndex _doi) throws IOException { + bitFileReader = _bitFileReader; + numEntries = _numEntries; + doi = _doi; + if (this.getClass() == BasicIterablePosting.class) + readFirst(); + } + + public boolean next() throws IOException { + if (numEntries-- == 0) + return false; + docid = bitFileReader.readGamma() + docid; + tf = bitFileReader.readUnary(); + return true; + } + + public int getDocId() { + return docid; + } + + public int getDocumentLength() { + return doi.getDocumentLength(docid); + } + + public int getFrequency() { + return tf; + } + + public void close() throws IOException { + bitFileReader.close(); + } + } + + public IterablePosting getPostings(BitIndexPointer pointer) throws IOException { + final BitIn file = this.file.readReset(pointer.getBytes(), pointer.getBits()); + final int fieldCount = FieldScore.FIELDS_COUNT; + if (fieldCount > 0) + { + throw new IOException("Fields not support this way"); + } + return new BasicIterablePosting(file, pointer.getNumberOfEntries(), doi); + } + public int[][] getDocuments(BitIndexPointer pointer) { if (pointer==null) Index: uk/ac/gla/terrier/structures/indexing/InvertedIndexBuilder.java =================================================================== --- uk/ac/gla/terrier/structures/indexing/InvertedIndexBuilder.java (revision 2532) +++ uk/ac/gla/terrier/structures/indexing/InvertedIndexBuilder.java (working copy) @@ -370,8 +370,8 @@ index.addIndexStructure( structureName, "uk.ac.gla.terrier.structures.InvertedIndex", - "uk.ac.gla.terrier.structures.Index,java.lang.String", - "index,structureName"); + "uk.ac.gla.terrier.structures.Index,java.lang.String,uk.ac.gla.terrier.structures.DocumentIndex", + "index,structureName,document"); index.addIndexStructureInputStream( structureName, "uk.ac.gla.terrier.structures.InvertedIndexInputStream", Index: uk/ac/gla/terrier/structures/indexing/BlockInvertedIndexBuilder.java =================================================================== --- uk/ac/gla/terrier/structures/indexing/BlockInvertedIndexBuilder.java (revision 2532) +++ uk/ac/gla/terrier/structures/indexing/BlockInvertedIndexBuilder.java (working copy) @@ -303,8 +303,8 @@ index.addIndexStructure( structureName, "uk.ac.gla.terrier.structures.BlockInvertedIndex", - "uk.ac.gla.terrier.structures.Index,java.lang.String", - "index,structureName"); + "uk.ac.gla.terrier.structures.Index,java.lang.String,uk.ac.gla.terrier.structures.DocumentIndex", + "index,structureName,document"); index.addIndexStructureInputStream( structureName, "uk.ac.gla.terrier.structures.BlockInvertedIndexInputStream", Index: uk/ac/gla/terrier/structures/BlockInvertedIndex.java =================================================================== --- uk/ac/gla/terrier/structures/BlockInvertedIndex.java (revision 2532) +++ uk/ac/gla/terrier/structures/BlockInvertedIndex.java (working copy) @@ -30,6 +30,8 @@ import java.io.IOException; import uk.ac.gla.terrier.compression.BitIn; +import uk.ac.gla.terrier.structures.postings.BlockPosting; +import uk.ac.gla.terrier.structures.postings.IterablePosting; import uk.ac.gla.terrier.utility.FieldScore; /** * This class implements the block field inverted @@ -40,6 +42,11 @@ public class BlockInvertedIndex extends InvertedIndex implements IndexConfigurable { protected int DocumentBlockCountDelta = 1; protected BlockInvertedIndex() {} + + public BlockInvertedIndex(Index index, String structureName, DocumentIndex doi) { + super(index, structureName, doi); + } + public BlockInvertedIndex(Index index, String structureName) { super(index, structureName); @@ -50,8 +57,8 @@ * using the given lexicon. * @param filename the name of the inverted file */ - public BlockInvertedIndex(String filename) { - super(filename); + public BlockInvertedIndex(String filename, DocumentIndex doi) { + super(filename, doi); } /** let it know which index to use */ @@ -155,4 +162,54 @@ return null; } } + + @Override + public IterablePosting getPostings(BitIndexPointer pointer) throws IOException + { + final BitIn file = this.file.readReset(pointer.getBytes(), pointer.getBits()); + final int fieldCount = FieldScore.FIELDS_COUNT; + if (fieldCount > 0) + { + throw new IOException("Fields not support this way"); + } + return new BlockIterablePosting(file, pointer.getNumberOfEntries(), doi); + } + + static class BlockIterablePosting extends InvertedIndex.BasicIterablePosting implements BlockPosting + { + int[] positions; + + BlockIterablePosting(BitIn _bitFileReader, int _numEntries, DocumentIndex doi) throws IOException { + super(_bitFileReader, _numEntries, doi); + if (this.getClass() == BlockIterablePosting.class) + this.readFirst(); + } + + protected void readFirst() throws IOException + { + super.readFirst(); //TODO: call from constructor problem + positions = new int[bitFileReader.readUnary() -1]; + positions[0] = bitFileReader.readGamma() -1; + for(int i=1;i0) { mqt.addDocumentScoreModifier(new BooleanScoreModifier(requirement_list)); @@ -594,11 +597,14 @@ mqt.setQuery(q); mqt.normaliseTermWeights(); - matching.match(rq.getQueryID(), mqt); - //matching.match(rq.getQueryID(), rq.getMatchingQueryTerms()); - //now crop the collectionresultset down to a query result set. - ResultSet outRs = matching.getResultSet(); - rq.setResultSet((ResultSet)(outRs.getResultSet(0, outRs.getResultSize()))); + try{ + ResultSet outRs = matching.match(rq.getQueryID(), mqt); + //now crop the collectionresultset down to a query result set. + rq.setResultSet((ResultSet)(outRs.getResultSet(0, outRs.getResultSize()))); + } catch (IOException ioe) { + logger.error("Problem running Matching, returning empty result set as query"+rq.getQueryID(), ioe); + rq.setResultSet(new QueryResultSet(0)); + } } else { @@ -607,7 +613,7 @@ } } /** Runs the PostProcessing modules in order added. PostProcess modules - * alter the resultset. Examples might be query expansions which completelty replaces + * alter the resultset. Examples might be query expansions which completely replaces * the resultset. * @param srq the current SearchRequest object. */ public void runPostProcessing(SearchRequest srq) Index: uk/ac/gla/terrier/matching/tsms/TermInFieldModifier.java =================================================================== --- uk/ac/gla/terrier/matching/tsms/TermInFieldModifier.java (revision 2526) +++ uk/ac/gla/terrier/matching/tsms/TermInFieldModifier.java (working copy) @@ -24,6 +24,9 @@ * Vassilis Plachouras (original author) */ package uk.ac.gla.terrier.matching.tsms; +import uk.ac.gla.terrier.matching.models.WeightingModel; +import uk.ac.gla.terrier.structures.postings.FieldPosting; +import uk.ac.gla.terrier.structures.postings.Posting; import uk.ac.gla.terrier.utility.FieldScore; /** * Resets the scores of documents according to whether a term appears in @@ -31,7 +34,7 @@ * @author Vassilis Plachouras * @version $Revision: 1.11 $ */ -public class TermInFieldModifier implements TermScoreModifier { +public class TermInFieldModifier extends WeightingModel implements TermScoreModifier { /** * The fields that a query term should appear in. @@ -115,6 +118,31 @@ return numOfModifiedDocs; } + + int fieldIndex = 0; + public void prepare() + { + //TODO populate fieldIndex if not already done + } + + //implementation assumes scores are additive + public double score(Posting _p) + { + FieldPosting p = (FieldPosting)_p; + if (requirement) + { + if (p.getFieldFrequencies()[fieldIndex] == 0) + return Double.NEGATIVE_INFINITY; + } + else + { + if (p.getFieldFrequencies()[fieldIndex] > 0) + return Double.NEGATIVE_INFINITY; + } + return 0; + } + + public String getName() { return "TermInFieldModifier("+field+","+requirement+")"; } @@ -124,4 +152,23 @@ //no need to clone field, as Strings are immutable return (Object)new TermInFieldModifier(field, requirement); } + + @Override + public String getInfo() { + // TODO Auto-generated method stub + return null; + } + + @Override + public double score(double tf, double docLength) { + // TODO Auto-generated method stub + return 0; + } + + @Override + public double score(double tf, double docLength, double n_t, double F_t, + double keyFrequency) { + // TODO Auto-generated method stub + return 0; + } } Index: uk/ac/gla/terrier/matching/tsms/RequiredTermModifier.java =================================================================== --- uk/ac/gla/terrier/matching/tsms/RequiredTermModifier.java (revision 2526) +++ uk/ac/gla/terrier/matching/tsms/RequiredTermModifier.java (working copy) @@ -25,6 +25,9 @@ */ package uk.ac.gla.terrier.matching.tsms; import java.io.Serializable; + +import uk.ac.gla.terrier.matching.models.WeightingModel; +import uk.ac.gla.terrier.structures.postings.Posting; /** * Resets the scores of documents according to whether a term is required * or not, and whether it appears in the retrieved documents. This class @@ -32,7 +35,7 @@ * @author Vassilis Plachouras * @version $Revision: 1.13 $ */ -public class RequiredTermModifier implements TermScoreModifier,Serializable { +public class RequiredTermModifier extends WeightingModel implements TermScoreModifier,Serializable { /** * Indicates whether a query term is required, not required, * or whether this has been left unspecified. @@ -89,4 +92,29 @@ { return (Object)new RequiredTermModifier(required); } + + @Override + public String getInfo() { + return getName(); + } + + public double score(Posting p) + { + if (! required) + return Double.NEGATIVE_INFINITY; + return 0; + } + + @Override + public double score(double tf, double docLength) { + // TODO Auto-generated method stub + return 0; + } + + @Override + public double score(double tf, double docLength, double n_t, double F_t, + double keyFrequency) { + // TODO Auto-generated method stub + return 0; + } } Index: uk/ac/gla/terrier/matching/Matching.java =================================================================== --- uk/ac/gla/terrier/matching/Matching.java (revision 2532) +++ uk/ac/gla/terrier/matching/Matching.java (working copy) @@ -25,17 +25,20 @@ * Craig Macdonald */ package uk.ac.gla.terrier.matching; -import org.apache.log4j.Logger; +import java.io.IOException; import java.util.ArrayList; + +import org.apache.log4j.Logger; + import uk.ac.gla.terrier.matching.dsms.DocumentScoreModifier; import uk.ac.gla.terrier.matching.models.WeightingModel; import uk.ac.gla.terrier.matching.tsms.TermScoreModifier; import uk.ac.gla.terrier.structures.CollectionStatistics; -import uk.ac.gla.terrier.structures.DocumentIndex; import uk.ac.gla.terrier.structures.Index; import uk.ac.gla.terrier.structures.InvertedIndex; import uk.ac.gla.terrier.structures.Lexicon; import uk.ac.gla.terrier.structures.LexiconEntry; +import uk.ac.gla.terrier.structures.postings.IterablePosting; import uk.ac.gla.terrier.utility.ApplicationSetup; import uk.ac.gla.terrier.utility.HeapSort; /** @@ -51,13 +54,8 @@ public class Matching { /** the logger for this class */ - protected static Logger logger = Logger.getRootLogger(); - - /** - * The default namespace for the term score modifiers that are - * specified in the properties file. - */ - protected static String tsmNamespace = "uk.ac.gla.terrier.matching.tsms."; + protected static Logger logger = Logger.getLogger(Matching.class); + /** * The default namespace for the document score modifiers that are * specified in the properties file. @@ -72,15 +70,6 @@ */ protected static int RETRIEVED_SET_SIZE; - /** - * The maximum term frequency that is considered within a - * document. This helps to deal with 'spamming' in documents, - * where a term appears suspiciously too many times. The - * corresponding property is frequency.upper.threshold - * and the default value is 0, meaning that there is no - * threshold. - */ - protected static int FREQUENCY_UPPER_THRESHOLD; /** * A property that enables to ignore the terms with a low @@ -111,8 +100,6 @@ */ protected Index index; - /** The document index used.*/ - protected DocumentIndex docIndex; /** The lexicon used.*/ protected Lexicon lexicon; /** The inverted file.*/ @@ -133,9 +120,7 @@ * to be applied for a query. */ protected ArrayList documentModifiers; - - /** The weighting model used for retrieval.*/ - protected WeightingModel wmodel; + protected Matching() { /* do nothing constructor */} @@ -150,7 +135,6 @@ termModifiers = new ArrayList(); documentModifiers = new ArrayList(); this.index = index; - this.docIndex = index.getDocumentIndex(); this.lexicon = index.getLexicon(); this.invertedIndex = index.getInvertedIndex(); this.collectionStatistics = index.getCollectionStatistics(); @@ -158,18 +142,18 @@ //adding document and term score modifiers that will be //used for all queries, independently of the query operators //only modifiers with default constructors can be used in this way. - String defaultTSMS = ApplicationSetup.getProperty("matching.tsms",""); + //String defaultTSMS = ApplicationSetup.getProperty("matching.tsms",""); String defaultDSMS = ApplicationSetup.getProperty("matching.dsms",""); try { - for(String modifierName : defaultTSMS.split("\\s*,\\s*")) - { - if (modifierName.length() == 0) - continue; - if (modifierName.indexOf('.') == -1) - modifierName = tsmNamespace + modifierName; - addTermScoreModifier((TermScoreModifier)Class.forName(modifierName).newInstance()); - } +// for(String modifierName : defaultTSMS.split("\\s*,\\s*")) +// { +// if (modifierName.length() == 0) +// continue; +// if (modifierName.indexOf('.') == -1) +// modifierName = tsmNamespace + modifierName; +// addTermScoreModifier((TermScoreModifier)Class.forName(modifierName).newInstance()); +// } for(String modifierName : defaultDSMS.split("\\s*,\\s*")) { @@ -198,7 +182,7 @@ protected void initialise() { resultSet.initialise(); RETRIEVED_SET_SIZE = Integer.parseInt(ApplicationSetup.getProperty("matching.retrieved_set_size", "1000")); - FREQUENCY_UPPER_THRESHOLD = Integer.parseInt(ApplicationSetup.getProperty("frequency.upper.threshold", "0")); + //FREQUENCY_UPPER_THRESHOLD = Integer.parseInt(ApplicationSetup.getProperty("frequency.upper.threshold", "0")); IGNORE_LOW_IDF_TERMS = Boolean.parseBoolean(ApplicationSetup.getProperty("ignore.low.idf.terms","true")); MATCH_EMPTY_QUERY = Boolean.parseBoolean(ApplicationSetup.getProperty("match.empty.query","false")); } @@ -211,30 +195,12 @@ protected void initialise(double[] scs) { resultSet.initialise(scs); RETRIEVED_SET_SIZE = Integer.parseInt(ApplicationSetup.getProperty("matching.retrieved_set_size", "1000")); - FREQUENCY_UPPER_THRESHOLD = Integer.parseInt(ApplicationSetup.getProperty("frequency.upper.threshold", "0")); + //FREQUENCY_UPPER_THRESHOLD = Integer.parseInt(ApplicationSetup.getProperty("frequency.upper.threshold", "0")); IGNORE_LOW_IDF_TERMS = Boolean.parseBoolean(ApplicationSetup.getProperty("ignore.low.idf.terms","true")); MATCH_EMPTY_QUERY = Boolean.parseBoolean(ApplicationSetup.getProperty("match.empty.query","false")); } /** - * Registers a term score modifier. If more than one modifiers - * are registered, then they applied in the order they were registered. - * @param termScoreModifier TermScoreModifier the score modifier to be - * applied. - */ - public void addTermScoreModifier(TermScoreModifier termScoreModifier) { - termModifiers.add(termScoreModifier); - } - - /** - * Returns the i-th registered term score modifier. - * @return the i-th registered term score modifier. - */ - public TermScoreModifier getTermScoreModifier(int i) { - return (TermScoreModifier)termModifiers.get(i); - } - - /** * Registers a document score modifier. If more than one modifiers * are registered, then they applied in the order they were registered. * @param documentScoreModifier DocumentScoreModifier the score modifier to be @@ -252,18 +218,20 @@ return (DocumentScoreModifier)documentModifiers.get(i); } /** - * Sets the weihting model used for retrieval. + * Sets the weighting model used for retrieval. * @param model the weighting model used for retrieval + * @deprecated */ public void setModel(Model model) { - wmodel = (WeightingModel)model; + //wmodel = (WeightingModel)model; } /** * Returns a descriptive string for the retrieval process performed. */ public String getInfo() { - return wmodel.getInfo(); + return "TODO"; + //return wmodel.getInfo(); } /** @@ -271,7 +239,7 @@ * @param queryNumber the identifier of the processed query. * @param queryTerms the query terms to be processed. */ - public void match(String queryNumber, MatchingQueryTerms queryTerms) { + public ResultSet match(String queryNumber, MatchingQueryTerms queryTerms) throws IOException { //the first step is to initialise the arrays of scores and document ids. initialise(); //load in the dsms @@ -280,15 +248,15 @@ if (dsms!=null) NumberOfQueryDSMs = dsms.length; - //and prepare for the tsms - TermScoreModifier[] tsms; //int NumberOfQueryTSMs = 0; + + String[] queryTermStrings = queryTerms.getTerms(); //check whether we need to match an empty query. //if so, then return the existing result set. if (MATCH_EMPTY_QUERY && queryTermStrings.length == 0) { resultSet.setExactResultSize(collectionStatistics.getNumberOfDocuments()); resultSet.setResultSize(collectionStatistics.getNumberOfDocuments()); - return; + return resultSet; } //in order to save the time from references to the arrays, we create local references int[] docids = resultSet.getDocids(); @@ -301,101 +269,61 @@ numberOfRetrievedDocuments = 0; //the pointers read from the inverted file - int[][] pointers; - - //the number of term score modifiers - int numOfTermModifiers = termModifiers.size(); + IterablePosting postings; //the number of document score modifiers int numOfDocModifiers = documentModifiers.size(); - //int numberOfModifiedDocumentScores =0; - - //inform the weighting model of the collection statistics - wmodel.setNumberOfTokens((double)collectionStatistics.getNumberOfTokens()); - wmodel.setNumberOfDocuments((double)collectionStatistics.getNumberOfDocuments()); - wmodel.setAverageDocumentLength((double)collectionStatistics.getAverageDocumentLength()); - wmodel.setNumberOfUniqueTerms((double)collectionStatistics.getNumberOfUniqueTerms()); - wmodel.setNumberOfPointers((double)collectionStatistics.getNumberOfPointers()); + //inform the weighting model of the collection statistics + //for each query term in the query final int queryLength = queryTermStrings.length; for (int i = 0; i < queryLength; i++) { //we seek the query term in the lexicon LexiconEntry lEntry = lexicon.getLexiconEntry(queryTermStrings[i]); - //boolean found = lexicon.findTerm(queryTermStrings[i]); + //and if it is not found, we continue with the next term if (lEntry==null) { logger.info("Term Not Found: "+queryTermStrings[i]); continue; } - //because when the TreeNode is created, the term code assigned is taken from - //the TermCodes class, the assigned term code is only valid during the indexing - //process. Therefore, at this point, the term code should be updated with the one - //stored in the lexicon file. queryTerms.setTermProperty(queryTermStrings[i], lEntry); - //the weighting model is prepared for assigning scores to documents - wmodel.setKeyFrequency(queryTerms.getTermWeight(queryTermStrings[i])); - wmodel.setDocumentFrequency((double)lEntry.getDocumentFrequency()); - wmodel.setTermFrequency((double)lEntry.getFrequency()); - logger.debug((i + 1) + ": " + queryTermStrings[i].trim() + " with " + lEntry.getDocumentFrequency() + " documents (TF is " + lEntry.getFrequency() + ")."); - - + //check if the IDF is very low. - if (IGNORE_LOW_IDF_TERMS && docIndex.getNumberOfDocuments() < lEntry.getFrequency()) { + if (IGNORE_LOW_IDF_TERMS && collectionStatistics.getNumberOfDocuments() < lEntry.getFrequency()) { logger.debug("query term " + queryTermStrings[i] + " has low idf - ignored from scoring."); continue; } - //the postings are being read from the inverted file. - pointers = invertedIndex.getDocuments(lEntry); - //TODO: some consideration here for the PositionInvertedIndex. - //ie PositionInvertedIndex will returns positions by default, - //eg, perhaps it should change and not this code - - final int numberOfPointers = pointers[0].length; - //the scores for the particular term - final double[] termScores = new double[numberOfPointers]; + //the weighting models are prepared for assigning scores to documents + WeightingModel[] termWeightingModels = queryTerms.getTermWeightingModels(queryTermStrings[i]); - //assign scores to documents for a term - //assignScores(termScores, pointers); - assignScores(termScores, pointers, lEntry, queryTerms.getTermWeight(queryTermStrings[i])); - //application dependent modification of scores - //of documents for a term - //numberOfModifiedDocumentScores = 0; - for (int t = 0; t < numOfTermModifiers; t++) - termModifiers.get(t).modifyScores(termScores, pointers); - //application dependent modification of scores - //of documents for a term. These are predefined by the query - tsms = queryTerms.getTermScoreModifiers(queryTermStrings[i]); - if (tsms!=null) { - for (int t=0; t 0.0d)) { - numberOfRetrievedDocuments++; - } else if ((scores[docid] > 0.0d) && (termScores[k] < 0.0d)) { - numberOfRetrievedDocuments--; - } - scores[docid] += termScores[k]; - occurences[docid] |= mask; + for (WeightingModel wmodel: termWeightingModels) + { + wmodel.setCollectionStatistics(collectionStatistics); + wmodel.setRequest(queryTerms.getRequest()); + wmodel.setKeyFrequency(queryTerms.getTermWeight(queryTermStrings[i])); + wmodel.setEntryStatistics(lEntry); + //this requests any pre-calculations to be made + wmodel.prepare(); } + + //the postings are being read from the inverted file. + postings = invertedIndex.getPostings(lEntry); + + //assign scores to documents for a term + //assignScores(termScores, pointers); + assignScores(i, termWeightingModels, resultSet, postings, lEntry, queryTerms.getTermWeight(queryTermStrings[i])); } logger.debug("Number of docs with +ve score: "+numberOfRetrievedDocuments); //sort in descending score order the top RETRIEVED_SET_SIZE documents @@ -443,45 +371,41 @@ HeapSort.descendingHeapSort(scores, docids, occurences, resultSet.getResultSize()); } logger.debug("number of retrieved documents: " + resultSet.getResultSize()); - + return resultSet; } - /** Slight overhead on an aditional method call layer, but makes easier to override behaviour in child classes - * Calls assignScores(double[], double[]) by default. */ - protected void assignScores(final double[] scores, final int[][] pointers, LexiconEntry lEntry, double queryTermWeight) + /** Assign scores method */ + protected void assignScores(int i, final WeightingModel[] wModels, ResultSet rs, final IterablePosting postings, LexiconEntry lEntry, double queryTermWeight) + throws IOException { - assignScores(scores, pointers); - } - - /** - * Assigns scores to documents for a particular term. - * @param scores double[] the scores of the documents for the query term. - * @param pointers int[][] the pointers read from the inverted file - * for a particular query term. - */ - protected void assignScores(final double[] scores, final int[][] pointers) { - final int[] pointers1 = pointers[0]; - final int[] pointers2 = pointers[1]; - final int numOfPointers = pointers1.length; + + //for each document that contains //the query term, the score is computed. double score; - - //checking whether we have setup an upper threshold - //for within document frequencies. If yes, we check - //whether we need to change the current term's frequency. - if (FREQUENCY_UPPER_THRESHOLD > 0) - for (int j = 0; j < numOfPointers; j++) - pointers2[j] = pointers2[j] > FREQUENCY_UPPER_THRESHOLD ? FREQUENCY_UPPER_THRESHOLD : pointers2[j]; - - for (int j = 0; j < numOfPointers; j++) { - //increase the number of retrieved documents if the - //previous score was zero and the added score is positive - //sometimes negative scores occur due to very low - //probabilities - if ((score = wmodel.score(pointers2[j], docIndex.getDocumentLength(pointers1[j]))) > 0) - scores[j] = score; - //else - // logger.debug("Wmodel gave -ve or 0 score for docid "+pointers1[j] + ", score was "+score); - } + double[] scores = rs.getScores(); + short[] occurences = rs.getOccurrences(); + + //finally setting the scores of documents for a term + //a mask for setting the occurrences + short mask = 0; + if (i<16) + mask = (short)(1 << i); + + int docid; + do + { + score = 0; docid = postings.getDocId(); + for (WeightingModel wmodel: wModels) + { + score += wmodel.score(postings); + } + if ((scores[docid] == 0.0d) && (score > 0.0d)) { + numberOfRetrievedDocuments++; + } else if ((scores[docid] > 0.0d) && (score < 0.0d)) { + numberOfRetrievedDocuments--; + } + scores[docid] += score; + occurences[docid] |= mask; + }while(postings.next()); } } Index: uk/ac/gla/terrier/matching/LMMatching.java =================================================================== --- uk/ac/gla/terrier/matching/LMMatching.java (revision 2532) +++ uk/ac/gla/terrier/matching/LMMatching.java (working copy) @@ -31,6 +31,7 @@ import uk.ac.gla.terrier.matching.dsms.DocumentScoreModifier; import uk.ac.gla.terrier.matching.models.languagemodel.LanguageModel; import uk.ac.gla.terrier.matching.tsms.TermScoreModifier; +import uk.ac.gla.terrier.structures.DocumentIndex; import uk.ac.gla.terrier.structures.Index; import uk.ac.gla.terrier.structures.LexiconEntry; import uk.ac.gla.terrier.structures.indexing.DocumentInitialWeightIndex; @@ -56,6 +57,8 @@ /** The TermEstimateIndex for assigning scores. */ protected TermEstimateIndex termEstimateIndex; + protected DocumentIndex docIndex; + /** * The term estimate for each query term. The length of the * array is equal to query length. @@ -79,6 +82,7 @@ */ public LMMatching(Index index) { super(index); + this.docIndex = index.getDocumentIndex(); int numberOfDocs = collectionStatistics.getNumberOfDocuments(); initialScores = new double[numberOfDocs]; if (! index.hasIndexStructure("documentinitialweight")) @@ -118,7 +122,7 @@ * @param queryNumber the identifier of the processed query. * @param queryTerms the query terms to be processed. */ - public void match(String queryNumber, MatchingQueryTerms queryTerms) { + public ResultSet match(String queryNumber, MatchingQueryTerms queryTerms) { this.initialise(queryTerms.length()); //load in the dsms @@ -135,7 +139,7 @@ if (MATCH_EMPTY_QUERY && queryTermStrings.length == 0) { resultSet.setExactResultSize(collectionStatistics.getNumberOfDocuments()); resultSet.setResultSize(collectionStatistics.getNumberOfDocuments()); - return; + return resultSet; } //in order to save the time from references to the @@ -195,7 +199,7 @@ } //check if the IDF is very low. if(logger.isInfoEnabled()){ - if (IGNORE_LOW_IDF_TERMS==true && docIndex.getNumberOfDocuments() < le.getFrequency()) { + if (IGNORE_LOW_IDF_TERMS==true && collectionStatistics.getNumberOfDocuments() < le.getFrequency()) { logger.info("query term " + queryTermStrings[i] + " has low idf - ignored from scoring."); continue; } @@ -208,19 +212,7 @@ //the scores for the particular term double[] termScores = new double[pointers[0].length]; - //application dependent modification of scores - //of documents for a term - //numberOfModifiedDocumentScores = 0; - for (int t = 0; t < numOfTermModifiers; t++) - ((TermScoreModifier)termModifiers.get(t)).modifyScores(termScores,pointers); - //application dependent modification of scores - //of documents for a term. These are predefined by the query - tsms = queryTerms.getTermScoreModifiers(queryTermStrings[i]); - if (tsms!=null) { - for (int t=0; t 0 && frequency > FREQUENCY_UPPER_THRESHOLD) { - //TODO check whether we need to update the document length as well - //int diff = frequency - FREQUENCY_UPPER_THRESHOLD; - //docLength -= diff; - frequency = FREQUENCY_UPPER_THRESHOLD; - } //compute the score int docid = pointers1[j]; Index: uk/ac/gla/terrier/matching/models/WeightingModel.java =================================================================== --- uk/ac/gla/terrier/matching/models/WeightingModel.java (revision 2526) +++ uk/ac/gla/terrier/matching/models/WeightingModel.java (working copy) @@ -29,6 +29,10 @@ import java.io.Serializable; import uk.ac.gla.terrier.matching.Model; +import uk.ac.gla.terrier.querying.Request; +import uk.ac.gla.terrier.structures.CollectionStatistics; +import uk.ac.gla.terrier.structures.EntryStatistics; +import uk.ac.gla.terrier.structures.postings.Posting; /** * This class should be extended by the classes used * for weighting terms and documents. @@ -82,6 +86,37 @@ * @return java.lang.String */ public abstract String getInfo(); + public void prepare() + { + averageDocumentLength = cs.getAverageDocumentLength(); + numberOfDocuments = (double)cs.getNumberOfDocuments(); + numberOfTokens = (double)cs.getNumberOfTokens(); + numberOfUniqueTerms = (double)cs.getNumberOfUniqueTerms(); + numberOfPointers = (double)cs.getNumberOfPointers(); + } + public double score(Posting p) + { + return this.score(p.getFrequency(), p.getDocumentLength()); + } + + CollectionStatistics cs; + public void setCollectionStatistics(CollectionStatistics _cs) + { + cs = _cs; + } + EntryStatistics es; + public void setEntryStatistics(EntryStatistics _es) + { + es = _es; + } + + Request rq; + public void setRequest(Request _rq) + { + rq = _rq; + } + + /** * This method provides the contract for implementing weighting models. * @param tf The term frequency in the document @@ -131,6 +166,7 @@ /** * Sets the document frequency of the term in the collection. * @param docFreq the document frequency of the term in the collection. + * @deprecated Use setEntryStatistics(EntryStatistics) */ public void setDocumentFrequency(double docFreq) { documentFrequency = docFreq; @@ -142,9 +178,13 @@ public void setKeyFrequency(double keyFreq) { keyFrequency = keyFreq; } + + + /** * Set the number of tokens in the collection. * @param value The number of tokens in the collection. + * @deprecated Use setCollectionStatistics(CollectionStatistics) */ public void setNumberOfTokens(double value){ this.numberOfTokens = value; @@ -152,6 +192,7 @@ /** * Sets the number of documents in the collection. * @param numOfDocs the number of documents in the collection. + * @deprecated Use setCollectionStatistics(CollectionStatistics) */ public void setNumberOfDocuments(double numOfDocs) { numberOfDocuments = numOfDocs; @@ -160,18 +201,21 @@ /** * Sets the term's frequency in the collection. * @param termFreq the term's frequency in the collection. + * @deprecated Use setEntryStatistics(EntryStatistics) */ public void setTermFrequency(double termFreq) { termFrequency = termFreq; } /** * Set the number of unique terms in the collection. + * @deprecated Use setCollectionStatistics(CollectionStatistics) */ public void setNumberOfUniqueTerms(double number) { numberOfUniqueTerms = number; } /** * Set the number of pointers in the collection. + * @deprecated Use setCollectionStatistics(CollectionStatistics) */ public void setNumberOfPointers(double number) { numberOfPointers = number; Index: uk/ac/gla/terrier/matching/models/PL2.java =================================================================== --- uk/ac/gla/terrier/matching/models/PL2.java (revision 2526) +++ uk/ac/gla/terrier/matching/models/PL2.java (working copy) @@ -26,6 +26,9 @@ * Vassilis Plachouras */ package uk.ac.gla.terrier.matching.models; + +import uk.ac.gla.terrier.structures.postings.Posting; + /** * This class implements the PL2 weighting model. * @author Gianni Amati, Ben He, Vassilis Plachouras @@ -90,8 +93,7 @@ double n_t, double F_t, double keyFrequency) { - double TF = - tf * Idf.log(1.0d + (c * averageDocumentLength) / docLength); + double TF = tf * Idf.log(1.0d + (c * averageDocumentLength) / docLength); double NORM = 1.0D / (TF + 1d); double f = F_t / numberOfDocuments; return NORM @@ -101,4 +103,26 @@ + 0.5d * Idf.log(2 * Math.PI * TF) + TF * (Idf.log(TF) - Idf.REC_LOG_2_OF_E)); } + @Override + public double score(Posting p) + { + double TF = ((double)p.getFrequency()) * Idf.log(1.0d + Norm2Numerator / (double)p.getDocumentLength()); + double NORM = 1.0D / (TF + 1d); + return NORM + * keyFrequency + * (TF * Idf.log(1d / f) + + f * Idf.REC_LOG_2_OF_E + + 0.5d * Idf.log(2 * Math.PI * TF) + + TF * (Idf.log(TF) - Idf.REC_LOG_2_OF_E)); + } + double Norm2Numerator; + double f; + + @Override + public void prepare() + { + c = Double.parseDouble(rq.getControl("c")); + Norm2Numerator = c * (double)cs.getAverageDocumentLength(); + f = (double)es.getFrequency() / (double)cs.getNumberOfDocuments(); + } } Index: uk/ac/gla/terrier/matching/dsms/BooleanScoreModifier.java =================================================================== --- uk/ac/gla/terrier/matching/dsms/BooleanScoreModifier.java (revision 2526) +++ uk/ac/gla/terrier/matching/dsms/BooleanScoreModifier.java (working copy) @@ -25,11 +25,15 @@ * Craig Macdonald */ package uk.ac.gla.terrier.matching.dsms; -import java.io.Serializable; +import java.io.Serializable; import java.util.ArrayList; import java.util.HashSet; + +import org.apache.log4j.Logger; + import uk.ac.gla.terrier.matching.MatchingQueryTerms; import uk.ac.gla.terrier.matching.ResultSet; +import uk.ac.gla.terrier.querying.parser.Query; import uk.ac.gla.terrier.querying.parser.SingleTermQuery; import uk.ac.gla.terrier.structures.Index; /** @@ -38,13 +42,18 @@ * @author Vassilis Plachouras and Craig Macdonald * @version $Revision: 1.17 $ */ -public class BooleanScoreModifier implements DocumentScoreModifier,Serializable { +public class BooleanScoreModifier implements DocumentScoreModifier, Serializable { + private static final long serialVersionUID = 8827289509840106672L; + + /** the logger for this class */ + protected static Logger logger = Logger.getLogger(BooleanScoreModifier.class); + /** * The terms to check. If this is null, then we * check for the whole query. This property can * only be set from the constructor. */ - protected ArrayList terms = null; + protected ArrayList terms = null; /** * An empty default constructor. @@ -58,7 +67,7 @@ * appear in the retrieved documents after * applying this modifier. */ - public BooleanScoreModifier(ArrayList ts) { + public BooleanScoreModifier(ArrayList ts) { terms = ts; } @@ -90,8 +99,9 @@ //set the bit mask if (terms !=null) { HashSet set = new HashSet(); - for (int i=0; i(terms)); } } Index: uk/ac/gla/terrier/matching/MatchingQueryTerms.java =================================================================== --- uk/ac/gla/terrier/matching/MatchingQueryTerms.java (revision 2532) +++ uk/ac/gla/terrier/matching/MatchingQueryTerms.java (working copy) @@ -32,7 +32,8 @@ import java.util.Set; import uk.ac.gla.terrier.matching.dsms.DocumentScoreModifier; -import uk.ac.gla.terrier.matching.tsms.TermScoreModifier; +import uk.ac.gla.terrier.matching.models.WeightingModel; +import uk.ac.gla.terrier.querying.Request; import uk.ac.gla.terrier.querying.parser.Query; import uk.ac.gla.terrier.structures.EntryStatistics; /** @@ -63,7 +64,7 @@ EntryStatistics stats; /** The term score modifiers associated with a particular query term.*/ - ArrayList modifiers = new ArrayList(); + ArrayList termModels = new ArrayList(); /** An empty default constructor.*/ public QueryTermProperties() {} @@ -88,8 +89,8 @@ * A constructor for setting a term score modifier for a term. * @param tsm TermScoreModifier the modifier associated with a query term. */ - public QueryTermProperties(TermScoreModifier tsm) { - modifiers.add(tsm); + public QueryTermProperties(WeightingModel model) { + termModels.add(model); } /** @@ -98,9 +99,9 @@ * @param w double the weight of a query term. * @param tsm TermScoreModifier the modifier associated with a query term. */ - public QueryTermProperties(double w, TermScoreModifier tsm) { + public QueryTermProperties(double w, WeightingModel model) { weight = w; - modifiers.add(tsm); + termModels.add(model); } /** @@ -120,8 +121,8 @@ * @param tsm TermScoreModifier the modifier associated with a query term. * @param code int the term code of a query term. */ - public QueryTermProperties(TermScoreModifier tsm, EntryStatistics _stats) { - modifiers.add(tsm); + public QueryTermProperties(WeightingModel model, EntryStatistics _stats) { + termModels.add(model); stats = _stats; } @@ -131,17 +132,17 @@ * @param tsm TermScoreModifier the modifier associated with a query term. * @param code int the term code of a query term. */ - public QueryTermProperties(double w, TermScoreModifier tsm, EntryStatistics _stats) { + public QueryTermProperties(double w, WeightingModel model, EntryStatistics _stats) { weight = w; - modifiers.add(tsm); + termModels.add(model); stats = _stats; } public Object clone() { QueryTermProperties newO = new QueryTermProperties(weight, stats); - for (TermScoreModifier tsm : modifiers) - newO.modifiers.add((TermScoreModifier)(tsm.clone())); + for (WeightingModel model : termModels) + newO.termModels.add((WeightingModel)(model.clone())); return (Object)newO; } @@ -149,9 +150,9 @@ { int hashCodeValue = stats.hashCode(); hashCodeValue += (new Double(weight)).hashCode(); - for (TermScoreModifier tsm : modifiers) + for (WeightingModel model : termModels) { - hashCodeValue += tsm.hashCode(); + hashCodeValue += model.hashCode(); } return hashCodeValue; } @@ -161,9 +162,13 @@ /** The query ID, if provided */ protected String queryId = null; + protected Request rq = null; + /** A mapping from the string of a query term to its properties.*/ protected HashMap termProperties = new HashMap(); + protected WeightingModel defaultWeightingModel; + /** * The document score modifiers associated with the query terms. * It should contain the phrase score modifiers for example. @@ -185,6 +190,17 @@ { queryId = qid; } + + public MatchingQueryTerms(String qid, Request rq) + { + queryId = qid; + this.rq = rq; + } + + public Request getRequest() + { + return this.rq; + } /** * Adds a document score modifier for the query. @@ -313,12 +329,12 @@ * @param term String the term for which to add a term score modifier. * @param tsm TermScoreModifier the term score modifier to apply for the given term. */ - public void setTermProperty(String term, TermScoreModifier tsm) { + public void setTermProperty(String term, WeightingModel tsm) { QueryTermProperties properties = (QueryTermProperties)termProperties.get(term); if (properties == null) { termProperties.put(term, new QueryTermProperties(tsm)); } else { - properties.modifiers.add(tsm); + properties.termModels.add(tsm); } } @@ -328,13 +344,13 @@ * @param weight int the weight of the query term. * @param tsm TermScoreModifier the term score modifier applied for the query term. */ - public void setTermProperty(String term, double weight, TermScoreModifier tsm) { + public void setTermProperty(String term, double weight, WeightingModel tsm) { QueryTermProperties properties = (QueryTermProperties)termProperties.get(term); if (properties == null) { termProperties.put(term, new QueryTermProperties(weight, tsm)); } else { properties.weight = weight; - properties.modifiers.add(tsm); + properties.termModels.add(tsm); } } @@ -345,7 +361,7 @@ * of the query, then it returns 0. */ public double getTermWeight(String term) { - QueryTermProperties tp = (QueryTermProperties)termProperties.get(term); + QueryTermProperties tp = termProperties.get(term); if (tp!=null) return tp.weight; return 0.0d; @@ -372,30 +388,6 @@ return tp.stats; } - /** - * Returns the term score modifiers assocciated with the given query term. - * @param term String a query term. - * @return TermScoreModifiers[] the term score modifiers associated with - * the given query term, or null if the query term is not part - * of the query. - */ - public TermScoreModifier[] getTermScoreModifiers(String term) { - QueryTermProperties tp = termProperties.get(term); - if (tp!=null) - return (TermScoreModifier[])tp.modifiers.toArray(tmpTSM); - return null; - } - /** - * Returns the terms of the query. - * @return String[] an array of the query terms, or null if the query - * does not contain any terms. - */ - /*public String[] getTerms() { - Set keySet = termProperties.keySet(); - if (keySet.size()>0) - return (String[])keySet.toArray(tmpString); - return null; - }*/ // /** Currently uses Array.sort(Object[]), will retain order of query terms @@ -442,8 +434,25 @@ /* * The following attributes are used for creating arrays of the correct type. */ - private static final TermScoreModifier[] tmpTSM = new TermScoreModifier[0]; private static final DocumentScoreModifier[] tmpDSM = new DocumentScoreModifier[0]; private static final String[] tmpString = new String[0]; + private static final WeightingModel[] tmpModels = new WeightingModel[0]; + + public WeightingModel[] getTermWeightingModels(String term) { + QueryTermProperties qtp = termProperties.get(term); + if (qtp == null) + return tmpModels; + if (qtp.termModels.size() != 0) + { + final ArrayList n = new ArrayList(qtp.termModels); + n.add(0, defaultWeightingModel); + return n.toArray(tmpModels); + } + return new WeightingModel[]{defaultWeightingModel}; + } + + public void setDefaultTermWeightingModel(WeightingModel weightingModel) { + defaultWeightingModel = weightingModel; + } }