Index: share/MANIFEST.txt =================================================================== --- share/MANIFEST.txt (revision 2540) +++ share/MANIFEST.txt (working copy) @@ -1196,3 +1196,8 @@ src//overview.html src//TrecTerrier.java src//package.html +src/uk/ac/gla/terrier/querying/RelevanceFeedbackSelectorDocids.java +src/uk/ac/gla/terrier/querying/RelevantOnlyFeedbackDocuments.java +src/uk/ac/gla/terrier/querying/RelevanceFeedbackSelector.java +src/uk/ac/gla/terrier/querying/FeedbackSelector.java +src/uk/ac/gla/terrier/querying/PseudoRelevanceFeedbackSelector.java Index: src/uk/ac/gla/terrier/querying/RelevanceFeedbackSelectorDocids.java =================================================================== --- src/uk/ac/gla/terrier/querying/RelevanceFeedbackSelectorDocids.java (revision 0) +++ src/uk/ac/gla/terrier/querying/RelevanceFeedbackSelectorDocids.java (revision 0) @@ -0,0 +1,93 @@ +package uk.ac.gla.terrier.querying; +import uk.ac.gla.terrier.utility.ApplicationSetup; +import uk.ac.gla.terrier.utility.Files; +import uk.ac.gla.terrier.structures.Index; +import uk.ac.gla.terrier.structures.DocumentIndex; +import gnu.trove.THashMap; +import gnu.trove.THashSet; + +import org.apache.log4j.Logger; +import gnu.trove.TIntByteHashMap; +import java.util.List; +import java.util.ArrayList; +import java.io.BufferedReader; +import java.io.IOException; +/** A feedback document selector that operates as RelevanceFeedbackSelector, except + * that this should be used when docids are specified in the qrels file, not docnos. + *

+ * Properties: + *

+ * @since 3.0 + * @author Craig Macdonald + */ +public class RelevanceFeedbackSelectorDocids extends FeedbackSelector +{ + protected static Logger logger = Logger.getLogger(RelevanceFeedbackSelectorDocids.class); + protected DocumentIndex doi; + protected THashMap queryidRelDocumentMap; + + public RelevanceFeedbackSelectorDocids() + { + String feedbackFilename = ApplicationSetup.getProperty("qe.feedback.filename", + ApplicationSetup.TERRIER_ETC+ + ApplicationSetup.FILE_SEPARATOR+"feedback"); + this.loadRelevanceInformation(feedbackFilename); + } + + public void setIndex(Index index){ + doi = index.getDocumentIndex(); + } + + public FeedbackDocument[] getFeedbackDocuments(Request request) + { + // get docids of the feedback documents + String queryid = request.getQueryID(); + TIntByteHashMap list = queryidRelDocumentMap.get(queryid); + //deal with undefined case + if (list == null) + return null; + //dela with empty case + if (list.size() == 0) + return new FeedbackDocument[0]; + final List rtrList = new ArrayList(list.size()); + for(int id: list.keys()) + { + FeedbackDocument doc = new FeedbackDocument(); + doc.docid = id; + doc.score = -1; + doc.rank = -1; + doc.relevance = list.get(id); + rtrList.add(doc); + } + logger.info("Found "+(rtrList.size())+" feedback documents for query "+request.getQueryID()); + return rtrList.toArray(new FeedbackDocument[0]); + } + + private void loadRelevanceInformation(String filename){ + logger.info("Loading relevance feedback assessments from "+ filename); + try{ + queryidRelDocumentMap = new THashMap(); + BufferedReader br = Files.openFileReader(filename); + String line = null; + int assessmentsCount =0; + while ((line=br.readLine())!=null){ + line=line.trim(); + if (line.length()==0) + continue; + String[] parts = line.split("\\s+"); + TIntByteHashMap list = queryidRelDocumentMap.get(parts[0]); + if (list == null) + { + queryidRelDocumentMap.put(parts[0], list = new TIntByteHashMap()); + } + list.put(Integer.parseInt(parts[2]), Byte.parseByte(parts[3])); + assessmentsCount++; + } + br.close(); + logger.info("Total "+ assessmentsCount+ " assessments found"); + }catch(IOException ioe){ + logger.error("Problem loading relevance feedback assessments from "+ filename, ioe); + } + } +} Index: src/uk/ac/gla/terrier/querying/RelevantOnlyFeedbackDocuments.java =================================================================== --- src/uk/ac/gla/terrier/querying/RelevantOnlyFeedbackDocuments.java (revision 0) +++ src/uk/ac/gla/terrier/querying/RelevantOnlyFeedbackDocuments.java (revision 0) @@ -0,0 +1,35 @@ +package uk.ac.gla.terrier.querying; +import uk.ac.gla.terrier.structures.Index; +import java.util.List; +import java.util.ArrayList; +import org.apache.log4j.Logger; +/** Select only feedback documents which have relevance &tg; 0 + * @author Craig Macdonald + * @since 3.0 + */ +public class RelevantOnlyFeedbackDocuments extends FeedbackSelector +{ + protected static Logger logger = Logger.getLogger(RelevantOnlyFeedbackDocuments.class); + protected final FeedbackSelector parent; + public RelevantOnlyFeedbackDocuments(FeedbackSelector _parent) + { + this.parent = _parent; + } + + public void setIndex(Index index){ + parent.setIndex(index); + } + + public FeedbackDocument[] getFeedbackDocuments(Request request) + { + FeedbackDocument[] parentReturn = parent.getFeedbackDocuments(request); + List rtr = new ArrayList(parentReturn.length); + for(FeedbackDocument candidateDocument : parentReturn) + { + if (candidateDocument.relevance > 0) + rtr.add(candidateDocument); + } + logger.info("Dropped "+(parentReturn.length - rtr.size())+" irrelevant feedback documents"); + return rtr.toArray(new FeedbackDocument[0]); + } +} Index: src/uk/ac/gla/terrier/querying/RelevanceFeedbackSelector.java =================================================================== --- src/uk/ac/gla/terrier/querying/RelevanceFeedbackSelector.java (revision 0) +++ src/uk/ac/gla/terrier/querying/RelevanceFeedbackSelector.java (revision 0) @@ -0,0 +1,107 @@ +package uk.ac.gla.terrier.querying; +import uk.ac.gla.terrier.utility.ApplicationSetup; +import uk.ac.gla.terrier.utility.Files; +import uk.ac.gla.terrier.structures.Index; +import uk.ac.gla.terrier.structures.DocumentIndex; +import gnu.trove.THashMap; +import gnu.trove.THashSet; + +import org.apache.log4j.Logger; + +import java.io.BufferedReader; +import java.io.IOException; +import java.util.List; +import java.util.ArrayList; +/** Selects feedback documents from a qrels file, using the query id. NB, will select + * all documents, irrespective of relevance. + *

+ * Properties: + *

  • qe.feedback.filename - filename of qrels file to use for feedback.
    • + *
+ * @since 3.0 + * @author Craig Macdonald + */ +public class RelevanceFeedbackSelector extends FeedbackSelector +{ + protected static Logger logger = Logger.getLogger(RelevanceFeedbackSelector.class); + protected DocumentIndex doi; + protected THashMap> queryidRelDocumentMap; + + static class FeedbackWithDocno extends FeedbackDocument + { + String docno; + } + + public RelevanceFeedbackSelector() + { + String feedbackFilename = ApplicationSetup.getProperty("qe.feedback.filename", + ApplicationSetup.TERRIER_ETC+ + ApplicationSetup.FILE_SEPARATOR+"feedback"); + this.loadRelevanceInformation(feedbackFilename); + } + + public void setIndex(Index index){ + doi = index.getDocumentIndex(); + } + + public FeedbackDocument[] getFeedbackDocuments(Request request) + { + // get docids of the feedback documents + String queryid = request.getQueryID(); + List list = queryidRelDocumentMap.get(queryid); + //deal with undefined case + if (list == null) + return null; + //dela with empty case + if (list.size() == 0) + return new FeedbackDocument[0]; + final List rtrList = new ArrayList(list.size()); + for(FeedbackWithDocno doc: list) + { + doc.docid = doi.getDocumentId(doc.docno); + if (doc.docid < 0) + { + logger.warn("Could not find docid for feedback document "+doc.docno+" of query "+ request.getQueryID()); + continue; + } + doc.score = -1; + doc.rank = -1; + logger.info("("+(rtrList.size()+1)+") Feedback document:"+doc.docno); + rtrList.add(doc); + } + logger.info("Found "+(rtrList.size())+" feedback documents for query "+request.getQueryID()); + return rtrList.toArray(new FeedbackDocument[0]); + } + + private void loadRelevanceInformation(String filename){ + logger.info("Loading relevance feedback assessments from "+ filename); + try{ + queryidRelDocumentMap = new THashMap>(); + BufferedReader br = Files.openFileReader(filename); + THashSet queryids = new THashSet(); + String line = null; + int assessmentsCount =0; + while ((line=br.readLine())!=null){ + line=line.trim(); + if (line.length()==0) + continue; + String[] parts = line.split("\\s+"); + FeedbackWithDocno doc = new FeedbackWithDocno(); + doc.docno = parts[2]; + doc.relevance = Byte.parseByte(parts[3]); + + List list = queryidRelDocumentMap.get(parts[0]); + if (list == null) + { + queryidRelDocumentMap.put(parts[0], list = new ArrayList()); + } + list.add(doc); + assessmentsCount++; + } + br.close(); + logger.info("Total "+ assessmentsCount+ " assessments found"); + }catch(IOException ioe){ + logger.error("Problem loading relevance feedback assessments from "+ filename, ioe); + } + } +} Index: src/uk/ac/gla/terrier/querying/FeedbackSelector.java =================================================================== --- src/uk/ac/gla/terrier/querying/FeedbackSelector.java (revision 0) +++ src/uk/ac/gla/terrier/querying/FeedbackSelector.java (revision 0) @@ -0,0 +1,23 @@ +package uk.ac.gla.terrier.querying; +import uk.ac.gla.terrier.structures.Index; +/** Implements of this class can be used to select feedback documents. + * Feedback documents are represented by the FeedbackDocument instances. + * @author Craig Macdonald + * @since 3.0 + */ +public abstract class FeedbackSelector +{ + + static public class FeedbackDocument + { + public int docid; + public int rank; + public byte relevance; + public double score; + } + + /** Set the index to be used */ + public void setIndex(Index index){} + /** Obtain feedback documents for the specified query request */ + public abstract FeedbackDocument[] getFeedbackDocuments(Request request); +} Index: src/uk/ac/gla/terrier/querying/QueryExpansion.java =================================================================== --- src/uk/ac/gla/terrier/querying/QueryExpansion.java (revision 2538) +++ src/uk/ac/gla/terrier/querying/QueryExpansion.java (working copy) @@ -36,6 +36,7 @@ import uk.ac.gla.terrier.matching.ResultSet; import uk.ac.gla.terrier.matching.models.queryexpansion.QueryExpansionModel; import uk.ac.gla.terrier.querying.parser.SingleTermQuery; +import uk.ac.gla.terrier.querying.FeedbackSelector.FeedbackDocument; import uk.ac.gla.terrier.structures.CollectionStatistics; import uk.ac.gla.terrier.structures.DirectIndex; import uk.ac.gla.terrier.structures.DocumentIndex; @@ -88,6 +89,8 @@ protected CollectionStatistics collStats; /** The query expansion model used. */ protected QueryExpansionModel QEModel; + /** The process by which to select feedback documents */ + protected FeedbackSelector selector = null; /** * The default constructor of QueryExpansion. */ @@ -102,10 +105,10 @@ * This method implements the functionality of expanding a query. * @param query MatchingQueryTerms the query terms of * the original query. - * @param resultSet CollectionResultSet the set of retrieved - * documents from the first pass retrieval. + * @param rq the Request thus far, giving access to the query and the result set */ - public void expandQuery(MatchingQueryTerms query, ResultSet resultSet) { + public void expandQuery(MatchingQueryTerms query, Request rq) + { // the number of term to re-weight (i.e. to do relevance feedback) is // the maximum between the system setting and the actual query length. // if the query length is larger than the system setting, it does not @@ -116,36 +119,35 @@ if (ApplicationSetup.EXPANSION_TERMS == 0) numberOfTermsToReweight = 0; - // If no document retrieved, keep the original query. - if (resultSet.getResultSize() == 0){ + if (selector == null) + selector = this.getFeedbackSelector(rq); + if (selector == null) return; - } - - int[] docIDs = resultSet.getDocids(); - double[] scores = resultSet.getScores(); + FeedbackDocument[] feedback = selector.getFeedbackDocuments(rq); + if (feedback == null || feedback.length == 0) + return; + double totalDocumentLength = 0; - - // if the number of retrieved documents is lower than the parameter - // EXPANSION_DOCUMENTS, reduce the number of documents for expansion - // to the number of retrieved documents. - int effDocuments = Math.min(docIDs.length, ApplicationSetup.EXPANSION_DOCUMENTS); - for (int i = 0; i < effDocuments; i++){ - totalDocumentLength += documentIndex.getDocumentLength(docIDs[i]); + for(FeedbackDocument doc : feedback) + { + totalDocumentLength += documentIndex.getDocumentLength(doc.docid); if(logger.isDebugEnabled()){ - logger.debug((i+1)+": " + documentIndex.getDocumentNumber(docIDs[i])+ - " ("+docIDs[i]+") with "+scores[i]); + logger.debug(doc.rank +": " + documentIndex.getDocumentNumber(doc.docid)+ + " ("+doc.docid+") with "+doc.score); } } ExpansionTerms expansionTerms = new ExpansionTerms(collStats, totalDocumentLength, lexicon); - for (int i = 0; i < effDocuments; i++) { - int[][] terms = directIndex.getTerms(docIDs[i]); + for(FeedbackDocument doc : feedback) + { + int[][] terms = directIndex.getTerms(doc.docid); if (terms == null) - logger.warn("document "+documentIndex.getDocumentLength(docIDs[i]) + "("+docIDs[i]+") not found"); + logger.warn("document "+documentIndex.getDocumentLength(doc.docid) + "("+doc.docid+") not found"); else for (int j = 0; j < terms[0].length; j++) expansionTerms.insertTerm(terms[0][j], (double)terms[1][j]); } + expansionTerms.setOriginalQueryTerms(query); SingleTermQuery[] expandedTerms = expansionTerms.getExpandedTerms(numberOfTermsToReweight, QEModel); @@ -157,7 +159,7 @@ + " appears in expanded query with normalised weight: " + Rounding.toString(query.getTermWeight(expandedTerms[i].getTerm()), 4)); } - } + } } @@ -168,14 +170,46 @@ return m.getIndex(); } + protected FeedbackSelector getFeedbackSelector(Request rq) + { + String[] names = ApplicationSetup.getProperty("qe.feedback.selector", "PseudoRelevanceFeedbackSelector").split("\\s*,\\s*"); + FeedbackSelector rtr = null; + for(int i=names.length -1;i>=0;i--) + { + String name = names[i]; + if (! name.contains(".")) + name = "uk.ac.gla.terrier.querying."+name; + FeedbackSelector next = null; + try{ + Class nextClass = Class.forName(name).asSubclass(FeedbackSelector.class); + if (names.length -1 == i) + { + next = nextClass.newInstance(); + } + else + { + next = nextClass.getConstructor(FeedbackSelector.class).newInstance(rtr); + } + + rtr = next; + } catch (Exception e) { + logger.error("Problem loading a FeedbackSelector called "+ name, e); + return null; + } + rtr.setIndex(lastIndex);//TODO index should come from Request + } + return rtr; + } + + protected Index lastIndex = null; //TODO remove - /** * Runs the actual query expansion * @see uk.ac.gla.terrier.querying.PostProcess#process(uk.ac.gla.terrier.querying.Manager,uk.ac.gla.terrier.querying.SearchRequest) */ public void process(Manager manager, SearchRequest q) { Index index = getIndex(manager); + lastIndex = index; documentIndex = index.getDocumentIndex(); invertedIndex = index.getInvertedIndex(); lexicon = index.getLexicon(); @@ -205,9 +239,8 @@ logger.warn("No query terms for this query. Skipping QE"); return; } - ResultSet resultSet = q.getResultSet(); // get the expanded query terms - expandQuery(queryTerms, resultSet); + expandQuery(queryTerms, (Request)q); if(logger.isInfoEnabled()){ logger.info("query length after expansion: " + queryTerms.length()); logger.info("Expanded query: "); Index: src/uk/ac/gla/terrier/querying/PseudoRelevanceFeedbackSelector.java =================================================================== --- src/uk/ac/gla/terrier/querying/PseudoRelevanceFeedbackSelector.java (revision 0) +++ src/uk/ac/gla/terrier/querying/PseudoRelevanceFeedbackSelector.java (revision 0) @@ -0,0 +1,36 @@ +package uk.ac.gla.terrier.querying; +import uk.ac.gla.terrier.structures.Index; +import uk.ac.gla.terrier.utility.ApplicationSetup; +import uk.ac.gla.terrier.matching.ResultSet; +/** A feedback selector for pseudo-relevance feedback. Selects the top ApplicationSetup.EXPANSION_DOCUMENTS + * documents from the ResultSet attached to the specified request. + * @since 3.0 + * @author Craig Macdonald + */ +public class PseudoRelevanceFeedbackSelector extends FeedbackSelector +{ + public PseudoRelevanceFeedbackSelector(){} + public FeedbackDocument[] getFeedbackDocuments(Request request) + { + final ResultSet rs = request.getResultSet(); + if (rs.getResultSize() == 0) + return null; + + final int[] docIDs = rs.getDocids(); + final double[] scores = rs.getScores(); + + // if the number of retrieved documents is lower than the parameter + // EXPANSION_DOCUMENTS, reduce the number of documents for expansion + // to the number of retrieved documents. + final int effDocuments = Math.min(docIDs.length, ApplicationSetup.EXPANSION_DOCUMENTS); + final FeedbackDocument[] rtr = new FeedbackDocument[effDocuments]; + for (int i = 0; i < effDocuments; i++) + { + rtr[i] = new FeedbackDocument(); + rtr[i].rank = i; + rtr[i].score = scores[i]; + rtr[i].docid = docIDs[i]; + } + return rtr; + } +}