lucene3.6.0的文档评估机制

lucene的评分机制：所有hits的分数<=1.0

每个document（d）的分数：

∑tf(t in d)*idf(t)*boost(t.field in d)*lengthNorm(t.field in d)

t In q

 

查询的得分：

score(q,d)=coord(q,d)·queryNorm(q)·∑tf(t in d)*idf(t)*boost(t.field in d)*lengthNorm(t.field in d)

t In q

 

tf(t in d):文档中d出现搜索项t的频率

idf(t):搜索项t在倒排文档中出现的频率

boost（t.field in d）：域的加权因子，在插入文档中设置

lengthNorm（t.field in d):域的标准化值，即在某一域中所有项的个数。通常在索引时计算该值并存储到索引文件中。

coord（q，d）：协调因子（normalization value），该因子的值基于文档中包含查询项的个数

queryNorm（q）：每个查询的标准化值，指每个查询项的权重的平方和

 

 

 

query对象的加权因子，查询时如果是多个子句，则可以通过加权某一个查询子句来加权某一个query对象。

 

DefaultSimilarity.java默认处理计分规则/** Expert: Default scoring implementation. */

public class DefaultSimilarity extends Similarity {

  /** Implemented as
   *  <code>state.getBoost()*lengthNorm(numTerms)</code>, where
   *  <code>numTerms</code> is {@link FieldInvertState#getLength()} if {@link
   *  #setDiscountOverlaps} is false, else it's {@link
   *  FieldInvertState#getLength()} - {@link
   *  FieldInvertState#getNumOverlap()}.
   *
   *  @lucene.experimental */
  @Override
  public float computeNorm(String field, FieldInvertState state) {
    final int numTerms;
    if (discountOverlaps)
      numTerms = state.getLength() - state.getNumOverlap();
    else
      numTerms = state.getLength();
    return state.getBoost() * ((float) (1.0 / Math.sqrt(numTerms)));
  }
  
  /** Implemented as <code>1/sqrt(sumOfSquaredWeights)</code>. */
  @Override
  public float queryNorm(float sumOfSquaredWeights) {
    return (float)(1.0 / Math.sqrt(sumOfSquaredWeights));
  }

  /** Implemented as <code>sqrt(freq)</code>. */
  @Override
  public float tf(float freq) {
    return (float)Math.sqrt(freq);
  }
    
  /** Implemented as <code>1 / (distance + 1)</code>. */
  @Override
  public float sloppyFreq(int distance) {
    return 1.0f / (distance + 1);
  }
    
  /** Implemented as <code>log(numDocs/(docFreq+1)) + 1</code>. */
  @Override
  public float idf(int docFreq, int numDocs) {
    return (float)(Math.log(numDocs/(double)(docFreq+1)) + 1.0);
  }
    
  /** Implemented as <code>overlap / maxOverlap</code>. */
  @Override
  public float coord(int overlap, int maxOverlap) {
    return overlap / (float)maxOverlap;
  }

  // Default true
  protected boolean discountOverlaps = true;

  /** Determines whether overlap tokens (Tokens with
   *  0 position increment) are ignored when computing
   *  norm.  By default this is true, meaning overlap
   *  tokens do not count when computing norms.
   *
   *  @lucene.experimental
   *
   *  @see #computeNorm
   */
  public void setDiscountOverlaps(boolean v) {
    discountOverlaps = v;
  }

  /** @see #setDiscountOverlaps */
  public boolean getDiscountOverlaps() {
    return discountOverlaps;
  }
}

 

 

IndexSearcher.java的explain方法返回的Explanation对象包含了所有评分因子中各个因子的详细信息。

测试程序和数据参考http://zhwj184.iteye.com/admin/blogs/1522709

import java.io.File;
import java.io.IOException;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.Version;

public class DocSearch {

	private static IndexSearcher isearcher = null;
	public static void search(String key) throws IOException, ParseException{
		 Directory directory = FSDirectory.open(new File("E:\\output\\lucence\\index"));
		 // Now search the index:
	    IndexReader ireader = IndexReader.open(directory); // read-only=true
	    isearcher  = new IndexSearcher(ireader);
	    // Parse a simple query that searches for "text":
	    Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_CURRENT);
	    QueryParser parser = new QueryParser(Version.LUCENE_CURRENT,"context", analyzer);
	    Query query = parser.parse(key);
	    ScoreDoc[] hits = isearcher.search(query, null, 1000).scoreDocs;
	    // Iterate through the results:
	    for (int i = 0; i < hits.length; i++) {
	      Document hitDoc = isearcher.doc(hits[i].doc);
	      System.out.println(hitDoc.getValues("id")[0] + "\t" + hitDoc.getValues("context")[0] + "\t" + hits[i].score);
	      Explanation explan = isearcher.explain(query, hits[i].doc);
	      System.out.println(explan);
	    }
	}
	
	public static void main(String[] args) throws IOException, ParseException {
		search("旧水泥袋");
		isearcher.close();
	}
	
}

 

输出结果：

只截取第一篇文档的评分信息

4801857        采购旧编织袋、旧水泥袋        4.0172114
4.0172114 = (MATCH) sum of:
  1.4140004 = (MATCH) weight(context:旧 in 1682), product of:
    0.54585564 = queryWeight(context:旧), product of:
      5.861472 = idf(docFreq=13, maxDocs=1809)
      0.09312603 = queryNorm
    2.5904293 = (MATCH) fieldWeight(context:旧 in 1682), product of:
      1.4142135 = tf(termFreq(context:旧)=2)
      5.861472 = idf(docFreq=13, maxDocs=1809)
      0.3125 = fieldNorm(field=context, doc=1682)
  0.60229266 = (MATCH) weight(context:水 in 1682), product of:
    0.42365694 = queryWeight(context:水), product of:
      4.549286 = idf(docFreq=51, maxDocs=1809)
      0.09312603 = queryNorm
    1.4216518 = (MATCH) fieldWeight(context:水 in 1682), product of:
      1.0 = tf(termFreq(context:水)=1)
      4.549286 = idf(docFreq=51, maxDocs=1809)
      0.3125 = fieldNorm(field=context, doc=1682)
  1.1562659 = (MATCH) weight(context:泥 in 1682), product of:
    0.58700174 = queryWeight(context:泥), product of:
      6.3033047 = idf(docFreq=8, maxDocs=1809)
      0.09312603 = queryNorm
    1.9697827 = (MATCH) fieldWeight(context:泥 in 1682), product of:
      1.0 = tf(termFreq(context:泥)=1)
      6.3033047 = idf(docFreq=8, maxDocs=1809)
      0.3125 = fieldNorm(field=context, doc=1682)
  0.84465253 = (MATCH) weight(context:袋 in 1682), product of:
    0.42188305 = queryWeight(context:袋), product of:
      4.5302377 = idf(docFreq=52, maxDocs=1809)
      0.09312603 = queryNorm
    2.0021012 = (MATCH) fieldWeight(context:袋 in 1682), product of:
      1.4142135 = tf(termFreq(context:袋)=2)
      4.5302377 = idf(docFreq=52, maxDocs=1809)
      0.3125 = fieldNorm(field=context, doc=1682)