lucene 分词器

lucene的英文分词器主要用到StandardAnalyzer，中文的主要是极易分词MMAnalyzer（需要单独引jar包je-analysis-1.5.3.jar）。

英文分词的过程：1,关键词切分->2,去除停用词（is of）->3,形态还原（ing,ed,复数等）->4,转化为小写

中文分词：：1,关键词切分->2,去除停用词（的 着）

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.SimpleAnalyzer;
import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.standard.StandardAnalyzer;

public class AnalyzerTest {


	static String  enText = "The PGP signatures can be verified using PGP or GPG. ";
	static String  chText = "世界发达国家居民消费1000度的电能的费用占全国月平均工资的6.79%";
	static Analyzer en1 = new StandardAnalyzer();
	static Analyzer en2 = new SimpleAnalyzer();
	static Analyzer ch1 = new MMAnalyzer();
	
	
	
	/**
	 * @param args
	 */
	public static void main(String[] args) throws Exception{
		// TODO Auto-generated method stub
		new AnalyzerTest().analyze(chText, ch1);
		
	}
	
	
	public void analyze(String text,Analyzer analyzer) throws Exception{
		TokenStream tokenStream = analyzer.tokenStream(null, new StringReader(text));
		for (Token token = new Token();(token = tokenStream.next(token))!= null;){
			System.out.println(token);
		}
	}
	
}