IkAnalyzer分词、词频、内链优化

IK 中文分词  版本 5.0

package com.test;

import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
import java.util.Arrays;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.wltea.analyzer.core.IKSegmenter;
import org.wltea.analyzer.core.Lexeme;

import com.test.entity.ArticleThesaurus;

public class IkAnalyzerTest {
	public static void main(String[] args) {
//		String str = "铜管方<a href='http://auto.ifeng.com/' target='_blank'><font color='#004276'>面</font></a>，3月铜管企业开工率为85.54%，4月达88%。据了解，由于3月铜价低迷，铜管企业提前备货，精铜需求提前放量。"
//				+ "3月铜杆线企业开工率为72.47%，4月上升至76%。开工率上行，"
//				+ "但前期备货并不充足，导致现货市场集中采购增加，供应紧张凸显。fdsf紫铜铜inf的说法都是紫铜,"
//				+ "我勒个去黄铜恩恩黄铜哈哈哈黄铜铜,我勒个去白铜，范德萨范德萨发白铜，古代有很多人用青铜器，是的就是这个东西它的名字是一种金属叫青铜。";
		//System.out.println("Jane Campion directed \"The Piano\" in 1993.");
		String str="由于对经济形势的的担忧,市场一直低迷,各方面消息被吸<a href='http://www.test.cn/quote/'>铜价</a> 收,市场经济困难局面早已被认同," +
				"随着各国政策的出台实施及各经济数据的公布,担忧情绪渐缓,沪铝即将迎来谷底反弹的时机。
  </p> <p> 操作思路<img src=\"####\" alt=\"锻铜铍铜\" />" +
				"15500<a href='http://www.test.cn/product/tjthj_ht/'>黄铜</a>锻铜以下紫铜板多单继续持有,15575以下果断购入多单,止损15250价位," +
				"如果突破15750价位并站稳,可继续加仓购入多单。<img src=\"www.baidu.com\" alt=\"范德萨发生的\" />";
		Pattern p = Pattern.compile("</?(A|a)(\n|.)*?>");
		Matcher m = p.matcher(str);
		str = m.replaceAll("");
		System.out.println("清除所有a标签:"+str);
	
		
		System.out.println("分词后："+ikAnalyzer(str));
		String afterFcStr = ikAnalyzer(str); // 分词后的字符串

		// 计算词频
		Map<String, Integer> words = new HashMap<String, Integer>();
		IKSegmenter seg = new IKSegmenter(new StringReader(afterFcStr), true);
		try {
			Lexeme l = null;
			while ((l = seg.next()) != null) {
				if (words.containsKey(l.getLexemeText()))
					words.put(l.getLexemeText(),
							words.get(l.getLexemeText()) + 1);
				else
					words.put(l.getLexemeText(), 1);
			}
		} catch (IOException e) {
			e.printStackTrace();
		}

		 for (Map.Entry<String, Integer> entry : words.entrySet()) {
		 System.out.println("key= " + entry.getKey() + " and value= "
		 + entry.getValue());
		 }

		 Integer count=words.get("铜价");
		 if(count!=null){
		 System.out.println("该词频："+count);
		 }else{
		 System.out.println("该词频不存在");
		 }
    //添加文章内链 一篇文章不超过五个内链 多个关键词 只替换一个关键词
		List<ArticleThesaurus> listKeyWord = new LinkedList<ArticleThesaurus>();
		ArticleThesaurus at1 = new ArticleThesaurus("铜", "http://www.test.cn");
		ArticleThesaurus at2 = new ArticleThesaurus("铜价","http://www.test.cn/quote/");
		ArticleThesaurus at3 = new ArticleThesaurus("紫铜",
				"http://www.test.cn/product/tjthj_ct_zt/");
		ArticleThesaurus at4 = new ArticleThesaurus("黄铜",
				"http://www.test.cn/product/tjthj_ht/");
		ArticleThesaurus at5 = new ArticleThesaurus("白铜",
				"http://www.test.cn/product/tjthj_bt/");
		ArticleThesaurus at6 = new ArticleThesaurus("青铜",
				"http://www.test.cn/product/tjthj_qt/ ");
		listKeyWord.add(at1);
		listKeyWord.add(at2);
		listKeyWord.add(at3);
		listKeyWord.add(at4);
		listKeyWord.add(at5);
		listKeyWord.add(at6);

		String newStr ;
		newStr = afterFcStr;
		String article[] = afterFcStr.split("\\|");
		int successcount = 0;
		for (int i = 0; i < listKeyWord.size(); i++) {
			if (successcount == 5) {
				break;
			}
			String wordname = listKeyWord.get(i).getWord();
			Map<String, Integer> map = new LinkedHashMap<String, Integer>(); // 防止重复添加内链
			for (int j = 0; j < article.length; j++) {

				if (wordname.equals(article[j])) {
					if (map.get(wordname)== null) {
						map.put(wordname, 1);
						Arrays.fill(article, j, j + 1, "<a href='"
								+ listKeyWord.get(i).getUrl() + "'>" + wordname
								+ "</a>");
						successcount++;
					}
				}

			}
		}

		// for(int i=0;i<listKeyWord.size();i++){
		// String wordname=listKeyWord.get(i).getWord();
		// Integer count=words.get(wordname);
		// if(successcount==5){
		// break;
		// }
		// if(count!=null){
		// //System.out.println("该词频："+count);
		// newStr=newStr.replaceFirst(wordname,
		// "<a href='"+listKeyWord.get(i).getUrl()+"'>"+wordname+"</a>");
		// successcount++;
		// }else{
		// //System.out.println("该词频不存在");
		// }
		// }
		System.out.println("内链优化后的文章:" + Arrays.toString(article));
		StringBuilder StrArticle=new StringBuilder();
		for(int i=0;i<article.length;i++){
			StrArticle.append(article[i]);
		}
		//System.out.println("被优化多少个内链:"+successcount);
		//System.out.println("内链优化后的文章字符串:" + StrArticle);
		String endStr=StrArticle.toString();
		if(successcount==0){	//可能分词导致部分关键词没有匹配到 则采用绝对字符匹配
			for (int i = 0; i < listKeyWord.size(); i++) {
				//判断文章里的超链接数
				int acount=occurTimes(endStr,"href=");
				if(acount==5){
					break;
				}
				String wordname = listKeyWord.get(i).getWord();
				endStr=endStr.replaceFirst(wordname, "<a href='"
								+ listKeyWord.get(i).getUrl() + "'>" + wordname
								+ "</a>");
				
			}
		}
		
		//去除alt标签内的a内链
		System.out.println("内链优化后的文章字符串:"+endStr);
		Document doc = Jsoup.parseBodyFragment(endStr); // or Jsoup.parse(...);
		Elements images = doc.select("img");
		
		List<String> listAltStr=new LinkedList<String>();
		
		for(Element image : images){
		   // System.out.printf("%s:%s%n", image.attr("src"), image.attr("alt"));
			//System.out.println(image.attr("alt"));
			
			String altStr=image.attr("alt");
			Pattern p1 = Pattern.compile("</?(A|a)(\n|.)*?>");
			Matcher m1 = p.matcher(altStr);
			altStr = m1.replaceAll("");
			listAltStr.add(altStr);
			image.attr("alt", altStr);
			
			//System.out.println(altStr);
		}
		doc.select("img").listIterator(); 
		System.out.println("end内链优化后的文章字符串:" + doc.select("body").html());
	}
	
	/**
	 * 字符在字符串中出现的次数
	 * 
	 * @param string
	 * @param a
	 * @return
	 */
	public static int occurTimes(String string, String a) {
	    int pos = -2;
	    int n = 0;
	 
	    while (pos != -1) {
	        if (pos == -2) {
	            pos = -1;
	        }
	        pos = string.indexOf(a, pos + 1);
	        if (pos != -1) {
	            n++;
	        }
	    }
	    return n;
	}

	public static String ikAnalyzer(String str) {

		Reader input = new StringReader(str);
		// 智能分词关闭（对分词的精度影响很大）
		IKSegmenter iks = new IKSegmenter(input, true);
		Lexeme lexeme = null;
		StringBuilder sb = new StringBuilder();

		try {
			while ((lexeme = iks.next()) != null) {

				sb.append(lexeme.getLexemeText()).append("|");
			}
		} catch (IOException e) {
			e.printStackTrace();
		}

		return sb.toString();
	}

}

下面提供源码包下载 所用ik没有用到jar包 用到了源码包 方便调试和修改源代码

iteye传不上来
只能让大家去csdn下载了http://download.csdn.net/detail/koproblem/7575521