java 实现tfidf

tfidf 做自然语言理解的经常用。文档的作为权重计算，
许多初学者搞不清楚，权重计算特征选择。
针对文本分类而言 作为很简单的说明，特征选择都跟 类别有关比如 卡方 信息增益
而权重计算则与类别无关 比如 tfidf，tf
具体公式请参看 http://zh.wikipedia.org/wiki/TF-IDF
下面代码 写的比较粗陋。高手请避开。，，，，，
训练集可以上搜狗数据集上下载

写完以后感觉还是matlab 比较爽啊。。几行 解决java100多行的代码

 

public class TfIdfcomputor {
	static String path = "E:\\训练集\\train\\";
	static String  vsmpath = "E:\\训练集\\vsm\\vsm.txt";
	//static String path = "E:\\训练集\\SogouC.reduced\\Reduced";
   // static String path ="E:\\训练集\\SogouC.mini\\Sample";
	static Map<String, Map<String, Double>> DFdic = new HashMap();
	static HashMap<String, List<Map<String, Double>>> dic = new HashMap();
	static int DCOUNT;
	static HashMap<String, Double> idfDic = new HashMap();
	 static      Map<String, Map<String,Double>> TFIDFdic = new HashMap();
	// static Map<String,List<String>> Simpledic= new HashMap();
	public static void main(String[] args) throws IOException {

		TfIdfcomputor tf = new TfIdfcomputor();
		File[] Filelist = tf.readData(path);
		DCOUNT = Filelist.length;
		tf.readir(Filelist);
		System.out.println("DFdic");
	
		tf.computeIDF(dic);
        tf.ComputeTFIDF(); 
      /* for( String s :TFIDFdic.keySet())
        {
         Map map  = TFIDFdic.get(s);
         for(Object key :map.keySet() )
         {
        	 System.out.println("file "+s +"word "+ key+" tfidf "+map.get(key));
        	 
         }
        	
        
        }*/
        
   System.out.println("计算完毕开始输出");
   tf.toLibData();
}

	public void readir(File[] dir) throws IOException {

		File[] fileList = dir;
		for(File f :fileList){
			System.out.println(f.getPath());
		}

		for (int i = 0; i < fileList.length; i++) {
			File f = fileList[i];
			//System.out.println(f.getPath());
			String[] textword = cutWord(FiletoText(f.getPath()));
			Map tf = computeTf(textword);
			DFdic.put(f.getPath(), tf);
			addDic(f.getPath(), tf);
		}

		System.out.println("TF PROCESS IS OVER");

		System.out.println(dic.size());
		for (Object o : dic.keySet()) {
			System.out.println(o);
			List list = dic.get(o);
			for (Object l : list) {
				Map pair = (Map) l;
				for (Object p : pair.keySet()) {
				/*	System.out.println("key" + (String) o + "..."
							+ "filepath...." + p + "tf..." + pair.get(p));*/
				}

			}

		}

	}

	public String FiletoText(String path) throws IOException {
		File f = new File(path);
		BufferedReader br = new BufferedReader(new InputStreamReader(
				new FileInputStream(f), "GB2312"));
		String result = "";
		String temp;
		while ((temp = br.readLine()) != null) {
			result = result + temp;
		}
		br.close();
		return result;
	}

	public String[] cutWord(String text) throws IOException {
		MMAnalyzer analyzer = new MMAnalyzer();
		String temp = analyzer.segment(text, "|");
		//System.out.println(temp);
		String[] result = temp.split("\\|");
		/*
		 * for(String s :result ) { System.out.println(s); }
		 */
		return result;
	}

	public HashMap<String, Double> computeTf(String[] textword) {
		double size = textword.length;
		HashMap<String, Integer> map = new HashMap();
		for (int i = 0; i < textword.length; i++) {
			//System.out.println(textword[i]);
			if (map.containsKey(textword[i])) {
				Integer count = map.get(textword[i]);

				map.put(textword[i], count + 1);

			} else {
				map.put(textword[i], 1);
			}

		}

		HashMap<String, Double> result = new HashMap();
		for (Object o : map.keySet()) {
			Integer temp = map.get(o);

			//System.out.println((String) o + "count..." + temp);
			result.put((String) o, temp / size);
			//System.out.println((String) o + "tf..." + result.get(o));

		}

		return result;

	}

	public void addDic(String path, Map tf) {
		//System.out.println(",,,,,,,,,,,,,");
		for (Object o : tf.keySet()) {
			if (dic.containsKey((String) o)) {

				ArrayList list = (ArrayList) dic.get(o);
				HashMap map = new HashMap();
				map.put(path, (Double) tf.get((String) o));
				list.add(map);
				dic.put((String) o, list);
			} else {
				HashMap map = new HashMap();
				map.put(path, (Double) tf.get((String) o));
				ArrayList list = new ArrayList<Map<String, Double>>();
				list.add(map);
				dic.put((String) o, list);
			}

		}

	}

	public static File[] readData(String path) {
		int size = 0;
		File[] floderdir = new File(path).listFiles();
		ArrayList list = new ArrayList();
		for (File f : floderdir) {
			// size = size+(int)f.length();
			File[] childdir = f.listFiles();
			for (File file : childdir) {
				list.add(file);

			}
		}
		size = list.size();
		File[] fdir = new File[size];
		for (int i = 0; i < size; i++) {
			fdir[i] = (File) list.get(i);

		}

		return fdir;

	}

	public void computeIDF(HashMap<String, List<Map<String, Double>>> map) {

		for (String key : map.keySet()) {
			List list = map.get(key);
			double hasCount = (double) list.size();
			double idf = DCOUNT / hasCount;

			idfDic.put(key, idf);
		}

	}

	public void ComputeTFIDF() {
		
		for (String filepath : DFdic.keySet()) {
			Map filedic = DFdic.get(filepath);
			HashMap<String, Double> tfidfPair = tfidfPair = new HashMap();
			for (Object key : filedic.keySet()) {
				 
				double tf = (Double) filedic.get(key);

				double idf = idfDic.get((String) key);
				double tfidf = tf* Math.log(idf);
			//	System.out.println( key+"tf" + tf + "idf" + idf + "tfidf" + tfidf);
                tfidfPair.put((String) key, tfidf);
			}
		//	System.out.println(tfidfPair.size());
            TFIDFdic.put(filepath, tfidfPair);
		}

	}

	
	public void toLibData() throws IOException// 转化成libsvm格式；

	{
		int count = 0;
		// int size =dic.entrySet().size();
		List wordList = new ArrayList();
		for (String word : dic.keySet()) {
			wordList.add(word);
			System.out.println("worddic add" + word);
		}
		// System.out.println("total word is"+wordList.size());
		BufferedWriter bw = new BufferedWriter(
				new FileWriter(new File(vsmpath)));
		/*
		 * String [] wordList = new String[size]; int num=0; for(String word:
		 * dic.keySet()) { wordList[num]=word; num++;
		 * System.out.println("worddic add"+word); }
		 */
		String vsm = "";
		for (String filename : TFIDFdic.keySet()) {
			String lable = new File(filename).getParentFile().getName();
			Map map = TFIDFdic.get(filename);// 获取某片文章对应的tfidf
			vsm = vsm + lable + " ";
			for (int i = 0; i < wordList.size(); i++) {

				// System.out.println( "map.."+ map.size());

				// String temp =wordList[i];
				// System.out.println("temp"+ temp);
				String temp = (String) wordList.get(i);
				if (map.containsKey(temp)) {
					vsm = vsm + i + ":" + map.get(temp) + " ";
					// System.out.println(filename + "...." + temp + "...."+
					// map.get(temp) + "...");
				}
			}
			count++;
			vsm = vsm + "\n";
			bw.write(vsm);
			vsm = "";
			System.out.println("format" + "  " + count + " " + filename);
		}
		System.out.println("begin output");
		// BufferedWriter bw = new BufferedWriter(new FileWriter(new
		// File(vsmpath)));
		// bw.write(vsm);
		System.out.println(".............................");
		// System.out.println(vsm);
		bw.close();

	}
}