TF-IDF[2]

package com.sap.research.semantic;

import java.io.File;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;

import javax.print.attribute.standard.OutputDeviceAssigned;

import cc.mallet.types.Vector;
import cc.mallet.util.CommandOption.Set;

import com.sap.research.util.Pair;

public class SemanticInterpretor {
	private InvertedIndex invertedIndex;

	public SemanticInterpretor(InvertedIndex invertedIndex) {
		this.invertedIndex = invertedIndex;
	}

	// vector representation for a document based on concepts in DB
	public HashMap<String, Double> vectorRepreforDoc(File file) {
		HashMap<String, Pair<String, Integer>> docVec = invertedIndex
				.termCount(file);
		// vector representation for a document, only term frequency considered
		// HashMap<term, frequency>
		HashMap<String, Integer> vec = new HashMap<String, Integer>();
		for (Map.Entry<String, Pair<String, Integer>> entry : docVec.entrySet()) {
			vec.put(entry.getKey(), entry.getValue().second);
		}

		// semantical vector representation for a document
		// HashMap<concept, relatedness>
		HashMap<String, Double> semanticVec = new HashMap<String, Double>();
		// vector representation based on concepts
		HashMap<String, ArrayList<Pair<String, Double>>> indices = invertedIndex
				.getInvertedIndicesDouble();
		for (Map.Entry<String, Integer> entry : vec.entrySet()) {
			if (indices.keySet().contains(entry.getKey())) {
				// if certain word is contained in the invertedIndex
				ArrayList<Pair<String, Double>> pairList = indices.get(entry
						.getKey());
				for (Pair<String, Double> pair : pairList) {
					double relatedness = entry.getValue() * pair.second;
					if (!semanticVec.keySet().contains(pair.first)) {
						semanticVec.put(pair.first, relatedness);
					} else {
						semanticVec.put(pair.first, semanticVec.get(pair.first)
								+ relatedness);
					}

				}
			} else {
				// if certain word is not contained in the invertedIndex
				// simply ignore?
			}
		}
		output(semanticVec);
		return semanticVec;
	}

	private void output(HashMap<String, Double> semanticVec) {
		System.out.println(semanticVec);
	}

	public double semanticalSimilarity(HashMap<String, Double> docVec1,
			HashMap<String, Double> docVec2) {
		ArrayList<String> base = new ArrayList<String>();
		for (Map.Entry<String, Double> entry : docVec1.entrySet()) {
			base.add(entry.getKey());
		}
		for (Map.Entry<String, Double> entry : docVec2.entrySet()) {
			if (!base.contains(entry.getKey())) {
				base.add(entry.getKey());
			}
		}

		HashMap<String, Double> docVec1Std = new HashMap<String, Double>();
		ArrayList<Double> docVec1Double = new ArrayList<Double>();
		HashMap<String, Double> docVec2Std = new HashMap<String, Double>();
		ArrayList<Double> docVec2Double = new ArrayList<Double>();

		for (String string : base) {
			if (docVec1.keySet().contains(string)) {
				docVec1Std.put(string, docVec1.get(string));
				docVec1Double.add(docVec1.get(string));
			} else {
				docVec1Std.put(string, 0.0);
				docVec1Double.add(0.0);
			}

			if (docVec2.keySet().contains(string)) {
				docVec2Std.put(string, docVec2.get(string));
				docVec2Double.add(docVec2.get(string));
			} else {
				docVec2Std.put(string, 0.0);
				docVec2Double.add(0.0);
			}
		}
		
		output(docVec1Std);
		output(docVec2Std);

		double similarity = 0.0;
		if (docVec1Double.size() == docVec2Double.size()) {
			double prod = 0.0;
			double doc1Sqr = 0.0;
			double doc2Sqr = 0.0;
			for (int i = 0; i < docVec1Double.size(); i++) {
				prod += docVec1Double.get(i) * docVec2Double.get(i);
				doc1Sqr += docVec1Double.get(i) * docVec1Double.get(i);
				doc2Sqr += docVec2Double.get(i) * docVec2Double.get(i);
			}

			similarity = prod / (Math.sqrt(doc1Sqr) * Math.sqrt(doc2Sqr));
		} else {
			System.err.println("cannot compare vectors of different length!");
		}
		return similarity;
	}

	public double computeSemanticRelatedness(File fileA, File fileB) {
		// invertedIndex.addIndices(dirPath, filePath, recursive)
		return semanticalSimilarity(vectorRepreforDoc(fileA),
				vectorRepreforDoc(fileB));
	}

	public static void main(String[] args) {
		String propertyFilePath = "settings/html.properties";
		InvertedIndex invertedIndex = new InvertedIndex(propertyFilePath);
		invertedIndex.addIndices("test/input/html_20", "html", true);
		SemanticInterpretor interpretor = new SemanticInterpretor(invertedIndex);
		File fileA = new File("test/input/html/topic1.html");
		File fileB = new File("test/input/html/topic2.html");
		double similarity = interpretor
				.computeSemanticRelatedness(fileA, fileB);
		System.out.println("Semantic relatedness of " + fileA.getName()
				+ " and " + fileB.getName() + " is: " + similarity);

	}

}