set 去重

转载  Java合并两个文件，Set去重。 收藏

package com.joyhoo.words;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Set;

public class HashSetInFile
{


//simpleFile、insertFile为两个输入文件，outFile为输出文件
public void readFile(String simpleFile,String insertFile,String outFile)
{
  Set set = new HashSet();
 
  try
  {
   String line="";

   FileInputStream fisf = new FileInputStream(simpleFile);
   InputStreamReader isrf = new InputStreamReader(fisf, "UTF8");
   BufferedReader br =new BufferedReader(isrf);
  
   while((line=br.readLine())!=null ){

    if(!line.contains(" "))
     set.add(line);
   }
   br.close();
   System.out.println(set.size());
  
   FileInputStream fis = new FileInputStream(insertFile);
   InputStreamReader isr = new InputStreamReader(fis, "UTF8");
   BufferedReader inbr=new BufferedReader(isr);

   String temp = "";
   while((temp = inbr.readLine()) != null){
    if(!set.contains(temp) ){
     set.add(temp);
    }
   }
   inbr.close();
  
   FileOutputStream fos = new FileOutputStream(outFile);
   BufferedWriter out = new BufferedWriter(new OutputStreamWriter(fos,"UTF8"));
   Iterator it = set.iterator();
  
   while(it.hasNext()){
   
    String element = (String)it.next();

    out.write(element);
    out.newLine();
   
   }
   out.close();
  
  }catch(Exception e){
   System.out.println("readFile errors:" + e.getMessage());
  }

}

public static void main(String[] args)
{
  HashSetInFile hf = new HashSetInFile();

  String simple = "D:\\tmp\\simplexu8.txt";

  String insert = "D:\\tmp\\sogou.txt";
 
  String outfile = "D:\\tmp\\simplexu81.txt";
 
  hf.readFile(simple, insert, outfile);
}

}