java模拟浏览器抓取网站信息和下载附件

package com.teamdev.jxbrowser.chromium.demoTest.Huhehaote;
import java.awt.BorderLayout;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.logging.Level;

import javax.swing.JFrame;
import javax.swing.WindowConstants;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.teamdev.jxbrowser.chromium.Browser;
import com.teamdev.jxbrowser.chromium.BrowserPreferences;
import com.teamdev.jxbrowser.chromium.LoggerProvider;
import com.teamdev.jxbrowser.chromium.events.FinishLoadingEvent;
import com.teamdev.jxbrowser.chromium.events.LoadAdapter;
import com.teamdev.jxbrowser.chromium.swing.BrowserView;
/**
* 爬取食药以及附件信息
* */
public class Test_Jsopu {


public static void main(String[] args) {
//模拟浏览器
LoggerProvider.getBrowserLogger().setLevel(Level.SEVERE);
LoggerProvider.getIPCLogger().setLevel(Level.SEVERE);
LoggerProvider.getChromiumProcessLogger().setLevel(Level.SEVERE);

        final Browser browser = new Browser();
        BrowserView browserView = new BrowserView(browser);
       
        BrowserPreferences preferences = browser.getPreferences();
        preferences.setImagesEnabled(false);
       
        JFrame frame = new JFrame();
        frame.setDefaultCloseOperation(WindowConstants.EXIT_ON_CLOSE);
        frame.add(browserView, BorderLayout.CENTER);
        frame.setSize(700, 500);
        frame.setLocationRelativeTo(null);
        frame.setVisible(true);
try {
//
String url="";
for(int i=0;i<7;i++){
if(i==0){
url="http://www.sda.gov.cn/WS01/CL1698/index.html";
}else{
url="http://www.sda.gov.cn/WS01/CL1698/index_"+i+".html";
}
search(browser,frame,url);
}

} catch (Exception e) {
e.printStackTrace();
}
}
//一次页面检索
public static void search(final Browser br ,JFrame frm,final String url) throws Exception {
new Thread().sleep(3000);
    Document doc = null;
    String attrUrl =url;
System.out.println("attrUrl" + attrUrl);
try{
      invokeAndWaitReady(br, new Runnable() {
    public void run() {
    br.loadURL(url);
    }
});
         } catch (Exception e2) {
e2.printStackTrace();
}
         String html = br.getHTML();
         doc = Jsoup.parse(html);
String text = doc.text();
System.out.println(text);
Elements els= doc.getElementsByAttributeValue("class", "ListColumnClass15");
System.out.println(els.size());
for (Element el : els) {
String title=el.getElementsByTag("a").get(0).text();
String href="http://www.sda.gov.cn/WS01"+el.getElementsByTag("a").attr("href").replace("..", "");
//System.out.println("title="+title);
System.out.println("href="+href);
try {
search2( br,frm,href,title);
} catch (Exception e) {
// TODO: handle exception
e.printStackTrace();
continue;
}
}
}

public static void search2(final Browser br ,JFrame frm,final String url,String title) throws Exception {
new Thread().sleep(3000);
    Document doc = null;
    String attrUrl =url;
System.out.println("attrUrl" + attrUrl);
     try{
    invokeAndWaitReady(br, new Runnable() {
    public void run() {
    br.loadURL(url);
    }
});
      } catch (Exception e2) {
e2.printStackTrace();
}
      String html = br.getHTML();
      doc = Jsoup.parse(html);
        
     
String  strs[]=doc.html().split("line-height:16px");
for(int i=0;i<strs.length;i++){
Thread.sleep(3000);
if(i!=0&&i!=strs.length){
String str="http://www.sda.gov.cn/"+strs[i].split("href=\"")[1].split("\">")[0];
String firlname=strs[i].split("href=\"")[1].split("\">")[1].split("</")[0];

System.out.println("title:"+title);
System.out.println("str:"+str);
System.out.println("firlname:"+firlname);
runDownLoad(title, str,firlname);

}
System.out.println("---------------------------------------");
}





}

public static void invokeAndWaitReady(Browser browser,Runnable runnable) {
    final CountDownLatch latch = new CountDownLatch(1);
    LoadAdapter listener = new LoadAdapter() {
        @Override
        public void onFinishLoadingFrame(FinishLoadingEvent event) {
            if (event.isMainFrame()) {
                latch.countDown();
            }
        }
    };
    browser.addLoadListener(listener);
    try {
        runnable.run();
        try {
            if (!latch.await(60, TimeUnit.SECONDS)) {
                //throw new RuntimeException(new TimeoutException());
            }
        } catch (InterruptedException ignore) {
        ignore.printStackTrace();
            Thread.currentThread().interrupt();
        }
    } finally {
        browser.removeLoadListener(listener);
    }
}


public static void runDownLoad(String title,String fileurl,String filename){ 
         // 构造URL 
        URL url; 
        try { 
            url = new URL(fileurl); 
             // 打开URL连接 
            URLConnection con = (URLConnection)url.openConnection(); 
             // 得到URL的输入流 
            InputStream input = con.getInputStream(); 
            // 设置数据缓冲 
            byte[] bs = new byte[1024 * 2]; 
            // 读取到的数据长度 
            int len; 
            // 输出的文件流保存图片至本地
            String path1 = "c:/aa/"+title;
            File f = new File(path1);
            f.mkdirs();
           // String hzm=fileurl.split("\\.")[fileurl.split("\\.").length-1];
            OutputStream os = new FileOutputStream(path1+"\\"+filename);
         
            while ((len = input.read(bs)) != -1) { 
            os.write(bs, 0, len); 
            } 
            os.close(); 
            input.close(); 
        } catch (MalformedURLException e) { 
            // TODO 自动生成的 catch 块 
            e.printStackTrace(); 
        } catch (IOException e) { 
            // TODO 自动生成的 catch 块 
            e.printStackTrace(); 
        } 
         
    }

}