简单爬虫的实现原理

import java.io.BufferedReader;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.net.URL;
import java.net.URLConnection;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class Test {
/**
* *
*
* @param args
*/
public static void main(String[] args) {
try {
URL url = new URL("http://www.sina.com.cn");
URLConnection conn = url.openConnection();
conn.setDoOutput(true);
InputStream in = null;
in = url.openStream();
String content = pipe(in, "gbk");
System.out.println(content);

String regex = "<(a|A)(\\s|\\w|\\.|/|#|\\?|\\&|\\=|%|:|,|;|-|\\)|\\(|\'|\")*>";
java.util.regex.Pattern pattern = Pattern.compile(regex);

Matcher matcher = pattern.matcher(content);

while(matcher.find()) {
System.out.println(matcher.group());
System.out.println(getURL(matcher.group()));
}



} catch (Exception e) {
e.printStackTrace();
}
}

public static String getURL(String content) {
String result = "";

int l = 0;
int s = 0;
int e = 0;

l = content.length();

content = content.toLowerCase();
s = content.indexOf("href=\"") + 6;

e = content.indexOf("\" ", s);
if(e == -1) {
e = content.indexOf("\">", s);
}

if(s > 0 && e > s) {
result = content.substring(s, e);
}

return result;
}


public static String pipe(InputStream in, String charset) throws IOException {
StringBuffer s = new StringBuffer();
if (charset == null || "".equals(charset)) {
charset = "utf-8";
}
String rLine = null;
BufferedReader bReader = new BufferedReader(new InputStreamReader(in,
charset));
PrintWriter pw = null;
FileOutputStream fo = new FileOutputStream("../index.html");
OutputStreamWriter writer = new OutputStreamWriter(fo, "utf-8");
pw = new PrintWriter(writer);
while ((rLine = bReader.readLine()) != null) {
String tmp_rLine = rLine;
int str_len = tmp_rLine.length();
if (str_len > 0) {
s.append(tmp_rLine);
pw.println(tmp_rLine);
pw.flush();
s.append("\n");
}
tmp_rLine = null;
}
in.close();
pw.close();
return s.toString();
}
}