nodejs 扒页面 api-docs

var continuation = require('../continuation');
var http = require("http");
var fs = require('fs');
var url = require('url');
var ps = require("path");
//var request = require("request");

function main_entry() {
  var $baseurl = "http://directwebremoting.org/dwr/javadoc/";
  var $basepath = "E:\\dwrdoc\\";
  var $lefttopurl = "http://directwebremoting.org/dwr/javadoc/overview-frame.html";
  var getLocal = function(str) {
  	  str = str.replace($baseurl , "");
  	  str = str.replace(/^\// , "");
  	  if(/^http\:/.test(str)) {
  	  	return $basepath;
  	  }
  	  var local = ps.resolve($basepath , str);
  	  var dirname = ps.dirname(local);
  	  console.log(local + "," + dirname);
  	  mkdir(dirname);
  	  return local;
  };
  var mkdir = function(str) {
  	 var arr = str.split(/\\/g);
  	 var step = "";
  	 while(arr.length) {
  	 	var shift = arr.shift();
  	 	step += shift + "\\";
  	 	if(!fs.existsSync(step)) {
  	 		fs.mkdirSync(step , 0777);
  	 	}
  	 }
  };
  var getLinks = function(str , $base) {
  	$base = $base || $baseurl;
  	var reg = /<a.*?href="(.+?)".*?>.*?<\/a>/gi , result;
  	var list = [];
  	while((result = reg.exec(str)) != null) {
  		var location = url.resolve($base , result[1]).replace(/\?.*$/ , "");
  		if(!/#.*$/.test(location)) {
  			list.push(location);
  		}
  	}
  	
  	var reg = /<frame.*?src="(.+?)".*?>/gi;
  	while((result = reg.exec(str)) != null) {
  		var location = url.resolve($base , result[1]).replace(/\?.*$/ , "");
  		if(!/^#/.test(location)) {
  			list.push(location);
  		}
  	}
  	/*
  	var reg = /<img.*?src="(.+?)".*?>/gi;
  	while((result = reg.exec(str)) != null) {
  		var location = url.resolve($base , result[1]).replace(/\?.*$/ , "");
  		var local = getLocal(location);
  		console.log(location + "............");
  		console.log(local + "...............");
  		http.request(location).pipe(fs.createWriteStream(local));
  	}
  	*/
  	return list;
  };
  
  var $httpget = function(url , next) {
  	//console.log('正在处理 ' + url);
  	if(url.indexOf("mailto:") == 0) {
  		return "";
  	}
  	var result = [];
  	http.get(url , cont(res));
  	res.on("data" , function(data) {
  		result.push(data.toString());
  	});
  	res.on("end" , cont(callback));
  	next(null , result.join(''));
  };
  
  $indexpath = $basepath + "index.html";
  
  if(fs.existsSync($indexpath)) {
  	  return;
  }
  
  $httpget($baseurl , obtain(result));
  
  fs.writeFileSync($indexpath , result , "utf-8");
  
  var listO = getLinks(result);
  
  listO.forEach(function(item) {
  	var localPathO = getLocal(item);
  	if(fs.existsSync(localPathO)) {
		return;
	}
	$httpget(item , obtain(resultO));
  	fs.writeFileSync(localPathO , resultO , "utf-8");
  });
  
  
  $httpget($lefttopurl , obtain(result1));
  
  var list1 = getLinks(result1);
  
  list1.forEach(function(item) {
  		var localPath = getLocal(item);
  		
  		if(fs.existsSync(localPath)) {
  			return;
  		}
  		
  		$httpget(item , obtain(result1));
  		fs.writeFileSync(localPath , result1 , "utf-8");
  		
  		var $basedir = url.resolve(item , "./");
  		
  		var list2 = getLinks(result1 , $basedir);
  		
  		list2.forEach(function(item1) {
  			
  			var localPath1 = getLocal(item1);
  			
  			if(fs.existsSync(localPath1)) {
	  			return;
	  		}
  			
  			$httpget(item1 , obtain(result2));
  			
  			
  			
  			fs.writeFileSync(localPath1 , result2 , 'utf-8');
  			
  			return;
  			/*
  			var list3 = getLinks(result2 , $basedir);
  			
  			list3.forEach(function(item2) {
  				var localPath2 = getLocal(item2);
  				if(fs.existsSync(localPath2)) {
		  			return;
		  		}
		  		$httpget(item2 , obtain(result3));
	  			fs.writeFileSync(localPath2 , result3 , 'utf-8');
  			});
  			*/
  			
  			
  		});
  		
  		
  });
};

var code = main_entry.toString();
var compiledCode = continuation.compile(code);
eval(compiledCode);
main_entry();