头文字D，Spider

贴一段我用D写的Spider的代码。
越来越爱D了。

import std.stdio;
import std.string;
import std.conv;
import std.socket;
import std.socketstream;
import std.stream;
import std.regexp;
import std.thread;
import std.c.time;

char[][] g_queue;
int g_task_amount=0;
const char[] homepage="http://mobile.younet.com/";
const ushort max_thread=20;

alias std.string.find strfind;

char[] getHTML(char[] url){
	
	char[] domain,html;
	ubyte[1024] buf;
	ushort port=80;
	if(!isURL(url))return null;
	int i=strfind(url,"://")+3;
	url=url[i..$];
	int j=strfind(url,":");
	int e=strfind(url,"/");
	if(e<0){
		e=url.length;
	}
	if(j>0){
		port=toUshort(url[j+1..e]);
		domain=url[0..j];
	}
	else{
		domain=url[0..e];
	}

	if(e==url.length){
		url="/";
	}
	else{
		url=url[e..$];
	}

	debug(younet){
		writefln(toString(port) ~"  "~ domain ~"  "~ url);
	}

	Socket sock=new TcpSocket(new InternetAddress(domain,port));
	Stream ss=new SocketStream(sock);
	ss.writeString("GET " ~ url ~ " HTTP/1.0\r\n"
		 "Host: " ~ domain ~ "\r\n"
		 "Connection: close\r\n"
		 "Referer: http://" ~ domain ~ url ~ "\r\n"
		 "\r\n\r\n\r\n\r\n");
	int recv_amount=ss.read(buf);
	while(recv_amount>0){
		html ~= cast(char[])buf[0..recv_amount];
		recv_amount=ss.read(buf);
	}

	ss.close();
	sock.close();
	
	char[][] mc=RegExp("(URL=|Location: )(.*?)[\"\r]").match(html);
	if(mc.length==3){
		char[] new_location=mc[2];
		html=getHTML(new_location);
		return html;
	}
	int start_pos=strfind(html,"\r\n\r\n") ;
	
	html=html[start_pos+4 .. $];
	return html[0..$];
}

int crawl(void * p){
	while(true){
		char[] url,html;
		synchronized{
			if(g_queue.length==0)
				sleep(1);
			if(g_queue.length==0)
				break;
			url=g_queue[0];
			
			writefln("begin:" ~ url);
			if(g_queue.length>0)
				g_queue=g_queue[1..$];	
		}
		try{
			html=getHTML(url);
		}
		catch(Exception ex){
			synchronized{
				if(g_task_amount>0)
					g_task_amount-=1;
			}
			writefln(ex);
			writefln("failed:" ~ url);
			writefln("remains" ~ toString(g_task_amount));
			continue;
		}
		debug(younet){
			printf(toStringz("!!!" ~ html[0..200]));
		}
		if(strfind(url,"files/list")<0){
			synchronized{
				foreach(m;RegExp("files/list_\\d+\\.html").search(html)){
					g_queue ~= homepage ~ m.match(0);
					g_task_amount+=1;
				}
				g_task_amount-=1;
			}
			writefln("done:" ~ url);
			debug(younet){
				writefln(g_queue);
			}
		}
		else{
			writefln("done:" ~ url);
			synchronized{
				g_task_amount-=1;
				writefln("remains" ~ toString(g_task_amount));
			}
		}
		
	}
	return 1;
}
int main(char[][] args){
	//writefln("Hello");
	g_queue ~= homepage;
	g_task_amount+=1;
	Thread[] tds;
	for(int i=0;i<max_thread;i++){
		Thread t=new Thread(&crawl,null);
		t.start();
		tds ~= t;
	}
	sleep(5);
	while(true){
		sleep(1);
		if(g_task_amount<=0)break;
	}
	return 0;
}

评论

2 楼 ideage 2008-10-04  

缺少引用的库了.
试试
dmd Spider.d ws2_32.lib

1 楼 reddust 2008-09-23  

编译提示如下错误：

OPTLINK (R) for Win32  Release 8.00.1
Copyright (C) Digital Mars 1989-2004  All rights reserved.
D:\d\dmd\bin\..\lib\phobos.lib(socket)
Error 42: Symbol Undefined _getprotobyname@4
D:\d\dmd\bin\..\lib\phobos.lib(socket)
Error 42: Symbol Undefined _getprotobynumber@4
D:\d\dmd\bin\..\lib\phobos.lib(socket)
Error 42: Symbol Undefined _getservbyname@8
D:\d\dmd\bin\..\lib\phobos.lib(socket)
Error 42: Symbol Undefined _getservbyport@8
D:\d\dmd\bin\..\lib\phobos.lib(socket)
Error 42: Symbol Undefined _WSAGetLastError@0
D:\d\dmd\bin\..\lib\phobos.lib(socket)
Error 42: Symbol Undefined _gethostbyname@4
D:\d\dmd\bin\..\lib\phobos.lib(socket)
Error 42: Symbol Undefined _gethostbyaddr@12
D:\d\dmd\bin\..\lib\phobos.lib(socket)
Error 42: Symbol Undefined _inet_addr@4
D:\d\dmd\bin\..\lib\phobos.lib(socket)
Error 42: Symbol Undefined _inet_ntoa@4
D:\d\dmd\bin\..\lib\phobos.lib(socket)
Error 42: Symbol Undefined _ioctlsocket@12
D:\d\dmd\bin\..\lib\phobos.lib(socket)
Error 42: Symbol Undefined _getsockopt@20
D:\d\dmd\bin\..\lib\phobos.lib(socket)
Error 42: Symbol Undefined _bind@12
D:\d\dmd\bin\..\lib\phobos.lib(socket)
Error 42: Symbol Undefined _connect@12
D:\d\dmd\bin\..\lib\phobos.lib(socket)
Error 42: Symbol Undefined _listen@8
D:\d\dmd\bin\..\lib\phobos.lib(socket)
Error 42: Symbol Undefined _accept@12
D:\d\dmd\bin\..\lib\phobos.lib(socket)
Error 42: Symbol Undefined _closesocket@4
D:\d\dmd\bin\..\lib\phobos.lib(socket)
Error 42: Symbol Undefined _shutdown@8
D:\d\dmd\bin\..\lib\phobos.lib(socket)
Error 42: Symbol Undefined _getpeername@12
D:\d\dmd\bin\..\lib\phobos.lib(socket)
Error 42: Symbol Undefined _getsockname@12
D:\d\dmd\bin\..\lib\phobos.lib(socket)
Error 42: Symbol Undefined _send@16
D:\d\dmd\bin\..\lib\phobos.lib(socket)
Error 42: Symbol Undefined _sendto@24
D:\d\dmd\bin\..\lib\phobos.lib(socket)
Error 42: Symbol Undefined _recv@16
D:\d\dmd\bin\..\lib\phobos.lib(socket)
Error 42: Symbol Undefined _recvfrom@24
D:\d\dmd\bin\..\lib\phobos.lib(socket)
Error 42: Symbol Undefined _setsockopt@20
D:\d\dmd\bin\..\lib\phobos.lib(socket)
Error 42: Symbol Undefined _socket@12
D:\d\dmd\bin\..\lib\phobos.lib(socket)
Error 42: Symbol Undefined _WSAStartup@8
D:\d\dmd\bin\..\lib\phobos.lib(socket)
Error 42: Symbol Undefined _WSACleanup@0
--- errorlevel 27