今天把myhttpd的连接处理模式由select改为epoll

Linux 2.6内核完全支持epoll.

epoll的IO效率不随FD数目增加而线性下降

传统的select/poll每次调用都会线性扫描全部的集合，导致效率呈现线性下降。

内核实现中epoll是根据每个fd上面的callback函数实现的。只有"活跃"的socket才会主

动的去调用 callback函数，其他idle状态socket则不会。

如果所有的socket基本上都是活跃的---比如一个高速LAN环境，过多使用epoll,效率

相比还有稍微的下降。

但是一旦使用idle connections模拟WAN环境,epoll的效率就远在select/poll之上了。

 

poll的执行分三部分：
1.将用户传入的pollfd数组拷贝到内核空间，因为拷贝操作和数组长度相关，时间

上这是一个O（n）操作

2.
查询每个文件描述符对应设备的状态，如果该设备尚未就绪，则在该设备的等

待队列中加入一项并继续查询下一设备的状态。
查询完所有设备后如果没有一个设备就绪，这时则需要挂起当前进程等待，直

到设备就绪或者超时。
设备就绪后进程被通知继续运行，这时再次遍历所有设备，以查找就绪设备。
这一步因为两次遍历所有设备，时间复杂度也是O（n），这里面不包括等待时

间。。 

3.
将获得的数据传送到用户空间并执行释放内存和剥离等待队列等善后工作，向

用户空间拷贝数据与剥离等待队列等操作的的时间复杂度同样是O（n）。

epoll用到的所有函数都是在头文件sys/epoll.h中声明的，下面简要说明所用到的数

据结构和函数：
所用到的数据结构
typedef union epoll_data {
                void *ptr;
                int fd;
                __uint32_t u32;
                __uint64_t u64;
        } epoll_data_t;

        struct epoll_event {
                __uint32_t events;      /* Epoll events */
                epoll_data_t data;      /* User data variable */
        };
结构体epoll_event 被用于注册所感兴趣的事件和回传所发生待处理的事件.

其中epoll_data 联合体用来保存触发事件的某个文件描述符相关的数据.

例如一个client连接到服务器，服务器通过调用accept函数可以得到于这个client对

应的socket文件描述符，可以把这文件描述符赋给epoll_data的fd字段以便后面的读

写操作在这个文件描述符上进行。epoll_event 结构体的events字段是表示感兴趣的

 

出处张沈鹏的javaeye博客

epoll学习笔记

http://zsp.iteye.com/blog/146850

 

——————注:上面的内容皆是摘录————

 

之前的select模式的代码

/* server.c */
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <netinet/in.h>

#include "wrap.h"
#include "httpprocesser.h"

#define MAXLINE 800
#define SERV_PORT 7080
#define BACKLOG 200

int main(int argc, char **argv)
{
	int i, maxi, maxfd, listenfd, connfd, sockfd;
	int nready, client[FD_SETSIZE];
	ssize_t n;
	fd_set rset, allset;
	char buf[MAXLINE];
	char str[INET_ADDRSTRLEN];
	socklen_t cliaddr_len;
	struct sockaddr_in	cliaddr, servaddr;

	listenfd = my_socket(AF_INET, SOCK_STREAM, 0);

	bzero(&servaddr, sizeof(servaddr));
	servaddr.sin_family      = AF_INET;
	servaddr.sin_addr.s_addr = htonl(INADDR_ANY);
	servaddr.sin_port        = htons(SERV_PORT);

	my_bind(listenfd, (struct sockaddr *)&servaddr, sizeof(servaddr));

	my_listen(listenfd, BACKLOG);

	maxfd = listenfd;		/* initialize */
	maxi = -1;			/* index into client[] array */
	for (i = 0; i < FD_SETSIZE; i++)
		client[i] = -1;	/* -1 indicates available entry */
	FD_ZERO(&allset);
	FD_SET(listenfd, &allset);

	for ( ; ; ) {
		rset = allset;	/* structure assignment */
		nready = select(maxfd+1, &rset, NULL, NULL, NULL);
		if (nready < 0)
			perr_exit("select error");

		if (FD_ISSET(listenfd, &rset)) { /* new client connection */
			cliaddr_len = sizeof(cliaddr);
			connfd = my_accept(listenfd, (struct sockaddr *)&cliaddr, &cliaddr_len);

			printf("received from %s at PORT %d\n",
			       inet_ntop(AF_INET, &cliaddr.sin_addr, str, sizeof(str)),
			       ntohs(cliaddr.sin_port));

			for (i = 0; i < FD_SETSIZE; i++)
				if (client[i] < 0) {
					client[i] = connfd; /* save descriptor */
					break;
				}
			if (i == FD_SETSIZE) {
				fputs("too many clients\n", stderr);
				exit(1);
			}

			FD_SET(connfd, &allset);	/* add new descriptor to set */
			if (connfd > maxfd)
				maxfd = connfd; /* for select */
			if (i > maxi)
				maxi = i;	/* max index in client[] array */

			if (--nready == 0)
				continue;	/* no more readable descriptors */
		}

		for (i = 0; i <= maxi; i++) {	/* check all clients for data */
			if ( (sockfd = client[i]) < 0)
				continue;
			if (FD_ISSET(sockfd, &rset)) {
				if ( (n = my_read(sockfd, buf, MAXLINE)) > 0) {
                    struct http_msg msg;
                    parse_msg(buf, &msg);
                    printf("%s\n", msg.response);
					my_write(sockfd, buf, n);
				}
					/* connection closed by client */
					my_close(sockfd);
					FD_CLR(sockfd, &allset);
					client[i] = -1;
				if (--nready == 0)
					break;	/* no more readable descriptors */
			}
		}
	}
}

 现在使用的epoll模式的代码

这部分的代码由http://blog.csdn.net/mote_li/archive/2004/12/08/209450.aspx 修改而来

另外还增加了基于pthread库实现的工作队列方式的线程池。

 

#ifndef __myhttpd_h
#define __myhttpd_h

#include <sys/types.h>
#include <sys/socket.h>
#include <sys/epoll.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <fcntl.h>
#include <unistd.h>
#include <stdlib.h>
#include <errno.h>
#include <stdio.h>
#include <strings.h>

#include <pthread.h>

#include "wrap.h"
#include "task.h"


#define OPEN_MAX 100
#define LISTENQ 20
#define INFTIM 1000

#define LOCAL_IP "127.0.0.1" /* 修改为自己本地机器就可以测试了 */
#define SERV_PORT 5555

extern pthread_mutex_t mutex; /* 线程安全使用 */
extern pthread_cond_t cond1; /* 线程条件等待使用 */

extern struct task *readhead ,
                                *readtail ,
                                *writehead ;
extern struct epoll_event ev, events[20];

extern int epfd;


void setnonblocking(int sock)
{
  int opts;

  opts = fcntl(sock, F_GETFL);
  if(opts<0)
   {
     perror("fcntl(sock,GETFL)");
     exit(1); /* 其实这样做不怎么好, 最好自己做好出错处理的工作, 不光是进程退出就可以了 */
   }

  if(fcntl(sock, F_SETFL, opts | O_NONBLOCK)<0)
   {
    perror("fcntl(sock,SETFL,opts)");
    exit(1);
   }
}

int main()
{
  int i, maxi, listenfd, connfd, sockfd, nfds;
  socklen_t clilen;
  pthread_t tid1,tid2;
  struct task *new_task=NULL;
  struct user_data *rdata=NULL;

  readhead = readtail = writehead = NULL;

  /* initialize the thread pool */
  pthread_mutex_init(&mutex, NULL);
  pthread_cond_init(&cond1, NULL);

   /* 创建线程, 最好做好错误处理工作, 自己也比较懒. 真正作东西千万别这样噢! */
  pthread_create(&tid1, NULL, readtask, NULL);
  pthread_create(&tid2, NULL, readtask, NULL);

   /* 生成用于处理accept的epoll专用的文件描述符
    * 以前从没用过
    *

Create a new epoll file descriptor by requesting the kernel allocate an event backing store dimensioned
[n. 尺寸, 尺度, 维(数), 度(数), 元] for size descriptors.
 The size is not the maximum size of the backing store but just a hint to the kernel about
how to dimension internal structures.
The returned file descriptor will be used for all the subsequent calls to the epoll interface.
 The file descriptor returned by epoll_create must be closed by using POSIX::close.

When successful, epoll_create returns a positive integer identifying the descriptor. When an error occurs,
 epoll_create returns -1 and errno is set appropriately.

    *
    */
  epfd = epoll_create(256);

  struct sockaddr_in clientaddr;
  struct sockaddr_in serveraddr;

  listenfd = my_socket(AF_INET, SOCK_STREAM, 0);

  //把socket设置为非阻塞方式

  setnonblocking(listenfd);

   //设置与要处理的事件相关的文件描述符

  ev.data.fd = listenfd;

   //设置要处理的事件类型

  ev.events = EPOLLIN | EPOLLET;

  /*注册epoll事件

Control an epoll descriptor, $epfd, by requesting the operation op be performed on the target file descriptor, fd.

$epfd is an epoll descriptor returned from epoll_create.
$op is one of EPOLL_CTL_ADD, EPOLL_CTL_MOD or EPOLL_CTL_DEL.
$fd is the file desciptor to be watched.
$eventmask is a bitmask of events defined by EPOLLIN, EPOLLOUT, etc.

When successful, epoll_ctl returns 0. When an error occurs, epoll_ctl returns -1 and errno is set appropriately.
   */
  epoll_ctl(epfd, EPOLL_CTL_ADD, listenfd, &ev);

  bzero(&serveraddr, sizeof(serveraddr));
  serveraddr.sin_family = AF_INET;

  char *local_addr = LOCAL_IP;
  inet_aton(local_addr, &(serveraddr.sin_addr));

  serveraddr.sin_port = htons(SERV_PORT);
  my_bind(listenfd, (struct sockaddr *)&serveraddr, sizeof(serveraddr));
  my_listen(listenfd, LISTENQ);

  maxi = 0;
  for ( ; ; )
   {
     /*等待epoll事件的发生

Wait for events on the epoll file descriptor $epfd.

$epfd is an epoll descriptor returned from epoll_create.
$maxevents is an integer specifying the maximum number of events to be returned.
$timeout is a timeout, in milliseconds

When successful, epoll_wait returns a reference to an array of events. Each event is a two element array,
the first element being the file descriptor which triggered the event,
and the second is the mask of event types triggered.
For example, if epoll_wait returned the following data structure:

     */
     nfds = epoll_wait(epfd, events, 20, 500);

      //处理所发生的所有事件

     for(i = 0; i < nfds; ++i)
      {
       if(events[i].data.fd == listenfd)
         {
         connfd = my_accept(listenfd, (struct sockaddr *)&clientaddr, &clilen);
         if(connfd < 0)
           {
          perror("connfd<0");
          exit(1);
           }

         setnonblocking(connfd);

         char *str = inet_ntoa(clientaddr.sin_addr);

         printf("connect_from >> %s \n", str);

         ev.data.fd = connfd; //设置用于读操作的文件描述符

         ev.events = EPOLLIN | EPOLLET; //设置用于注测的读操作事件


           //注册ev

         epoll_ctl(epfd, EPOLL_CTL_ADD, connfd, &ev);
         }
       else if (events[i].events & EPOLLIN)
         {
          printf("reading!\n");

          if ((sockfd = events[i].data.fd) < 0)
            continue;

          new_task = (struct task *)malloc(sizeof(struct task));
          new_task->fd = sockfd;
          new_task->next = NULL;

          pthread_mutex_lock(&mutex); //添加新的读任务


          if(readhead == NULL)
             {
             readhead = new_task;
             readtail = new_task;
             }
          else
             {
             readtail->next = new_task;
             readtail = new_task;
             }

              //唤醒所有等待cond1条件的线程

           pthread_cond_broadcast(&cond1);
           pthread_mutex_unlock(&mutex);
           }
        else if (events[i].events & EPOLLOUT)
          {
           rdata = (struct user_data *)events[i].data.ptr;
           sockfd = rdata->fd;

         printf("thread.%u Write data fd.%d len.%d data.%s \n"
        , (uint32_t)pthread_self(), sockfd, rdata->n_size, rdata->line);

           my_write(sockfd, rdata->line, rdata->n_size);
           my_close(sockfd);

           free(rdata);

           ev.data.fd = sockfd; //设置用于读操作的文件描述符

           ev.events = EPOLLIN | EPOLLET; //设置用于注测的读操作事件


              //修改sockfd上要处理的事件为EPOLIN

           epoll_ctl(epfd, EPOLL_CTL_MOD, sockfd, &ev);
           }
       }
   }
}


#endif