<cloud>iscsi kernel模块-网络部分

接口就是ioctl  没什么好说的
定义了一个上下文的 static DECLARE_MUTEX(ioctl_sem); 用来限制一次只能调用一个ioctl
主要分为

Session / Conn/ volume 增加 减少
Param的配置修改

先看临时一下 iet_socket_bind

struct iscsi_conn {
 struct list_head list;   /* list entry in session list */
 struct iscsi_session *session;  /* owning session */

 u16 cid;
 unsigned long state;

 u32 stat_sn;
 u32 exp_stat_sn;

 int hdigest_type;
 int ddigest_type;

 struct list_head poll_list;

 struct file *file;
/*通用 BSD socket 关键是socket->sk<L2 L3>*/
 struct socket *sock;
//...

};

 

static void iet_socket_bind(struct iscsi_conn *conn)
{
	int opt = 1;
	mm_segment_t oldfs;
	struct iscsi_session *session = conn->session;
	struct iscsi_target *target = session->target;

	dprintk(D_GENERIC, "%llu\n", (unsigned long long) session->sid);
	/*userspace传递的是struct file得到对应的sock */
	conn->sock = SOCKET_I(conn->file->f_dentry->d_inode);
	conn->sock->sk->sk_user_data = conn;
/*下面iscsi用了自己socket线程处理唤醒 方法，和kernel独立开*/
	write_lock_bh(&conn->sock->sk->sk_callback_lock);
	/*sk_state_change: callback to indicate change in the state of the sock*/
	target->nthread_info.old_state_change = conn->sock->sk->sk_state_change;
	conn->sock->sk->sk_state_change = iet_state_change;
	/*sk_data_ready: callback to indicate there is data to be processed*/
	target->nthread_info.old_data_ready = conn->sock->sk->sk_data_ready;
	conn->sock->sk->sk_data_ready = iet_data_ready;

	target->nthread_info.old_write_space = conn->sock->sk->sk_write_space;
	conn->sock->sk->sk_write_space = iet_write_space;
	write_unlock_bh(&conn->sock->sk->sk_callback_lock);
	/*没什么好说的 其实应该用int kernel_setsockopt(struct socket *sock, int level, int optname,
			char *optval, unsigned int optlen)*/
	oldfs = get_fs();
	set_fs(get_ds());
	conn->sock->ops->setsockopt(conn->sock, SOL_TCP, TCP_NODELAY, (void *)&opt, sizeof(opt));
	set_fs(oldfs);
}

 

非常著名的函数：  优化的源泉
void sock_init_data(struct socket *sock, struct sock *sk)
初始化 sock 结构中和IP相关的可扩展接口部分

void sock_init_data(struct socket *sock, struct sock *sk)
{
//...
 sk->sk_state_change = sock_def_wakeup;
/*主要执行 wake_up_interruptible_sync_poll  唤醒socket_wq 的线程 固定CPU*/
 sk->sk_data_ready = sock_def_readable;
 sk->sk_write_space = sock_def_write_space;
 sk->sk_error_report = sock_def_error_report;
 sk->sk_destruct  = sock_def_destruct;
//...

}
struct socket_wq {
 wait_queue_head_t wait;
 struct fasync_struct *fasync_list;
 struct rcu_head  rcu;
} ____cacheline_aligned_in_smp;

static void sock_def_wakeup(struct sock *sk)
{
 struct socket_wq *wq;

 rcu_read_lock();
 wq = rcu_dereference(sk->sk_wq);
/*如果有等待数据的进程就返回ture*/
 if (wq_has_sleeper(wq))  
 wake_up_interruptible_all(&wq->wait);
 rcu_read_unlock();
}

 
static void iet_data_ready(struct sock *sk, int len)
{
 struct iscsi_conn *conn = sk->sk_user_data;
 struct iscsi_target *target = conn->session->target;
 /*就是调用 wake_up_process(info->task); struct task_struct *task; 通知他们起来处理*/
 nthread_wakeup(target);
 target->nthread_info.old_data_ready(sk, len);
}

static int is_data_available(struct iscsi_conn *conn)
{
 int avail, res;
 mm_segment_t oldfs;
 struct socket *sock = conn->sock;

 oldfs = get_fs();
 set_fs(get_ds());
 /*这里作者很懒啊 ，直接通过FIONREAD 来得到数据情况*/
 res = sock->ops->ioctl(sock, SIOCINQ, (unsigned long) &avail);
 set_fs(oldfs);
 return (res >= 0) ? avail : res;
}

下面看看 add_conn
static int add_conn(struct iscsi_target *target, unsigned long ptr)
{
//...
 err = copy_from_user(&info, (void *) ptr, sizeof(info));
 if (err)
  return -EFAULT;
 /*找到对应会话*/
 session = session_lookup(target, info.sid);
 if (!session)
  return -ENOENT;
 
 return conn_add(session, &info);
}

int conn_add(struct iscsi_session *session, struct conn_info *info)
{
//...
/*如果有了 就删掉 重新生成*/
 conn = conn_lookup(session, info->cid);
 if (conn)
  conn_close(conn);

 err = iet_conn_alloc(session, info);
 if (!err && conn)
  err = -EEXIST;

 return err;
}

iet_conn_alloc 初始化iscsi_conn 结构体 ，所有的核心操作都和其有关
static int iet_conn_alloc(struct iscsi_session *session, struct conn_info *info)
{
 struct iscsi_conn *conn;

 dprintk(D_SETUP, "%#Lx:%u\n", (unsigned long long) session->sid, info->cid);

 conn = kzalloc(sizeof(*conn), GFP_KERNEL);
 if (!conn)
  return -ENOMEM;

//...
 spin_lock_init(&conn->list_lock);
 atomic_set(&conn->nr_cmnds, 0);
 atomic_set(&conn->nr_busy_cmnds, 0);
 INIT_LIST_HEAD(&conn->pdu_list);
 INIT_LIST_HEAD(&conn->write_list);
 INIT_LIST_HEAD(&conn->poll_list);
 init_timer(&conn->nop_timer); /*连接超时器*/

 list_add(&conn->list, &session->conn_list);

 set_bit(CONN_ACTIVE, &conn->state);
 
 /*用户空间建立的连接传下来处理 */
 conn->file = fget(info->fd);
 iet_socket_bind(conn);
 /*把自己加到该session 的激活链接中*/
 list_add(&conn->poll_list, &session->target->nthread_info.active_conns);

 nthread_wakeup(conn->session->target);

 return 0;
}

 

add_session()->session_add()-> iet_session_alloc

和上面类似 不过建立了一个 命令的hash 数组链表

{
//...
for (i = 0; i < ARRAY_SIZE(session->cmnd_hash); i++)
  INIT_LIST_HEAD(&session->cmnd_hash[i]);

 spin_lock_init(&session->ua_hash_lock);
 for (i = 0; i < ARRAY_SIZE(session->ua_hash); i++)
  INIT_LIST_HEAD(&session->ua_hash[i]);

 list_for_each_entry(vol, &target->volumes, list)
  /* power-on, reset, or bus device reset occurred */
  ua_establish_for_session(session, vol->lun, 0x29, 0x0);
//...

}

然后来看看干活的进程

static long ioctl(struct file *file, unsigned int cmd, unsigned long arg)
{
//...
  if (cmd == ADD_TARGET) {
   err = add_target(arg);
   goto done;
  }
//...
 
}

add_target()->通过用户入参target_info 初始化
target_add()->必要的检验
iscsi_target_create()->开始target线程 完成结构体初始化
target_thread_start()->

int nthread_start(struct iscsi_target *target)
{
//...

  task = kthread_run(istd, target, "istd%d", target->tid);
//...
}

 

这是最主要关键的后台进程

static int istd(void *arg)
{
 struct iscsi_target *target = arg;
 struct network_thread_info *info = &target->nthread_info;
 struct iscsi_conn *conn, *tmp;
/*进入调度*/
 __set_current_state(TASK_RUNNING);
 do {
  spin_lock_bh(&info->nthread_lock);
  __set_current_state(TASK_INTERRUPTIBLE);
 /*数据没有好就让出调度 （这里要注意设置顺序，防止丢失事件）*/
  if (!test_bit(D_DATA_READY, &info->flags)) {
   spin_unlock_bh(&info->nthread_lock);
   schedule();
   spin_lock_bh(&info->nthread_lock);
  }
  __set_current_state(TASK_RUNNING);
  clear_bit(D_DATA_READY, &info->flags);
  spin_unlock_bh(&info->nthread_lock);

  target_lock(target, 0);
  list_for_each_entry_safe(conn, tmp, &info->active_conns, poll_list) {
   if (test_bit(CONN_ACTIVE, &conn->state))
    process_io(conn);/*处理IO事物*/
   else
    close_conn(conn);
  }
  target_unlock(target);

 } while (!kthread_should_stop());

 return 0;
}

 

 

先回顾一下 socket-> ops 的一个函数指针结构

struct proto_ops {
//...
ssize_t  (*sendpage)  (struct socket *sock, struct page *page,
          int offset, size_t size, int flags);
ssize_t  (*splice_read)(struct socket *sock,  loff_t *ppos,
           struct pipe_inode_info *pipe, size_t len, unsigned int flags);
//

 ...

}
在 tcp4中主要是
struct proto tcp_prot = {
 .sendpage  = tcp_sendpage,

}

Tcp_sendpage 也就是do_tcp_sendpages  先把页面拆分后
放到sk_write_queue 中然后按需通过 __tcp_push_pending_frames (多页) tcp_push_one(单页 头部) 发送出去。

static void process_io(struct iscsi_conn *conn)
{
 struct iscsi_target *target = conn->session->target;
 int res, wakeup = 0;
 /*从网络上接受过来的数据请求*/
 res = recv(conn);
 //...
 /*将带数据相应包发送出去*/
 res = send(conn);
 /*如果还有数据就等待下次处理*/
 if (!list_empty(&conn->write_list) || conn->write_cmnd) {
  conn_reset_nop_timer(conn);
  wakeup = 1;
 }

//...
 return;
}

 函数很简单没什么好说的  看看recv

static int recv(struct iscsi_conn *conn)  -->  case RX_DATA: do_recv()

这里作者还是很懒

直接 res = sock_recvmsg(conn->sock, &msg, len, MSG_DONTWAIT | MSG_NOSIGNAL);

然后就是 send
static int send(struct iscsi_conn *conn) -> 先是 case TX_INIT:
cmnd_tx_start(cmnd);  case ISCSI_OP_SCSI_RSP:

static void cmnd_send_pdu(struct iscsi_conn *conn, struct iscsi_cmnd *cmnd)
将 iscsi_conn-> write_tcmnd->tio 该tio 通过随后的 do_send()--->write_data()-->sendpage
发送出去