C# 采集6-这个程序的采集关键流程

想到不能演示，我也没心情写了，这里一步把关键点贴出来吧，其实程序还有一部分是结果整理，然后上传到服务器的。大家努力吧。

 

 

   //绑定数据源
        private void loaddata()
        {
            //重新绑定，需先移除原来的数据源
            listBox1.DataSource = null;
            listBox2.DataSource = null;
            offtask = xmldao.getTasks("off", xdoc);
            ontask = xmldao.getTasks("on", xdoc);
            listBox1.DataSource = offtask;
            listBox2.DataSource = ontask;
            listBox1.DisplayMember = "Name";
            listBox1.ValueMember = "Name";
            listBox2.DisplayMember = "Name";
            listBox2.ValueMember = "Name";
        }

        //采集列表
        private void colectList()
        {
            try
            {
                star.Enabled = false;//采集按钮不可用
                string htmlcode = null;
                Regex regexarticles = null;
                MatchCollection marticles = null;
                List<Urls> l = new List<Urls>();
                foreach (Task t in ontask)//逐项任务采集
                {
                    label4.Text="正在进行对任务"+t.Name+"的列表采集";
                    //htmlcode = Colect.GetHtmlSource(t.Url, t.Ecode);
                    htmlcode = Colect.GetHtmlSource(t.Url);
                    htmlcode = Colect.SniffwebCode(htmlcode, t.ListAreaf, t.ListAreae);
                    regexarticles = new Regex(t.RexTitle);
                    marticles = regexarticles.Matches(htmlcode);
                    foreach (Match m in marticles)
                    {
                        Urls u = new Urls();
                        u.Title = Regex.Replace(m.Groups["title"].Value, "<[^>]*>", "");
                        u.Url = m.Groups["url"].Value;
                        if (u.Url.IndexOf("./") > 0)
                        {
                            u.Url.Replace("../", "");
                        }
                        if (u.Url.IndexOf("ttp:")<1)
                        {
                            Regex re=new Regex("http://[^/]+/");
                            if(re.Match(t.Url).Success)
                            {
                                u.Url = re.Match(t.Url).Value + u.Url;
                            }
                            
                        }
                        u.Name = t.Name;
                        l.Add(u);
                    }
                }
                list = l;
                label4.Text = "列表采集完毕";
                //保存列表
                saveList();
            }
            catch (Exception err)
            {
                label4.Text = "列表采集出错，错误原因："+err.Message;
            }
        }

        //保存列表
        private void saveList()
        {
            if (list.Count < 1)
            {
                MessageBox.Show("无数据可添加");
                return;
            }
            else
            {
                string sqlstr;
                try
                {
                    dco.OpenConn();
                    foreach (Urls u in list)
                    {
                        if (u.Title != "")
                        {
                            sqlstr = "insert into list (title,url,isc,name) values ('" + u.Title + "','" + u.Url + "',0,'"+u.Name+"')";
                            dco.ExceteQuery(sqlstr);
                        }
                    }
                    //删除重复,保留最先添加的项
                    label4.Text = "保存文章列表成功，正在删除重复列表...";
                    sqlstr = "delete * from list Where oid Not In (select min(oid) from list group by url)";
                    dco.ExceteQuery(sqlstr);
                }
                catch (Exception ex)
                {
                    label4.Text = "列表保存出错，错误原因："+ex.Message;
                }
                finally
                {
                    dco.conn.Close();
                }
            }
        }

        //读取列表中未被采集的文章，进行采集
        private void getList()
        {
            label4.Text = "文章列表成功，正在读取未被采集的文章...";
            DataSet ds = new DataSet();
            string sqlstr = "select * from list where isc=0";
            dco.OpenConn();
            ds=dco.ExecuteDataSet(sqlstr);
            dco.conn.Close();
            List<Urls> l = new List<Urls>();
            foreach (DataRow r in ds.Tables[0].Rows)
            {
                Urls u = new Urls();
                u.Oid=(int)r["oid"];
                u.Title = r["title"].ToString();
                u.Url = r["url"].ToString();
                u.Isc = 0;
                u.Name=r["name"].ToString();
                l.Add(u);
            }
            label4.Text = "读取未被采集的文章成功，未被采集的文章" + l.Count + "篇...";
            list = l;//真正需要采集的文章，去除了重复项，已采集过的项
        }


        //采集内容
        private void colectArticle()
        {
            if (list.Count < 1)
            {
                return;
            }
            else
            {
                label4.Text = "开始采集文章内容...";
                List<Article> arl = new List<Article>();
                string htmlcode = null;
                dco.OpenConn();
                foreach (Urls u in list)
                {
                    if (u.Title != "")
                    {
                        label4.Text = "正在采集文章"+u.Title;
                        Task t = xmldao.getOne(u.Name, xdoc);
                        htmlcode = Colect.GetHtmlSource(u.Url,t.Ecode);
                        htmlcode = Colect.SniffwebCode(htmlcode, t.Contentf, t.Contente);
                        Article ar = new Article();
                        ar.Author = Colect.SniffwebCode(htmlcode, t.Authorf, t.Authore);
                        ar.Source = Colect.SniffwebCode(htmlcode, t.Sourcef, t.Sourcee);
                        ar.Postdate = Colect.SniffwebCode(htmlcode, t.Datef, t.Datee);
                        ar.Content = Regex.Replace(Colect.SniffwebCode(htmlcode, t.Articlef, t.Articlee), "<[^>]*>", "");
                        ar.Title = u.Title;
                        ar.Url = u.Url;
                        ar.Category = t.Category;
                        arl.Add(ar);
                        string sqlstr = "update list set isc=1 where oid="+u.Oid;
                        dco.ExceteQuery(sqlstr);
                    }
                }
                dco.conn.Close();
                label4.Text = "采集文章内容完毕...";
                articles = arl;
            }
        }

        //保存采集到的文章
        private void saveArticle()
        {
            if (articles.Count < 1)
            {
                label4.Text = "没有新文章需要采集..，采集结束";
                return;
            }
            else
            {
                label4.Text = "采集文章保存开始...";
                dco.OpenConn();
                foreach (Article ar in articles)
                {
                    string sqlstr = "insert into article (title,url,postdate,source,content,category,author) values ('" + ar.Title + "','" + ar.Url + "','" + ar.Postdate + "','" + ar.Source + "','" +ar.Content+ "','" +ar.Category+ "','" +ar.Author+ "')";
                    dco.ExceteQuery(sqlstr);
                }
                dco.conn.Close();
                label4.Text = "采集文章保存完毕，本次采集文章" + articles.Count + "篇，采集结束";
            }

        }