java获取网页主信息之四:抽取信息块

从已经建立完成的html树中抽取主信息。

package Source;

public class ChooseBlock
{
    //构造方法,设置允许错误率
    public ChooseBlock(double th)
    {
        threshold = th;
    }
    //获取一棵html树中的内容
    public String getContent(HTree tree)
    {
        int maxIndex = 0;
        String str = "";
        String contBlock[] = tree.getBlock();
        sizeBlock = getSizeBlock(contBlock);
        int len = sizeBlock.length;
        int aid[] = new int[len];
        staBlock = new boolean[len];
        iniStaBlock();
        double val = calError();
        //方差必须大于阈值,才认为有主要内容
        if(val < threshold) return null;
        int i = 0;
        double max = 0.0;
        do
        {
            //获得当前最可能的块
            int index = getIndex();
            aid[i] = index;
            setBlock(index);
            double tmp = val;
            val = calError();
            double err = tmp - val;
            if(err > max)
            {
                max = err;
                maxIndex = i;
            }
            if(err > val) break;
            i++;
        } while(true);
        //选出来的块信息集合
        for(i = 0; i <= maxIndex; i++)
        {
            int index = aid[i];
            str = (new StringBuilder(String.valueOf(str))).append("\n").append(contBlock[index]).toString();
        }
        return str;
    }

    //获取每一信息块长度
    private int[] getSizeBlock(String contBlock[])
    {
        int len = contBlock.length;
        int sizeBlock[] = new int[len];
        for(int i = 0; i < len; i++)
            sizeBlock[i] = contBlock[i].length();
        return sizeBlock;
    }

    //设置已被处理块
    private void setBlock(int index)
    {
        staBlock[index] = true;
    }

    //初始化块
    private void iniStaBlock()
    {
        int len = staBlock.length;
        for(int i = 0; i < len; i++)
            staBlock[i] = false;
    }

    //计算错误率,以方差记
    private double calError()
    {
        int sum = 0;
        int num = 0;
        int len = sizeBlock.length;
        //获得还未被选取块个数及其总长度
        for(int i = 0; i < len; i++)
            if(!staBlock[i])
            {
                num++;
                sum += sizeBlock[i];
            }
        //定义还未被选取块平均长度
        double avg = (double)sum / (1.0 * (double)num);
        //计算方差
        double err = 0.0;
        for(int i = 0; i < len; i++)
            if(!staBlock[i])
            {
                double val = (double)sizeBlock[i] - avg;
                val *= val;
                err += val;
            }
        //归一化
        return Math.sqrt(err) / (1.0 * (double)num);
    }

    //获取最大未被选块的下标
    private int getIndex()
    {
        int index = 0;
        int max = 0;
        int len = sizeBlock.length;
        for(int i = 0; i < len; i++)
            if(!staBlock[i] && sizeBlock[i] > max)
            {
                max = sizeBlock[i];
                index = i;
            }
        return index;
    }

    private double threshold;
    int sizeBlock[];
    boolean staBlock[];
}

分享到：

java获取网页主信息之五:测试 | java获取网页主信息之三:html to tree

• 2007-09-01 22:24
• 浏览 2269
• 评论(0)
• 查看更多