ictclas4j分词模块

         ictclas4j中文分词模块ms也是采用了Viterbi算法进行切词，在切词基础上进行词性标注。具体可参阅其代码：

 

package org.ictclas4j.segment;   
  
import java.io.BufferedReader;   
import java.io.IOException;   
import java.io.InputStreamReader;   
import java.util.ArrayList;   
import org.ictclas4j.bean.Atom;   
import org.ictclas4j.bean.Dictionary;   
import org.ictclas4j.bean.MidResult;   
import org.ictclas4j.bean.SegNode;   
import org.ictclas4j.bean.SegResult;   
import org.ictclas4j.bean.Sentence;   
import org.ictclas4j.utility.POSTag;   
import org.ictclas4j.utility.Utility;   
  
public class SegTag {   
    private Dictionary coreDict;   
    private Dictionary bigramDict;   
    private PosTagger personTagger;   
    private PosTagger transPersonTagger;   
    private PosTagger placeTagger;   
    private PosTagger lexTagger;   
  
    private int segPathCount = 1;// 分词路径的数目   
  
    public SegTag(int segPathCount) {   
        this.segPathCount = segPathCount;   
        coreDict = new Dictionary("data\\coreDict.dct");   
  
        bigramDict = new Dictionary("data\\bigramDict.dct");   
        personTagger = new PosTagger(Utility.TAG_TYPE.TT_PERSON, "data\\nr",   
                coreDict);   
        transPersonTagger = new PosTagger(Utility.TAG_TYPE.TT_TRANS_PERSON,   
                "data\\tr", coreDict);   
        placeTagger = new PosTagger(Utility.TAG_TYPE.TT_TRANS_PERSON,   
                "data\\ns", coreDict);   
        lexTagger = new PosTagger(Utility.TAG_TYPE.TT_NORMAL, "data\\lexical",   
                coreDict);   
    }   
  
    public SegResult split(String src) {   
        SegResult sr = new SegResult(src);// 分词结果   
        String finalResult = null;   
  
        if (src != null) {   
            finalResult = "";   
            int index = 0;   
            String midResult = null;   
            sr.setRawContent(src);   
            SentenceSeg ss = new SentenceSeg(src);   
            ArrayList<Sentence> sens = ss.getSens();   
  
            for (Sentence sen : sens) {   
                long start = System.currentTimeMillis();   
                MidResult mr = new MidResult();   
                mr.setIndex(index++);   
                mr.setSource(sen.getContent());   
                if (sen.isSeg()) {   
                    // 原子分词   
                    AtomSeg as = new AtomSeg(sen.getContent());   
                    ArrayList<Atom> atoms = as.getAtoms();   
                    mr.setAtoms(atoms);   
                    println2Err("[atom time]:"  
                            + (System.currentTimeMillis() - start));   
                    start = System.currentTimeMillis();   
  
                    // 生成分词图表,先进行初步分词，然后进行优化，最后进行词性标记   
                    SegGraph segGraph = GraphGenerate.generate(atoms, coreDict);   
                    mr.setSegGraph(segGraph.getSnList());   
                    // 生成二叉分词图表   
                    SegGraph biSegGraph = GraphGenerate.biGenerate(segGraph,   
                            coreDict, bigramDict);   
                    mr.setBiSegGraph(biSegGraph.getSnList());   
                    println2Err("[graph time]:"  
                            + (System.currentTimeMillis() - start));   
                    start = System.currentTimeMillis();   
  
                    // 求N最短路径   
                    NShortPath nsp = new NShortPath(biSegGraph, segPathCount);   
                    ArrayList<ArrayList<Integer>> bipath = nsp.getPaths();   
                    mr.setBipath(bipath);   
                    println2Err("[NSP time]:"  
                            + (System.currentTimeMillis() - start));   
                    start = System.currentTimeMillis();   
  
                    for (ArrayList<Integer> onePath : bipath) {   
                        // 得到初次分词路径   
                        ArrayList<SegNode> segPath = getSegPath(segGraph,   
                                onePath);   
                        ArrayList<SegNode> firstPath = AdjustSeg   
                                .firstAdjust(segPath);   
                        String firstResult = outputResult(firstPath);   
                        mr.addFirstResult(firstResult);   
                        println2Err("[first time]:"  
                                + (System.currentTimeMillis() - start));   
                        start = System.currentTimeMillis();   
  
                        // 处理未登陆词，进对初次分词结果进行优化   
                        SegGraph optSegGraph = new SegGraph(firstPath);   
                        ArrayList<SegNode> sns = clone(firstPath);   
                        personTagger.recognition(optSegGraph, sns);   
                        transPersonTagger.recognition(optSegGraph, sns);   
                        placeTagger.recognition(optSegGraph, sns);   
                        mr.setOptSegGraph(optSegGraph.getSnList());   
                        println2Err("[unknown time]:"  
                                + (System.currentTimeMillis() - start));   
                        start = System.currentTimeMillis();   
  
                        // 根据优化后的结果，重新进行生成二叉分词图表   
                        SegGraph optBiSegGraph = GraphGenerate.biGenerate(   
                                optSegGraph, coreDict, bigramDict);   
                        mr.setOptBiSegGraph(optBiSegGraph.getSnList());   
  
                        // 重新求取N－最短路径   
                        NShortPath optNsp = new NShortPath(optBiSegGraph,   
                                segPathCount);   
                        ArrayList<ArrayList<Integer>> optBipath = optNsp   
                                .getPaths();   
                        mr.setOptBipath(optBipath);   
  
                        // 生成优化后的分词结果，并对结果进行词性标记和最后的优化调整处理   
                        ArrayList<SegNode> adjResult = null;   
                        for (ArrayList<Integer> optOnePath : optBipath) {   
                            ArrayList<SegNode> optSegPath = getSegPath(   
                                    optSegGraph, optOnePath);   
                            lexTagger.recognition(optSegPath);   
                            String optResult = outputResult(optSegPath);   
                            mr.addOptResult(optResult);   
                            adjResult = AdjustSeg.finaAdjust(optSegPath,   
                                    personTagger, placeTagger);   
                            String adjrs = outputResult(adjResult);   
                            println2Err("[last time]:"  
                                    + (System.currentTimeMillis() - start));   
                            start = System.currentTimeMillis();   
                            if (midResult == null)   
                                midResult = adjrs;   
                            break;   
                        }   
                    }   
                    sr.addMidResult(mr);   
                } else  
                    midResult = sen.getContent();   
                finalResult += midResult;   
                midResult = null;   
            }   
  
            sr.setFinalResult(finalResult);   
        }   
  
        return sr;   
    }   
  
    private ArrayList<SegNode> clone(ArrayList<SegNode> sns) {   
        ArrayList<SegNode> result = null;   
        if (sns != null && sns.size() > 0) {   
            result = new ArrayList<SegNode>();   
            for (SegNode sn : sns)   
                result.add(sn.clone());   
        }   
        return result;   
    }   
  
    // 根据二叉分词路径生成分词路径   
    private ArrayList<SegNode> getSegPath(SegGraph sg, ArrayList<Integer> bipath) {   
        ArrayList<SegNode> path = null;   
  
        if (sg != null && bipath != null) {   
            ArrayList<SegNode> sns = sg.getSnList();   
            path = new ArrayList<SegNode>();   
  
            for (int index : bipath)   
                path.add(sns.get(index));   
        }   
        return path;   
    }   
  
    // 根据分词路径生成分词结果   
    private String outputResult(ArrayList<SegNode> wrList) {   
        String result = null;   
        String temp = null;   
        char[] pos = new char[2];   
        if (wrList != null && wrList.size() > 0) {   
            result = "";   
            for (int i = 0; i < wrList.size(); i++) {   
                SegNode sn = wrList.get(i);   
                if (sn.getPos() != POSTag.SEN_BEGIN   
                        && sn.getPos() != POSTag.SEN_END) {   
                    int tag = Math.abs(sn.getPos());   
                    pos[0] = (char) (tag / 256);   
                    pos[1] = (char) (tag % 256);   
                    temp = "" + pos[0];   
                    if (pos[1] > 0)   
                        temp += "" + pos[1];   
                    result += sn.getSrcWord() + "/" + temp + " ";   
                }   
            }   
        }   
  
        return result;   
    }   
  
    public void setSegPathCount(int segPathCount) {   
        this.segPathCount = segPathCount;   
    }   
  
    public static void main(String[] args) {   
        SegTag segTag = new SegTag(1);   
  
        BufferedReader reader = new BufferedReader(new InputStreamReader(   
                System.in));   
        String line = null;   
        try {   
            while ((line = reader.readLine()) != null) {   
                try {   
                    SegResult seg_res = segTag.split(line);   
                    System.out.println(seg_res.getFinalResult());   
                } catch (Throwable t) {   
                    t.printStackTrace();   
                }   
            }   
        } catch (IOException e) {   
            e.printStackTrace();   
        }   
    }   
  
    private static void println2Err(String str) {   
        // System.err.println(str);   
    }   
}