XULRunner with Java: JavaXPCOM Tutorial 4

7. 用JavaXPCOM来自动浏览网页
   在这一节里我们将解决一些自动浏览网页中的一些问题，尝试把一下通用的任务抽象成人类可读的方法，因此你可以轻易的阅读

代码并知道它的功能。我们将构建一个Web Browser来加载网页，点击按钮或者超链接，使用XPath来抽取一下有用的信息。在每一个

小节里面，我们都将在我们的浏览器里增加新的功能，因此在最后，我们将有一个Web Browser，它能够实现网页的自动
浏览。

7.1 使用我们的浏览器来加载网页
    在例子 SimpleBrowser里，我们使用方法 public boolean setUrl(String url) 来让浏览器加载一个url。这个方法的问题是它

可以让浏览器开始加载一个页面，但是不会等待浏览器加载页面完成。我们写了一个方法叫做go来实现这个功能，因此后面我们将使

用这种方法来安全的加载一个页面，阻塞住执行流程直到页面加载成功或者超时。
    注：实现的方法是在setUrl的时候定义一个CountDownLatch，然后监听browser.addProgressListener，等页面完成后把这个

CountDownLatch countDown一下。调用setUrl后使用CountDownLatch的await方法等待加载完成或者超时。注意调用setUrl时要启动

一个SWT线程       
               display.syncExec(new Runnable() {
            public void run() {
                browser.setUrl(url);
            }
        });
    另外，我们可能还想获得加载后的页面内容，所有译者增加了一个变量content。增加的代码如下
                browser.addProgressListener(new ProgressListener() {
                    public void changed(ProgressEvent event) {
                    }

                    public void completed(ProgressEvent event) {
                        // When a page is loaded, decrement the latch,
                        // which count will be 0 after this call.
                        latch.countDown();
                        content=browser.getText();// added by LiLi
                    }
                });

    package es.ladyr.javaxpcom.browser;  
      
    import java.util.concurrent.CountDownLatch;  
    import java.util.concurrent.TimeUnit;  
    import org.eclipse.swt.SWT;  
    import org.eclipse.swt.browser.ProgressEvent;  
    import org.eclipse.swt.browser.ProgressListener;  
    import org.eclipse.swt.widgets.Display;  
    import org.eclipse.swt.widgets.Shell;  
    import org.eclipse.swt.browser.Browser;  
    import org.eclipse.swt.SWTError;  
      
    public class SimpleBrowserWithGo {  
             
            // We will need SWT display to execute methods  
            // into the SWT event thread.  
             
                    Browser browser;  
            private Display display;  
             
            // Latch used to manage page loading  
            // Uses a count of 1, so when the browser starts loading  
            // a page, we create a new latch, which will be  
            // decremented when the page is loaded.  
            private CountDownLatch latch;  
             
            // Default timeout to 60 seconds  
            private long defaultTimeout = 60000;  
            /** 
             * Creates a web browser which is able to load pages waiting until 
             * the page is completely loaded. 
             * 
             */  
            public SimpleBrowserWithGo (final String xulrunnerPath) {  
                     
                    // Use a latch to wait for the browser initialization.  
                    final CountDownLatch initLatch = new CountDownLatch(1);  
                     
                     
                     
                    // MozillaBrowser needs a window manager to work. We are using SWT  
                    // for the graphical interface, so we need to execute MozillaBrowser  
                    // methods into the SWT event thread. If we were use another thread,  
                    // that methods could not work properly and throw an exception,  
                    // breaking the execution flow and crashing our application.  
                    new Thread("SWT-Event-Thread") {  
                            @Override  
                            public void run() {  
                                             
                                    display = new Display();  
                                    Shell shell = new Shell(display);  
                                    shell.setSize(800, 600);  
                                    shell.open();  
                                    // If you have XULRunner installed, you can call the constructor without  
                                    // the last parameter:  
                                    //  
                                    // final MozillaBrowser browser = new MozillaBrowser(shell,SWT.BORDER);  
                                    //  
                                    // That last parameter is the path for XULRunner files  
                                    // (where you have uncompressed downloaded XULRunner package).  
                                    try {  
                                                    browser = new Browser(shell, SWT.MOZILLA);  
                                            } catch (SWTError e) {  
                                                    System.out.println("Could not instantiate Browser: " + e.getMessage  
    ());  
                                                    return;  
                                            }  
                                     
                                     
                                    // Adapt browser size to shell size  
                                    browser.setBounds(shell.getClientArea());  
                                     
                                    // Listens for page loading status.  
                                    browser.addProgressListener(new ProgressListener() {  
                                            public void changed(ProgressEvent event) {  
                                            }  
                                            public void completed(ProgressEvent event) {  
                                                    // When a page is loaded, decrement the latch,  
                                                    // which count will be 0 after this call.  
                                                    latch.countDown();  
                                            }  
                                    });  
                                     
                                    // Realease the initialization latch, which has value 1,  
                                    // so after this call its value will be 0.  
                                    initLatch.countDown();  
                                     
                                    while (!shell.isDisposed()) {  
                                            if (!display.readAndDispatch()) {  
                                                    display.sleep();  
                                            }  
                                    }  
                                    System.exit(0);  
                            }  
                    }.start();  
                     
                    try {  
                            // Waits until the initialization latch is released.  
                            initLatch.await();  
                    } catch (InterruptedException e) {  
                            Thread.interrupted();  
                    }                
            }  
             
            /** 
             * Loads an URL into the browser and waits until the page is 
             * totally loaded. 
             * @param url 
             * @throws SimpleBrowserException 
             */  
            public void go(final String url) throws SimpleBrowserException {  
                     
                    // Creates a latch with count 1  
                    latch = new CountDownLatch(1);  
                    // Uses the SWT event thread to execute the method to  
                    // load an URL in the browser.  
                    display.syncExec(new Runnable() {  
                            public void run() {  
                                    browser.setUrl(url);  
                            }  
                    });  
                     
                    // Waits for the finish of the page loading, or for a given  
                    // timeout in case that the loading doesn't finish in a  
                    // reasonable time.  
                    boolean timeout = waitLoad(defaultTimeout);  
                    if (timeout) {  
                            throw new SimpleBrowserException("Timeout waiting page loading.");  
                    }  
            }  
             
            private boolean waitLoad(long millis) {  
                    try {  
                            // Uses the latch, created by 'go' method to wait for  
                            // the finish of the page loading (it will occurs when  
                            // our 'progressListener' receives a event for its method  
                            // 'completed'), or for a given timeout in case that the  
                            // loading doesn't finish in a reasonable time.  
                            boolean timeout;  
                            timeout = !latch.await(millis,TimeUnit.MILLISECONDS);  
                            if (timeout) {  
                                    // If the timeout expired, then we will stop  
                                    // page loading.  
                                    display.syncExec(new Runnable() {  
                                            public void run() {  
                                                    browser.stop();  
                                            }  
                                    });  
                                    // Waits for the loading is stopped  
                                    latch.await(millis,TimeUnit.MILLISECONDS);  
                            }  
                            return timeout;  
                    } catch (InterruptedException e) {  
                            throw new Error(e);  
                    }  
            }  
             
            public static void main(String[] args) {  
                    String xulrunnerPath = null;  
                    if ( args.length > 0 ) {  
                            xulrunnerPath = args[0];  
                    }  
                     
                    // Instantiate our simple web browser  
                    SimpleBrowserWithGo simpleBrowser = new SimpleBrowserWithGo(xulrunnerPath);  
             
                    try{  
                            // Use the new functionality to load some URLs  
                            // with our browser.  
                            simpleBrowser.go("http://www.google.com");  
                            Thread.sleep(3000);  
                            simpleBrowser.go("http://www.urjc.es");  
                            Thread.sleep(3000);  
                            simpleBrowser.go("http://www.mozilla.org");  
                            Thread.sleep(3000);  
                             
                    } catch (SimpleBrowserException e) {  
                            System.err.println("Problems calling go method.");  
                            e.printStackTrace();  
                    } catch (InterruptedException e) {  
                            System.err.println("Problems calling sleep.");  
                            e.printStackTrace();  
                            Thread.interrupted();  
                    }  
                                                     
       ime().halt(0);  
                     
            }  
             
    }  

 

7.2 解析XPath来获得W3C Node
    一旦我们能够在浏览器里加载一个HTML页面，我们可能想访问DOM节点来抽取信息。前面我们花了一节来吧Mozilla Node转换成

W3C node。现在我们使用那个方法来用标准的方法操作W3C Node。我们实现了一些方法来创建XPath Evaluator和XPath resolver来

抽取节点。当Xpath evaluator返回一个结果，我们把每个返回的DOM node转换成响应的W3C DOM element，使用的方法是 static

Node getNodeInstance( nsIDOMNode node ) 。因此，使用我们的browser可以直接调用下面的方法：

    ...  
    import org.w3c.dom.Node;  
    ...  
    /** 
     * 
     * @param xpath 
     * @return      a list with the nodes corresponding to a given xpath. 
     * @throws SimpleBrowserException 
     */  
    public List<Node> xpathNodes(String xpath) {  
    ...  
    /** 
     * 
     * @param <T> 
     * @param xpath 
     * @param nodeClass 
     * @return      a list of <code>nodeClass</code> nodes corresponding 
     *      to a given xpath. 
     * @throws SimpleBrowserException 
     */  
    public <T extends Node> List<T> xpathNodes(String xpath, Class<T> nodeClass) {  
    ...  

 

 

下面是完整的例子：

译注：核心代码就是下面这个方法。它有两个参数，xpath和nsIDOMNode,返回的是满足XPath的Node的list。
Node是W3C Node 。其实就是调用xpcom的接口来做xpath解析，然后把nsIDOMNode的List转成W3C Node的list。
不过我感觉其实没有必要，nsIDOMNode的属性更多，而且W3C Node没办法转回去成为nsIDOMNode。在使用XPath的时候，我们可能先

用绝对XPath找到某个table，然后根据相对XPath找tr，td等。第二次调用xPathNodes是的参数 nsIDOMNode context就是第一次返回

的结果里的nsIDOMNode。

private List<Node> xPathNodes(String xpath, nsIDOMNode context) {  
        // Obtain the Mozilla DOM HTML document  
        HTMLDocumentImpl documentImpl = (HTMLDocumentImpl) getW3CDocument();  
        nsIDOMHTMLDocument document = documentImpl.getInstance();  
        // Creates a name space resolver for the document  
        nsIDOMXPathNSResolver res = xpathEval.createNSResolver(document);  
        List<Node> resultNodes = null;  
        // Evaluates given XPath in a given context, using the resolver created  
        // for the current document as an ordered iterator  
        nsISupports obj = xpathEval.evaluate(xpath, context, res,  
                nsIDOMXPathResult.ORDERED_NODE_ITERATOR_TYPE, null);  
        // Obtain the interface corresponding to the XPath XPCOM results object  
        nsIDOMXPathResult result = (nsIDOMXPathResult) obj  
                .queryInterface(nsIDOMXPathResult.NS_IDOMXPATHRESULT_IID);  
        try {  
            // Extract result nodes for the XPath and add them  
            // to a standard List.  
            resultNodes = getNodes(result);  
        } catch (org.mozilla.xpcom.XPCOMException e) {  
            throw e;  
        }  
        return resultNodes;  
    }  
  
package es.ladyr.javaxpcom.browser;  
import java.util.ArrayList;  
import java.util.List;  
import java.util.concurrent.CountDownLatch;  
import java.util.concurrent.TimeUnit;  
import org.eclipse.swt.SWT;  
import org.eclipse.swt.SWTError;  
import org.eclipse.swt.browser.Browser;  
import org.eclipse.swt.browser.ProgressEvent;  
import org.eclipse.swt.browser.ProgressListener;  
import org.eclipse.swt.widgets.Display;  
import org.eclipse.swt.widgets.Shell;  
import org.mozilla.dom.NodeFactory;  
import org.mozilla.dom.html.HTMLDocumentImpl;  
import org.mozilla.interfaces.nsIComponentManager;  
import org.mozilla.interfaces.nsIDOMDocument;  
import org.mozilla.interfaces.nsIDOMHTMLDocument;  
import org.mozilla.interfaces.nsIDOMNode;  
import org.mozilla.interfaces.nsIDOMWindow;  
import org.mozilla.interfaces.nsIDOMXPathEvaluator;  
import org.mozilla.interfaces.nsIDOMXPathNSResolver;  
import org.mozilla.interfaces.nsIDOMXPathResult;  
import org.mozilla.interfaces.nsISupports;  
import org.mozilla.interfaces.nsIWebBrowser;  
import org.mozilla.xpcom.Mozilla;  
import org.w3c.dom.Node;  
import org.w3c.dom.html.HTMLAnchorElement;  
import org.w3c.dom.html.HTMLDocument;  
  
public class SimpleBrowserWithXPath {  
         
        private final static String NS_IDOMXPATHEVALUATOR_CONTRACTID = "@mozilla.org/dom/xpath-evaluator;1";  
         
        private Browser browser;  
         
        // We will need SWT display to execute methods  
        // into the SWT event thread.  
        private Display display;  
         
         
        // Latch used to manage page loading  
        // Uses a count of 1, so when the browser starts loading  
        // a page, we create a new latch, which will be  
        // decremented when the page is loaded.  
        private CountDownLatch latch;  
         
        // Default timeout to 60 seconds  
        private long defaultTimeout = 60000;  
         
        // XPath evaluator  
        private nsIDOMXPathEvaluator xpathEval;  
        /** 
         * Creates a web browser which is able to load pages waiting until 
         * the page is completely loaded and solve xpaths returning 
         * the corresponding nodes. 
         * 
         */  
        public SimpleBrowserWithXPath (final String xulrunnerPath) {  
                 
                // Use a latch to wait for the browser initialization.  
                final CountDownLatch initLatch = new CountDownLatch(1);  
                 
                 
                // MozillaBrowser needs a window manager to work. We are using SWT  
                // for the graphical interface, so we need to execute MozillaBrowser  
                // methods into the SWT event thread. If we were use another thread,  
                // that methods could not work properly and throw an exception,  
                // breaking the execution flow and crashing our application.  
                new Thread("SWT-Event-Thread") {  
                        @Override  
                        public void run() {  
                                        display = new Display();  
                                Shell shell = new Shell(display);  
                                shell.setSize(800, 600);  
                                shell.open();  
                                // If you have XULRunner installed, you can call the constructor without  
                                // the last parameter:  
                                //  
                                // final MozillaBrowser browser = new MozillaBrowser(shell,SWT.BORDER);  
                                //  
                                // That last parameter is the path for XULRunner files  
                                // (where you have uncompressed downloaded XULRunner package).  
                                try {  
                                                browser = new Browser(shell, SWT.MOZILLA);  
                                        } catch (SWTError e) {  
                                                System.out.println("Could not instantiate Browser: " + e.getMessage  
());  
                                                return;  
                                        }  
                                 
                                 
                                // Adapt browser size to shell size  
                                browser.setBounds(shell.getClientArea());  
                                 
                                // Listens for page loading status.  
                                browser.addProgressListener(new ProgressListener() {  
                                        public void changed(ProgressEvent event) {  
                                        }  
                                        public void completed(ProgressEvent event) {  
                                                // When a page is loaded, decrement the latch,  
                                                // which count will be 0 after this call.  
                                                latch.countDown();  
                                        }  
                                });  
                                 
                                // Realease the initialization latch, which has value 1,  
                                // so after this call its value will be 0.  
                                initLatch.countDown();  
                                 
                                while (!shell.isDisposed()) {  
                                        if (!display.readAndDispatch()) {  
                                                display.sleep();  
                                        }  
                                }  
                                System.exit(0);  
                        }  
                }.start();  
                 
                try {  
                        // Waits until the initialization latch is released.  
                        initLatch.await();  
                } catch (InterruptedException e) {  
                        Thread.interrupted();  
                }  
                 
                // Creates the XPath evaluator XPCOM component  
                Mozilla moz = Mozilla.getInstance();  
                nsIComponentManager componentManager = moz.getComponentManager();  
                xpathEval = (nsIDOMXPathEvaluator) componentManager.createInstanceByContractID(  
                                                NS_IDOMXPATHEVALUATOR_CONTRACTID, null,  
nsIDOMXPathEvaluator.NS_IDOMXPATHEVALUATOR_IID);  
                 
        }  
         
        /** 
         * Loads an URL into the browser and waits until the page is 
         * totally loaded. 
         * @param url 
         * @throws SimpleBrowserException 
         */  
        public void go(final String url) throws SimpleBrowserException {  
                 
                // Creates a latch with count 1  
                latch = new CountDownLatch(1);  
                // Uses the SWT event thread to execute the method to  
                // load an URL in the browser.  
                display.syncExec(new Runnable() {  
                        public void run() {  
                                browser.setUrl(url);  
                        }  
                });  
                 
                // Waits for the finish of the page loading, or for a given  
                // timeout in case that the loading doesn't finish in a  
                // reasonable time.  
                boolean timeout = waitLoad(defaultTimeout);  
                if (timeout) {  
                        throw new SimpleBrowserException("Timeout waiting page loading.");  
                }  
        }  
         
        /** 
         * 
         * @return      an W3C HTML Document implementation corresponding to 
         *      the Mozilla DOM HTML document currently loaded in the browser. 
         * @throws SimpleBrowserException 
         */  
        public HTMLDocument getW3CDocument() {  
                 
   //           System.out.println("El browser es " + browser.toString());  
                 
                class DocumentGetter implements Runnable {  
                        private nsIDOMHTMLDocument htmldoc;  
                                public void run(){  
                nsIWebBrowser webBrowser = (nsIWebBrowser)browser.getWebBrowser();  
                        if (webBrowser == null) {  
                                System.out.println("Could not get the nsIWebBrowser from the Browser widget");  
                        }        
         
                        nsIDOMWindow dw = webBrowser.getContentDOMWindow();  
                nsIDOMDocument nsDoc = dw.getDocument();  
                htmldoc = (nsIDOMHTMLDocument) nsDoc  
                                                                .queryInterface  
(nsIDOMHTMLDocument.NS_IDOMHTMLDOCUMENT_IID);  
                 
                }  
                                public nsIDOMHTMLDocument getHtmldoc() {  
                                        // TODO Auto-generated method stub  
                                        return htmldoc;  
                                }}  
                 
                DocumentGetter dg = new DocumentGetter();  
                 
                display.syncExec(dg);  
                 
                 
                 
                return HTMLDocumentImpl.getDOMInstance(dg.getHtmldoc());  
        }  
         
        /** 
         * 
         * @param xpath 
         * @return      a list with the nodes corresponding to a given xpath. 
         * @throws SimpleBrowserException 
         */  
        public List<Node> xpathNodes(String xpath) {  
                return xPathNodes(xpath,  
                                ((HTMLDocumentImpl) getW3CDocument()).getInstance());            
        }  
         
        /** 
         * 
         * @param <T> 
         * @param xpath 
         * @param nodeClass 
         * @return      a list of <code>nodeClass</code> nodes corresponding 
         *      to a given xpath. 
         * @throws SimpleBrowserException 
         */  
        public <T extends Node> List<T> xpathNodes(String xpath, Class<T> nodeClass) {  
                return (List<T>)xPathNodes(xpath,  
                                ((HTMLDocumentImpl) getW3CDocument()).getInstance());  
        }        
         
        private boolean waitLoad(long millis) {  
                try {  
                        // Uses the latch, created by 'go' method to wait for  
                        // the finish of the page loading (it will occurs when  
                        // our 'progressListener' receives a event for its method  
                        // 'completed'), or for a given timeout in case that the  
                        // loading doesn't finish in a reasonable time.  
                        boolean timeout;  
                        timeout = !latch.await(millis,TimeUnit.MILLISECONDS);  
                        if (timeout) {  
                                // If the timeout expired, then we will stop  
                                // page loading.  
                                display.syncExec(new Runnable() {  
                                        public void run() {  
                                                browser.stop();  
                                        }  
                                });  
                                // Waits for the loading is stopped  
                                latch.await(millis,TimeUnit.MILLISECONDS);  
                        }  
                        return timeout;  
                } catch (InterruptedException e) {  
                        throw new Error(e);  
                }  
        }  
         
        private List<Node> xPathNodes(String xpath, nsIDOMNode context) {  
                 
                // Obtain the Mozilla DOM HTML document  
                HTMLDocumentImpl documentImpl = (HTMLDocumentImpl) getW3CDocument();  
                nsIDOMHTMLDocument document =  documentImpl.getInstance();  
                 
                // Creates a name space resolver for the document  
                nsIDOMXPathNSResolver res = xpathEval.createNSResolver(document);  
                 
                List<Node> resultNodes = null;  
                 
                // Evaluates given XPath in a given context, using the resolver created  
                // for the current document as an ordered iterator  
                nsISupports obj = xpathEval.evaluate(xpath, context, res,  
                                nsIDOMXPathResult.ORDERED_NODE_ITERATOR_TYPE, null);  
                // Obtain the interface corresponding to the XPath XPCOM results object  
                nsIDOMXPathResult result = (nsIDOMXPathResult) obj.queryInterface(  
                                nsIDOMXPathResult.NS_IDOMXPATHRESULT_IID);  
                 
                try {  
                        // Extract result nodes for the XPath and add them  
                        // to a standard List.  
                        resultNodes = getNodes(result);  
                } catch(org.mozilla.xpcom.XPCOMException e){  
                        throw e;  
                }                        
                                 
                return resultNodes;  
        }  
         
        private <T> List<T> getNodes(nsIDOMXPathResult result) {  
                List<T> nodes = new ArrayList<T>();  
                 
                nsIDOMNode node;  
                while((node = result.iterateNext()) != null){  
                        // Use the functionality provided by the mozdom4java  
                        // (in our case, patched) library to obtain the corresponding  
                        // W3C implementation of a node.  
                        nodes.add((T)NodeFactory.getNodeInstance(node));  
                }  
                return nodes;  
        }  
         
        public static void main(String[] args) {  
                String xulrunnerPath = null;  
                if ( args.length > 0 ) {  
                        xulrunnerPath = args[0];  
                }  
                 
                // Instantiate our simple web browser  
                SimpleBrowserWithXPath simpleBrowser = new SimpleBrowserWithXPath(xulrunnerPath);  
         
                try{  
                        // Load a page in the browser  
                        simpleBrowser.go("http://www.google.com");  
                         
                        // Obtain a list of nodes, without a concrete class,  
                        // because the XPath may return nodes of different  
                        // types, so we work with them in a generic way.  
                        List<Node> nodes = simpleBrowser.xpathNodes("//*");  
                        for (Node node: nodes) {  
                                System.out.println("Node Type: " + node.getNodeName()  
                                                + " -- Content: " + node.getTextContent());  
                        }  
                         
                        // Obtain a list of HTMLAnchorElements, because  
                        // we can be sure about the result of our XPath,  
                        // if it has any result, will be only of  
                        // HTMLAnchorElement type.  
                        for (HTMLAnchorElement a: simpleBrowser.xpathNodes(  
                                        "//a", HTMLAnchorElement.class)) {  
                                System.out.println("Anchor: " + a.getHref());  
                        }  
                         
                } catch (SimpleBrowserException e) {  
                        System.err.println("Problems calling go method.");  
                        e.printStackTrace();  
                }  
                                                 
       
          Runtime.getRuntime().halt(0);  
                 
        }  
         
}