浏览 8785 次
|
精华帖 (0) :: 良好帖 (0) :: 新手帖 (0) :: 隐藏帖 (0)
|
|
|---|---|
| 作者 | 正文 |
|
发表时间：2010-04-08
最后修改：2010-04-26
lxml.html 和 libxml2dom 都很强大
libxml2dom 和 lxml.html 都是基于 libxml的python包装，性能也都没有什么差别 lxml.html 使用起来比较方便，但是api 不标准 libxml2dom 使用标准的api 用起来会比较舒服 和javascript的api一样
#coding:utf-8
test_html="""
<html>
<body>
ddd
<p>
p1
</html>
<html>
<body> <p>p2
fff<p>
p3
<i>iiii</i>
xxx</p>
<br>
d<b>bb</b>
</p>
"""
def test_lxml():
import lxml.html
doc=lxml.html.fromstring(test_html)
#print dir(doc)
#print doc.text
for p in doc.xpath('//p'):
#print '*', p.text
#print dir(p )
print 'p========',id(p)
print lxml.html.tostring(p)
print 'getchildren'
for i in p.getchildren():
print '#',i
print 'itertext'
for i in p.itertext():
print '$',type(i),i
print 'iter'
for i in p.iter():
print '$',type(i),i
print 'items'
for i in p.items():
print '$',type(i),i
def test_xml2dom():
import libxml2dom
#help(libxml2dom)
doc = libxml2dom.parseString(test_html, html=1)
for p in doc.xpath('//p'):
print 'p========='
print p,type(p)
#print dir(p)
print p.childNodes
for n in p.childNodes:
#print dir(n)
print n.textContent,n.nodeName,n.nodeType
def test_speed():
import lxml.html
import libxml2dom
import time
def load_lxml(html):
doc=lxml.html.fromstring(html)
return len(doc.xpath('//p'))
def load_dom(html):
doc=libxml2dom.parseString(html)
return len(doc.xpath('//p'))
def test_fun(f):
t1=time.time()
s=0
for i in range(10000):
s+=load_lxml(test_html)
print f, time.time()-t1,s
test_fun(load_lxml)
test_fun(load_dom)
test_fun(load_lxml)
test_fun(load_dom)
if __name__=='__main__':
test_xml2dom()
test_lxml()
test_speed()
主要测试下速度 和 比较不正规的html解析 都比较不错，性能也相差不大 执行结果 引用 p========= <libxml2dom.Node object at 0x8f1cf2c> <class 'libxml2dom.Node'> [<libxml2dom.Node object at 0x8f4a36c>] p1 text 3 p========= <libxml2dom.Node object at 0x8f1cf8c> <class 'libxml2dom.Node'> [<libxml2dom.Node object at 0x8f4a38c>] p2 fff text 3 p========= <libxml2dom.Node object at 0x8f4a32c> <class 'libxml2dom.Node'> [<libxml2dom.Node object at 0x8f4a3ac>, <libxml2dom.Node object at 0x8f4a34c>, <libxml2dom.Node object at 0x8f4a3cc>] p3 text 3 iiii i 1 xxx text 3 p======== 150665036 <p> p1 </p> getchildren itertext $ <type 'str'> p1 iter $ <class 'lxml.html.HtmlElement'> <Element p at 8faf74c> items p======== 150666476 <p>p2 fff</p> getchildren itertext $ <type 'str'> p2 fff iter $ <class 'lxml.html.HtmlElement'> <Element p at 8fafcec> items p======== 150666620 <p> p3 <i>iiii</i> xxx</p> getchildren # <Element i at 8faf56c> itertext $ <type 'str'> p3 $ <type 'str'> iiii $ <type 'str'> xxx iter $ <class 'lxml.html.HtmlElement'> <Element p at 8fafd7c> $ <class 'lxml.html.HtmlElement'> <Element i at 90e6ecc> items <function load_lxml at 0x90f6e2c> 1.59452700615 30000 <function load_dom at 0x90f6e64> 1.68855881691 30000 <function load_lxml at 0x90f6e2c> 1.71267414093 30000 <function load_dom at 0x90f6e64> 1.74115109444 30000 以后还是倾向于使用 libxml2dom这个模块了 －－－－－－－－－－－－－－ 附上libxml2dom的api －－－－－－ Help on package libxml2dom: NAME libxml2dom - DOM wrapper around libxml2, specifically the libxml2mod Python extension module. FILE /usr/local/lib/python2.6/dist-packages/libxml2dom/__init__.py DESCRIPTION Copyright (C) 2003, 2004, 2005, 2006, 2007 Paul Boddie <paul@boddie.org.uk> This program is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by the Free Software Foundation; either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. You should have received a copy of the GNU Lesser General Public License along with this program. If not, see <http://www.gnu.org/licenses/>. PACKAGE CONTENTS events macrolib (package) soap svg xmlrpc xmpp CLASSES __builtin__.list(__builtin__.object) NodeList __builtin__.object DocumentType Implementation NamedNodeMap NamedNodeMapIterator Node Attribute Document(_Document, Node) _Document Document(_Document, Node) class Attribute(Node) | A class providing attribute access. | | Method resolution order: | Attribute | Node | __builtin__.object | | Methods defined here: | | __init__(self, node, impl, ownerDocument=None, ownerElement=None) | | ---------------------------------------------------------------------- | Data descriptors defined here: | | parentNode | | ---------------------------------------------------------------------- | Methods inherited from Node: | | __eq__(self, other) | | __hash__(self) | | __ne__(self, other) | | appendChild(self, tmp) | | as_native_node(self) | | cloneNode(self, deep) | | createAttribute(self, name) | | createAttributeNS(self, ns, name) | | createCDATASection(self, value) | | createComment(self, value) | | createElement(self, name) | | createElementNS(self, ns, name) | | createTextNode(self, value) | | getAttribute(self, name) | | getAttributeNS(self, ns, localName) | | getAttributeNode(self, localName) | | getAttributeNodeNS(self, ns, localName) | | getElementById(self, identifier) | | getElementsByTagName(self, tagName) | | getElementsByTagNameNS(self, namespaceURI, localName) | | hasAttribute(self, name) | | hasAttributeNS(self, ns, localName) | | importNode(self, node, deep) | | insertBefore(self, tmp, oldNode) | | isSameNode(self, other) | | normalize(self) | | removeAttribute(self, name) | | removeAttributeNS(self, ns, localName) | | removeChild(self, tmp) | | replaceChild(self, tmp, oldNode) | | setAttribute(self, name, value) | | setAttributeNS(self, ns, name, value) | | setAttributeNode(self, node) | | setAttributeNodeNS(self, node) | | toFile(self, f, encoding=None, prettyprint=0) | | toStream(self, stream, encoding=None, prettyprint=0) | | toString(self, encoding=None, prettyprint=0) | | xpath(self, expr, variables=None, namespaces=None) | | ---------------------------------------------------------------------- | Data descriptors inherited from Node: | | __dict__ | dictionary for instance variables (if defined) | | __weakref__ | list of weak references to the object (if defined) | | attributes | | childNodes | | data | | doctype | | firstChild | | lastChild | | localName | | name | | namespaceURI | | nextSibling | | nodeName | | nodeType | | nodeValue | | prefix | | previousSibling | | publicId | | systemId | | tagName | | textContent | | value | | ---------------------------------------------------------------------- | Data and other attributes inherited from Node: | | ATTRIBUTE_NODE = 2 | | COMMENT_NODE = 8 | | DOCUMENT_NODE = 9 | | DOCUMENT_TYPE_NODE = 10 | | ELEMENT_NODE = 1 | | ENTITY_NODE = 6 | | ENTITY_REFERENCE_NODE = 5 | | NOTATION_NODE = 12 | | PROCESSING_INSTRUCTION_NODE = 7 | | TEXT_NODE = 3 | | entities = {} | | notations = {} class Document(_Document, Node) | A generic document class. Specialised document classes should inherit from | the _Document class and their own variation of Node. | | Method resolution order: | Document | _Document | Node | __builtin__.object | | Methods inherited from _Document: | | __del__(self) | | __init__(self, node, impl) | | ---------------------------------------------------------------------- | Data descriptors inherited from _Document: | | documentElement | | ownerDocument | | ---------------------------------------------------------------------- | Methods inherited from Node: | | __eq__(self, other) | | __hash__(self) | | __ne__(self, other) | | appendChild(self, tmp) | | as_native_node(self) | | cloneNode(self, deep) | | createAttribute(self, name) | | createAttributeNS(self, ns, name) | | createCDATASection(self, value) | | createComment(self, value) | | createElement(self, name) | | createElementNS(self, ns, name) | | createTextNode(self, value) | | getAttribute(self, name) | | getAttributeNS(self, ns, localName) | | getAttributeNode(self, localName) | | getAttributeNodeNS(self, ns, localName) | | getElementById(self, identifier) | | getElementsByTagName(self, tagName) | | getElementsByTagNameNS(self, namespaceURI, localName) | | hasAttribute(self, name) | | hasAttributeNS(self, ns, localName) | | importNode(self, node, deep) | | insertBefore(self, tmp, oldNode) | | isSameNode(self, other) | | normalize(self) | | removeAttribute(self, name) | | removeAttributeNS(self, ns, localName) | | removeChild(self, tmp) | | replaceChild(self, tmp, oldNode) | | setAttribute(self, name, value) | | setAttributeNS(self, ns, name, value) | | setAttributeNode(self, node) | | setAttributeNodeNS(self, node) | | toFile(self, f, encoding=None, prettyprint=0) | | toStream(self, stream, encoding=None, prettyprint=0) | | toString(self, encoding=None, prettyprint=0) | | xpath(self, expr, variables=None, namespaces=None) | | ---------------------------------------------------------------------- | Data descriptors inherited from Node: | | __dict__ | dictionary for instance variables (if defined) | | __weakref__ | list of weak references to the object (if defined) | | attributes | | childNodes | | data | | doctype | | firstChild | | lastChild | | localName | | name | | namespaceURI | | nextSibling | | nodeName | | nodeType | | nodeValue | | parentNode | | prefix | | previousSibling | | publicId | | systemId | | tagName | | textContent | | value | | ---------------------------------------------------------------------- | Data and other attributes inherited from Node: | | ATTRIBUTE_NODE = 2 | | COMMENT_NODE = 8 | | DOCUMENT_NODE = 9 | | DOCUMENT_TYPE_NODE = 10 | | ELEMENT_NODE = 1 | | ENTITY_NODE = 6 | | ENTITY_REFERENCE_NODE = 5 | | NOTATION_NODE = 12 | | PROCESSING_INSTRUCTION_NODE = 7 | | TEXT_NODE = 3 | | entities = {} | | notations = {} class DocumentType(__builtin__.object) | A class providing a container for document type information. | | Methods defined here: | | __init__(self, localName, publicId, systemId) | | ---------------------------------------------------------------------- | Data descriptors defined here: | | __dict__ | dictionary for instance variables (if defined) | | __weakref__ | list of weak references to the object (if defined) class Implementation(__builtin__.object) | Contains an abstraction over the DOM implementation. | | Methods defined here: | | adoptDocument(self, node) | | createDocument(self, namespaceURI, localName, doctype) | | createDocumentType(self, localName, publicId, systemId) | | get_node(self, _node, context_node) | | get_node_or_none(self, _node, context_node) | | ---------------------------------------------------------------------- | Data descriptors defined here: | | __dict__ | dictionary for instance variables (if defined) | | __weakref__ | list of weak references to the object (if defined) class NamedNodeMap(__builtin__.object) | A wrapper around Node objects providing DOM and dictionary convenience | methods. | | Methods defined here: | | __delitem__(self, name) | | __getitem__(self, name) | | __init__(self, node, impl) | | __iter__(self) | | __repr__(self) | | __setitem__(self, name, node) | | __str__(self) | | getNamedItem(self, name) | | getNamedItemNS(self, ns, localName) | | items(self) | | keys(self) | | removeNamedItem(self, name) | | removeNamedItemNS(self, ns, localName) | | setNamedItem(self, node) | | setNamedItemNS(self, node) | | values(self) | | ---------------------------------------------------------------------- | Data descriptors defined here: | | __dict__ | dictionary for instance variables (if defined) | | __weakref__ | list of weak references to the object (if defined) | | length class NamedNodeMapIterator(__builtin__.object) | An iterator over a NamedNodeMap. | | Methods defined here: | | __init__(self, nodemap) | | next(self) | | ---------------------------------------------------------------------- | Data descriptors defined here: | | __dict__ | dictionary for instance variables (if defined) | | __weakref__ | list of weak references to the object (if defined) class Node(__builtin__.object) | A DOM-style wrapper around libxml2mod objects. | | Methods defined here: | | __eq__(self, other) | | __hash__(self) | | __init__(self, node, impl=None, ownerDocument=None) | | __ne__(self, other) | | appendChild(self, tmp) | | as_native_node(self) | | cloneNode(self, deep) | | createAttribute(self, name) | | createAttributeNS(self, ns, name) | | createCDATASection(self, value) | | createComment(self, value) | | createElement(self, name) | | createElementNS(self, ns, name) | | createTextNode(self, value) | | getAttribute(self, name) | | getAttributeNS(self, ns, localName) | | getAttributeNode(self, localName) | | getAttributeNodeNS(self, ns, localName) | | getElementById(self, identifier) | | getElementsByTagName(self, tagName) | | getElementsByTagNameNS(self, namespaceURI, localName) | | hasAttribute(self, name) | | hasAttributeNS(self, ns, localName) | | importNode(self, node, deep) | | insertBefore(self, tmp, oldNode) | | isSameNode(self, other) | | normalize(self) | | removeAttribute(self, name) | | removeAttributeNS(self, ns, localName) | | removeChild(self, tmp) | | replaceChild(self, tmp, oldNode) | | setAttribute(self, name, value) | | setAttributeNS(self, ns, name, value) | | setAttributeNode(self, node) | | setAttributeNodeNS(self, node) | | toFile(self, f, encoding=None, prettyprint=0) | | toStream(self, stream, encoding=None, prettyprint=0) | | toString(self, encoding=None, prettyprint=0) | | xpath(self, expr, variables=None, namespaces=None) | | ---------------------------------------------------------------------- | Data descriptors defined here: | | __dict__ | dictionary for instance variables (if defined) | | __weakref__ | list of weak references to the object (if defined) | | attributes | | childNodes | | data | | doctype | | firstChild | | lastChild | | localName | | name | | namespaceURI | | nextSibling | | nodeName | | nodeType | | nodeValue | | parentNode | | prefix | | previousSibling | | publicId | | systemId | | tagName | | textContent | | value | | ---------------------------------------------------------------------- | Data and other attributes defined here: | | ATTRIBUTE_NODE = 2 | | COMMENT_NODE = 8 | | DOCUMENT_NODE = 9 | | DOCUMENT_TYPE_NODE = 10 | | ELEMENT_NODE = 1 | | ENTITY_NODE = 6 | | ENTITY_REFERENCE_NODE = 5 | | NOTATION_NODE = 12 | | PROCESSING_INSTRUCTION_NODE = 7 | | TEXT_NODE = 3 | | entities = {} | | notations = {} class NodeList(__builtin__.list) | A wrapper around node lists. | | Method resolution order: | NodeList | __builtin__.list | __builtin__.object | | Methods defined here: | | item(self, index) | | ---------------------------------------------------------------------- | Data descriptors defined here: | | __dict__ | dictionary for instance variables (if defined) | | __weakref__ | list of weak references to the object (if defined) | | length | | ---------------------------------------------------------------------- | Methods inherited from __builtin__.list: | | __add__(...) | x.__add__(y) <==> x+y | | __contains__(...) | x.__contains__(y) <==> y in x | | __delitem__(...) | x.__delitem__(y) <==> del x[y] | | __delslice__(...) | x.__delslice__(i, j) <==> del x[i:j] | | Use of negative indices is not supported. | | __eq__(...) | x.__eq__(y) <==> x==y | | __ge__(...) | x.__ge__(y) <==> x>=y | | __getattribute__(...) | x.__getattribute__('name') <==> x.name | | __getitem__(...) | x.__getitem__(y) <==> x[y] | | __getslice__(...) | x.__getslice__(i, j) <==> x[i:j] | | Use of negative indices is not supported. | | __gt__(...) | x.__gt__(y) <==> x>y | | __iadd__(...) | x.__iadd__(y) <==> x+=y | | __imul__(...) | x.__imul__(y) <==> x*=y | | __init__(...) | x.__init__(...) initializes x; see x.__class__.__doc__ for signature | | __iter__(...) | x.__iter__() <==> iter(x) | | __le__(...) | x.__le__(y) <==> x<=y | | __len__(...) | x.__len__() <==> len(x) | | __lt__(...) | x.__lt__(y) <==> x<y | | __mul__(...) | x.__mul__(n) <==> x*n | | __ne__(...) | x.__ne__(y) <==> x!=y | | __repr__(...) | x.__repr__() <==> repr(x) | | __reversed__(...) | L.__reversed__() -- return a reverse iterator over the list | | __rmul__(...) | x.__rmul__(n) <==> n*x | | __setitem__(...) | x.__setitem__(i, y) <==> x[i]=y | | __setslice__(...) | x.__setslice__(i, j, y) <==> x[i:j]=y | | Use of negative indices is not supported. | | __sizeof__(...) | L.__sizeof__() -- size of L in memory, in bytes | | append(...) | L.append(object) -- append object to end | | count(...) | L.count(value) -> integer -- return number of occurrences of value | | extend(...) | L.extend(iterable) -- extend list by appending elements from the iterable | | index(...) | L.index(value, [start, [stop]]) -> integer -- return first index of value. | Raises ValueError if the value is not present. | | insert(...) | L.insert(index, object) -- insert object before index | | pop(...) | L.pop([index]) -> item -- remove and return item at index (default last). | Raises IndexError if list is empty or index is out of range. | | remove(...) | L.remove(value) -- remove first occurrence of value. | Raises ValueError if the value is not present. | | reverse(...) | L.reverse() -- reverse *IN PLACE* | | sort(...) | L.sort(cmp=None, key=None, reverse=False) -- stable sort *IN PLACE*; | cmp(x, y) -> -1, 0, 1 | | ---------------------------------------------------------------------- | Data and other attributes inherited from __builtin__.list: | | __hash__ = None | | __new__ = <built-in method __new__ of type object at 0x822be40> | T.__new__(S, ...) -> a new object with type S, a subtype of T FUNCTIONS adoptNodes(nodes, impl=None) A special utility method which adopts the given low-level 'nodes' and which returns a list of high-level equivalents. This is currently experimental and should not be casually used. createDocument(namespaceURI, localName, doctype) createDocumentType(localName, publicId, systemId) getDOMImplementation() Return the default DOM implementation. parse(stream_or_string, html=0, htmlencoding=None, unfinished=0, impl=None) Parse the given 'stream_or_string', where the supplied object can either be a stream (such as a file or stream object), or a string (containing the filename of a document). The optional parameters described below should be provided as keyword arguments. If the optional 'html' parameter is set to a true value, the content to be parsed will be treated as being HTML rather than XML. If the optional 'htmlencoding' is specified, HTML parsing will be performed with the document encoding assumed to that specified. If the optional 'unfinished' parameter is set to a true value, unfinished documents will be parsed, even though such documents may be missing content such as closing tags. A document object is returned by this function. parseFile(filename, html=0, htmlencoding=None, unfinished=0, impl=None) Parse the file having the given 'filename'. The optional parameters described below should be provided as keyword arguments. If the optional 'html' parameter is set to a true value, the content to be parsed will be treated as being HTML rather than XML. If the optional 'htmlencoding' is specified, HTML parsing will be performed with the document encoding assumed to that specified. If the optional 'unfinished' parameter is set to a true value, unfinished documents will be parsed, even though such documents may be missing content such as closing tags. A document object is returned by this function. parseString(s, html=0, htmlencoding=None, unfinished=0, impl=None) Parse the content of the given string 's'. The optional parameters described below should be provided as keyword arguments. If the optional 'html' parameter is set to a true value, the content to be parsed will be treated as being HTML rather than XML. If the optional 'htmlencoding' is specified, HTML parsing will be performed with the document encoding assumed to that specified. If the optional 'unfinished' parameter is set to a true value, unfinished documents will be parsed, even though such documents may be missing content such as closing tags. A document object is returned by this function. parseURI(uri, html=0, htmlencoding=None, unfinished=0, impl=None) Parse the content found at the given 'uri'. The optional parameters described below should be provided as keyword arguments. If the optional 'html' parameter is set to a true value, the content to be parsed will be treated as being HTML rather than XML. If the optional 'htmlencoding' is specified, HTML parsing will be performed with the document encoding assumed to that specified. If the optional 'unfinished' parameter is set to a true value, unfinished documents will be parsed, even though such documents may be missing content such as closing tags. XML documents are retrieved using libxml2's own network capabilities; HTML documents are retrieved using the urllib module provided by Python. To retrieve either kind of document using Python's own modules for this purpose (such as urllib), open a stream and pass it to the parse function: f = urllib.urlopen(uri) try: doc = libxml2dom.parse(f, html) finally: f.close() A document object is returned by this function. toFile(node, filename, encoding=None, prettyprint=0) Write the serialised form of the given 'node' and its children to a file having the given 'filename'. The optional 'encoding' can be used to override the default character encoding used in the serialisation. The optional 'prettyprint' indicates whether the serialised form is prettyprinted or not (the default setting). toStream(node, stream, encoding=None, prettyprint=0) Write the serialised form of the given 'node' and its children to the given 'stream'. The optional 'encoding' can be used to override the default character encoding used in the serialisation. The optional 'prettyprint' indicates whether the serialised form is prettyprinted or not (the default setting). toString(node, encoding=None, prettyprint=0) Return a string containing the serialised form of the given 'node' and its children. The optional 'encoding' can be used to override the default character encoding used in the serialisation. The optional 'prettyprint' indicates whether the serialised form is prettyprinted or not (the default setting). DATA HTML_PARSE_NOERROR = 32 HTML_PARSE_NONET = 2048 HTML_PARSE_NOWARNING = 64 XML_NAMESPACE = 'http://www.w3.org/XML/1998/namespace' XML_PARSE_NOERROR = 32 XML_PARSE_NONET = 2048 XML_PARSE_NOWARNING = 64 __version__ = '0.4.5' default_impl = <libxml2dom.Implementation object at 0xb748e0ec> default_ns = {'xml': 'http://www.w3.org/XML/1998/namespace'} label = 'pi' null_value_node_types = [9, 10, 1, 6, 5, 12] value = 7 VERSION 0.4.5 声明：ITeye文章版权属于作者，受法律保护。没有作者书面许可不得转载。
推荐链接
|
| 返回顶楼 | |
|
发表时间：2010-04-14
谢谢分享经验
不过以后还是希望能把结论放到明显的地方。方便大家阅读。 |
| 返回顶楼 | |
