Solr Schema XML DTD校验

mozhenghua

浏览: 324530 次
性别:
来自: 杭州

最近访客更多访客>>

huang_love_ok

wang_eye

贝铃-Turing

joechl

博主相关

博客

微博

相册

留言

关于我

文章分类

社区版块

存档分类

博客分类：

Solr Schema XML DTD validate

Solr Schema XML DTD validate

由于编辑solr的schema文件的过程中，时常会发生一些小错误，类似于，“true” 写成了“ture”

或者“multiValued”写成“multivalue”，虽然是手误但是往往会造成很诡异的错误，并且非常难排查错误。类似这样的笔误导致在生产环境中的bug，非常隐蔽不容易被发现，排除这样的错误经常要花上半天的时间或者更长。

解决这个问题的办法是通过在编辑后执行文档Schema校验的方式来校验文档的正确行，校验方式有两种一种是DTD，另外一种方式是Schema,就是XSD（XML Schema Definition)这种方式对XML校验更加完善，可以对属性进行类型校验。

简单起见，对schema.xml 利用DTD方式进行校验。以下是我针对solr的schema文件写DTD文件：

<!ELEMENT schema (types+,fields+,uniqueKey,defaultSearchField,similarity?,solrQueryParser,copyField*)>

<!ATTLIST schema name CDATA #REQUIRED>
<!ATTLIST schema version CDATA #REQUIRED>

<!ELEMENT types (fieldType+)>
<!ELEMENT fieldType (analyzer*)>

<!ATTLIST fieldType name CDATA #REQUIRED>
<!ATTLIST fieldType class CDATA #REQUIRED>
<!ATTLIST fieldType positionIncrementGap CDATA #IMPLIED> 
<!ATTLIST fieldType sortMissingLast (true|false) "true">
<!ATTLIST fieldType omitNorms (true|false) "true">
<!ATTLIST fieldType precisionStep CDATA #IMPLIED>
<!ATTLIST fieldType autoGeneratePhraseQueries (true|false) #IMPLIED>
<!ATTLIST fieldType tokenized (true|false) #IMPLIED>
<!ATTLIST fieldType omitTermFreqAndPositions (true|false) #IMPLIED>



<!ELEMENT analyzer (tokenizer*,filter*)>
<!ATTLIST analyzer 
  type (index|query) #IMPLIED
  class CDATA        #IMPLIED>

<!ELEMENT tokenizer EMPTY>
<!ATTLIST tokenizer 
   class CDATA #REQUIRED
   group CDATA #IMPLIED
   pattern CDATA #IMPLIED>

<!ELEMENT filter EMPTY>
<!ATTLIST filter class CDATA #REQUIRED>

<!ELEMENT uniqueKey (#PCDATA)>
<!ELEMENT defaultSearchField (#PCDATA)>

<!ELEMENT similarity (str*)>
<!ATTLIST similarity class CDATA #REQUIRED>

<!ELEMENT str (#PCDATA)>
<!ATTLIST str name CDATA #REQUIRED>


<!ELEMENT solrQueryParser EMPTY>
<!ATTLIST solrQueryParser defaultOperator (OR|AND) #REQUIRED>

<!ELEMENT fields (field|dynamicField)+>
<!ELEMENT field  EMPTY>
<!ATTLIST field  name CDATA #REQUIRED>
<!ATTLIST field  type CDATA #REQUIRED>
<!ATTLIST field  indexed (true|false) #IMPLIED>
<!ATTLIST field  stored (true|false) #IMPLIED>
<!ATTLIST field  multiValued (true|false) #IMPLIED>
<!ATTLIST field  required (true|false) #IMPLIED>
<!ATTLIST field  default CDATA #IMPLIED>

<!ELEMENT dynamicField EMPTY>
<!ATTLIST dynamicField  name CDATA #REQUIRED>
<!ATTLIST dynamicField  type CDATA #REQUIRED>
<!ATTLIST dynamicField  indexed (true|false) #IMPLIED>
<!ATTLIST dynamicField  stored (true|false) #IMPLIED>
<!ATTLIST dynamicField  multiValued (true|false) #IMPLIED>
<!ATTLIST dynamicField  required (true|false) #IMPLIED>

<!ELEMENT copyField EMPTY>
<!ATTLIST copyField  source CDATA #REQUIRED>
<!ATTLIST copyField  dest CDATA #REQUIRED>

然后就可以用这个dtd来校验XML文件了，以下是一个schema XML文件样例：

<?xml version="1.0" ?>
 <!DOCTYPE schema SYSTEM  "solrschema.dtd">
<schema name="example core" version="1.1">
	<types>
		<fieldType name="string" class="solr.TextField"
			sortMissingLast="true"></fieldType>

		<fieldType name="int" class="solr.TrieIntField"
			precisionStep="0" positionIncrementGap="0" />

		<fieldType name="tlong" class="solr.TrieLongField"
			precisionStep="8" omitNorms="true" positionIncrementGap="0" />

		<fieldType name="sint" class="solr.SortableIntField"
			sortMissingLast="true" omitNorms="true" />

		<fieldType name="slong" class="solr.SortableLongField"
			sortMissingLast="true" omitNorms="true" />

		<fieldType name="string_ws" class="solr.TextField"
			positionIncrementGap="100">
			<analyzer>
				<tokenizer class="solr.WhitespaceTokenizerFactory" />
			</analyzer>
		</fieldType>

		<fieldType name="string_sd" class="solr.TextField"
			sortMissingLast="true" omitNorms="true">
			<analyzer type="index">
				<tokenizer class="solr.StandardTokenizerFactory" />
				<filter class="solr.ChineseFilterFactory" />
				<filter class="solr.LowerCaseFilterFactory" />
			</analyzer>
			<analyzer type="query">
				<tokenizer class="solr.StandardTokenizerFactory" />
				<filter class="solr.ChineseFilterFactory" />
				<filter class="solr.LowerCaseFilterFactory" />
			</analyzer>

		</fieldType>

		<fieldType name="tint" class="solr.TrieIntField"
			precisionStep="8" omitNorms="true" positionIncrementGap="0" />
		<fieldType name="tfloat" class="solr.TrieFloatField"
			precisionStep="8" omitNorms="true" positionIncrementGap="0" />
		<fieldType name="sfloat" class="solr.SortableFloatField"
			sortMissingLast="true" omitNorms="true" />

	</types>

	<fields>
		<field name="id" type="slong" indexed="true" stored="true"
			multiValued="false" required="true" />
		<field name="userId" type="slong" indexed="true" stored="true"
			multiValued="false" />
		<field name="userNick" type="string" indexed="true" stored="true"
			multiValued="false" />
		<field name="title" type="string_sd" indexed="true" stored="true"
			multiValued="false" />
		<field name="contentKey" type="string" indexed="true" stored="true"
			multiValued="false" />
		<field name="coverPicpath" type="string" indexed="true" stored="true"
			multiValued="false" />
		<field name="itemId" type="string" indexed="true" stored="true"
			multiValued="false" />
		<field name="gmtCreated" type="slong" indexed="true" stored="true"
			multiValued="false" />
		<field name="gmtModified" type="slong" indexed="true" stored="true"
			multiValued="false" />
		<field name="tag" type="string_sd" indexed="true" stored="true"
			multiValued="false" />
		<field name="aScore" type="sfloat" indexed="true" stored="true"
			multiValued="false" />
		<field name="count" type="int" indexed="true" stored="true"
			multiValued="false" />
		<field name="feedId" type="string" stored="true" multiValued="false" />
		<field name="summary" type="string_sd" indexed="true" stored="true"
			multiValued="false" />
		<field name="status" type="int" indexed="true" stored="true"
			multiValued="false" />

		<dynamicField name="*-t" type="text_com" stored="false"
			indexed="true" />

	</fields>

	<uniqueKey>id</uniqueKey>
	<defaultSearchField>id</defaultSearchField>

	<similarity class="com.taobao.terminator.core.solrx.PayloadSimilarity">
		<str name="func">max</str>
	</similarity>

	<solrQueryParser defaultOperator="OR" />
	<copyField source="prov" dest="location" />
</schema>

写一个java的单元测试类来校验，XML的正确性：

import java.io.InputStream;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;

import junit.framework.Assert;
import junit.framework.TestCase;

import org.xml.sax.SAXException;
import org.xml.sax.SAXParseException;
import org.xml.sax.helpers.DefaultHandler;

/**
 * @author 百岁（baisui@taobao.com）
 * @date 2012-11-28
 */
public class TestSchemaDTDValidate extends TestCase {
	private final static DocumentBuilderFactory factory = DocumentBuilderFactory
			.newInstance();
	static {
		factory.setValidating(true);
	}

	public void testFile1() {

		Assert.assertTrue("schema3.xml", XMLValidateDTD("schema3.xml"));

	}

	public boolean XMLValidateDTD(String file) {

		MyHandler mh = new MyHandler();
		InputStream inputStream = null;
		try {
			DocumentBuilder builder = factory.newDocumentBuilder();
			builder.setErrorHandler(mh);

			inputStream = this.getClass().getResourceAsStream("/" + file);

			builder.parse(inputStream);
		} catch (Exception e) {

			Assert.assertFalse(e.getMessage(), true);

			return false;
		} finally {
			try {
				inputStream.close();
			} catch (Throwable e) {

			}
		}
		return mh.validate;
	}

	static class MyHandler extends DefaultHandler {
		String errorMessage = null;

		private boolean validate = true;

		public void error(SAXParseException e) throws SAXException {
			errorMessage = e.getMessage();

			System.out.println(errorMessage + " linenumber:"
					+ e.getLineNumber());

			Assert.assertFalse(errorMessage, true);

			validate = false;

		}

		public void fatalError(SAXParseException e) throws SAXException {
			errorMessage = e.getMessage();
			System.out.println(errorMessage);
			validate = false;
			Assert.assertFalse(errorMessage + " linenumber:"
					+ e.getLineNumber(), true);
		}
	}

}

使用DTD来校验XML有以下不足：

子元素在排列顺序上有限制，是实际的xml文档中，定义的元素节点，必须要按照DTD中定义的顺序，哪怕这几个子元素在逻辑上并没有逻辑以来关系，比如，以上xml中<uniqueKey/>必须在<defaultSearchField/> 之前，如果不是这样的话校验XML过程中就会校验失败。
不能对节点属性的类型进行校验，比如,fieldType节点的precisionStep属性必须为数字，在实际中如果用户填写了非数字的字符作为的属性的话，校验也会通过的。这样就会发生潜在的错误。

虽然现在使用DTD有以上不足，但是相对以前完全没有校验机制来说已经进步一大截了，可以帮助我们规避掉很大一部分不必要的麻烦问题。如果想要弥补以上不足的话可以尝试一下XSD的校验机制。

分享到：