使用SolrJ生成索引

代码很简单, 直接看就明白了, 可以在实际工作中借鉴, 原文在这里. 这个例子使用两种方式来演示如何生成全量索引:
一个是从db中通过sql生成全量索引
一个是通过tika解析文件生成全量索引

package SolrJExample;

import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.StreamingUpdateSolrServer;
import org.apache.solr.client.solrj.impl.XMLResponseParser;
import org.apache.solr.client.solrj.response.UpdateResponse;
import org.apache.solr.common.SolrInputDocument;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.parser.AutoDetectParser;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.sax.BodyContentHandler;
import org.xml.sax.ContentHandler;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.sql.*;
import java.util.ArrayList;
import java.util.Collection;

/* Example class showing the skeleton of using Tika and
   Sql on the client to index documents from
   both structured documents and a SQL database.

   NOTE: The SQL example and the Tika example are entirely orthogonal.
   Both are included here to make a
   more interesting example, but you can omit either of them.

 */
public class SqlTikaExample {
  private StreamingUpdateSolrServer _server;
  private long _start = System.currentTimeMillis();
  private AutoDetectParser _autoParser;
  private int _totalTika = 0;
  private int _totalSql = 0;

  private Collection _docs = new ArrayList();

  public static void main(String[] args) {
    try {
      SqlTikaExample idxer = new SqlTikaExample("http://localhost:8983/solr");

      idxer.doTikaDocuments(new File("/Users/Erick/testdocs"));
      idxer.doSqlDocuments();

      idxer.endIndexing();
    } catch (Exception e) {
      e.printStackTrace();
    }
  }

  private SqlTikaExample(String url) throws IOException, SolrServerException {
      // Create a multi-threaded communications channel to the Solr server.
      // Could be CommonsHttpSolrServer as well.
      //
    _server = new StreamingUpdateSolrServer(url, 10, 4);

    _server.setSoTimeout(1000);  // socket read timeout
    _server.setConnectionTimeout(1000);
    _server.setMaxRetries(1); // defaults to 0.  > 1 not recommended.
         // binary parser is used by default for responses
    _server.setParser(new XMLResponseParser()); 

      // One of the ways Tika can be used to attempt to parse arbitrary files.
    _autoParser = new AutoDetectParser();
  }

    // Just a convenient place to wrap things up.
  private void endIndexing() throws IOException, SolrServerException {
    if (_docs.size() > 0) { // Are there any documents left over?
      _server.add(_docs, 300000); // Commit within 5 minutes
    }
    _server.commit(); // Only needs to be done at the end,
                      // commitWithin should do the rest.
                      // Could even be omitted
                      // assuming commitWithin was specified.
    long endTime = System.currentTimeMillis();
    log("Total Time Taken: " + (endTime - _start) +
         " milliseconds to index " + _totalSql +
        " SQL rows and " + _totalTika + " documents");
  }

  // I hate writing System.out.println() everyplace,
  // besides this gives a central place to convert to true logging
  // in a production system.
  private static void log(String msg) {
    System.out.println(msg);
  }

  /**
   * ***************************Tika processing here
   */
  // Recursively traverse the filesystem, parsing everything found.
  private void doTikaDocuments(File root) throws IOException, SolrServerException {

    // Simple loop for recursively indexing all the files
    // in the root directory passed in.
    for (File file : root.listFiles()) {
      if (file.isDirectory()) {
        doTikaDocuments(file);
        continue;
      }
        // Get ready to parse the file.
      ContentHandler textHandler = new BodyContentHandler();
      Metadata metadata = new Metadata();
      ParseContext context = new ParseContext();

      InputStream input = new FileInputStream(file);

        // Try parsing the file. Note we haven't checked at all to
        // see whether this file is a good candidate.
      try {
        _autoParser.parse(input, textHandler, metadata, context);
      } catch (Exception e) {
          // Needs better logging of what went wrong in order to
          // track down "bad" documents.
        log(String.format("File %s failed", file.getCanonicalPath()));
        e.printStackTrace();
        continue;
      }
      // Just to show how much meta-data and what form it's in.
      dumpMetadata(file.getCanonicalPath(), metadata);

      // Index just a couple of the meta-data fields.
      SolrInputDocument doc = new SolrInputDocument();

      doc.addField("id", file.getCanonicalPath());

      // Crude way to get known meta-data fields.
      // Also possible to write a simple loop to examine all the
      // metadata returned and selectively index it and/or
      // just get a list of them.
      // One can also use the LucidWorks field mapping to
      // accomplish much the same thing.
      String author = metadata.get("Author");

      if (author != null) {
        doc.addField("author", author);
      }

      doc.addField("text", textHandler.toString());

      _docs.add(doc);
      ++_totalTika;

      // Completely arbitrary, just batch up more than one document
      // for throughput!
      if (_docs.size() >= 1000) {
          // Commit within 5 minutes.
        UpdateResponse resp = _server.add(_docs, 300000);
        if (resp.getStatus() != 0) {
          log("Some horrible error has occurred, status is: " +
                  resp.getStatus());
        }
        _docs.clear();
      }
    }
  }

    // Just to show all the metadata that's available.
  private void dumpMetadata(String fileName, Metadata metadata) {
    log("Dumping metadata for file: " + fileName);
    for (String name : metadata.names()) {
      log(name + ":" + metadata.get(name));
    }
    log("\n\n");
  }

  /**
   * ***************************SQL processing here
   */
  private void doSqlDocuments() throws SQLException {
    Connection con = null;
    try {
      Class.forName("com.mysql.jdbc.Driver").newInstance();
      log("Driver Loaded......");

      con = DriverManager.getConnection("jdbc:mysql://192.168.1.103:3306/test?"
                + "user=testuser&password=test123");

      Statement st = con.createStatement();
      ResultSet rs = st.executeQuery("select id,title,text from test");

      while (rs.next()) {
        // DO NOT move this outside the while loop
        // or be sure to call doc.clear()
        SolrInputDocument doc = new SolrInputDocument(); 
        String id = rs.getString("id");
        String title = rs.getString("title");
        String text = rs.getString("text");

        doc.addField("id", id);
        doc.addField("title", title);
        doc.addField("text", text);

        _docs.add(doc);
        ++_totalSql;

        // Completely arbitrary, just batch up more than one
        // document for throughput!
        if (_docs.size() > 1000) {
             // Commit within 5 minutes.
          UpdateResponse resp = _server.add(_docs, 300000);
          if (resp.getStatus() != 0) {
            log("Some horrible error has occurred, status is: " +
                  resp.getStatus());
          }
          _docs.clear();
        }
      }
    } catch (Exception ex) {
      ex.printStackTrace();
    } finally {
      if (con != null) {
        con.close();
      }
    }
  }
}

评论

2 楼 macrochen 2012-02-27  

huangfoxAgain 写道

请问在3.x的solr中怎么解决“实时检索”的问题呢?
这里采用的应该是hard commit~

这里是针对全量索引.

实时检索可以参考Sensei

然后定时或者主动更新索引

1 楼 huangfoxAgain 2012-02-25  

请问在3.x的solr中怎么解决“实时检索”的问题呢?
这里采用的应该是hard commit~