云计算实战 （海量日志管理）hadoop + scribe -- log4j 客户端写入scribe

 

上一篇文章已经安装完scribe，下面我们用java端，通过log4j 把日志写入scribe 日志系统。

 

 

一、生成scribe客户端

• 生成 java客户端api

   A. 修改配置文件scribe.thrift
      cd /usr/local/scribeInstall/scribe/if
      vi scribe.thrift
      修改scribe.thrift文件: 把 include "fb303/if/fb303.thrift" 改成
      include "[thrift解压路径]/thrift-0.5.0/contrib/fb303/if/fb303.thrift"
   B. 生成 java客户端api     
      运行命令 thrift --gen java scribe.thrift
      运行后会生成一个‘gen-java’的文件夹.里面会有3个java类，封装了所有java客户端发送log所需要的api。

 

 

• 生成客户端代码所需要的jar包

   A. 如果之前没设置ANT_HOME 和 PATH 请先设置这两个环境变量
      export ANT_HOME=/usr/local/apache-ant-1.8.0
      export PATH=$PATH:$ANT_HOME/bin
   B. 生成libthrift.jar
      cd /usr/local/scribeInstall/thrift-0.2.0/lib/java
      ant 
      (如果没有错误在本文夹夹下会生成libthrift.jar)
   C. 生成libfb303.jar

 

 cd /usr/local/scribeInstall/thrift-0.2.0/contrib/fb303/java

ant

ant 执行成功后 libfb303.jar 会出现在/usr/local/scribeInstall/contrib/fb303/java/build/lib下

 

 

二、创建项目，运行测试 

• 创建项目

     A.在eclipse 创建普通java项目

     B.在项目中导入以下jar

     

    C.在项目中添加gen-java 文件夹里的三个java类。

•    编写项目

    编写log4j 的scribe appender 

    AsyncScribeAppender.java:

    

package com.logtest;

import org.apache.log4j.AsyncAppender;
/**
 * log4j 的scribe appender
 * 用ScribeAppender 类连接scribe服务器，并把日志写如scribe
 * @author ninja
 */
public class AsyncScribeAppender extends AsyncAppender {

	private String hostname;
	private String scribeHost;
	private int scribePort;
	private String scribeCategory;
	private String encoading;

	public String getHostname() {
		return hostname;
	}

	public void setHostname(String hostname) {
		this.hostname = hostname;
	}

	public String getScribeHost() {
		return scribeHost;
	}

	public void setScribeHost(String scribeHost) {
		this.scribeHost = scribeHost;
	}

	public int getScribePort() {
		return scribePort;
	}

	public void setScribePort(int scribePort) {
		this.scribePort = scribePort;
	}

	public String getScribeCategory() {
		return scribeCategory;
	}

	public void setScribeCategory(String scribeCategory) {
		this.scribeCategory = scribeCategory;
	}

	public String getEncoading() {
		return encoading;
	}

	public void setEncoading(String encoading) {
		this.encoading = encoading;
	}

	@Override
	public void activateOptions() {
		super.activateOptions();
		synchronized (this) {
			ScribeAppender scribeAppender = new ScribeAppender();
			scribeAppender.setLayout(getLayout());
			scribeAppender.setHostname(getHostname());
			scribeAppender.setScribeHost(getScribeHost());
			scribeAppender.setScribePort(getScribePort());
			scribeAppender.setScribeCategory(getScribeCategory());
			scribeAppender.setEncoding(getEncoading());
			scribeAppender.activateOptions();
			addAppender(scribeAppender);
		}
	}

	@Override
	public boolean requiresLayout() {
		return true;
	}

}

 

 ScribeAppender .java

 

package com.logtest;

import net.scribe.LogEntry;
import net.scribe.scribe;

import org.apache.log4j.WriterAppender;
import org.apache.log4j.spi.LoggingEvent;
import org.apache.thrift.protocol.TBinaryProtocol;
import org.apache.thrift.transport.TFramedTransport;
import org.apache.thrift.transport.TSocket;
import org.apache.thrift.transport.TTransportException;

import java.util.List;
import java.util.ArrayList;
import java.net.Socket;
import java.net.UnknownHostException;
import java.net.InetAddress;
import java.io.IOException;

/**
 * 继承WriterAppender 
 * 实现了scribe 服务器的链接和日志的发送。
 * @author ninja
 */
public class ScribeAppender extends WriterAppender {

	private String hostname;
	private String scribeHost;
	private int scribePort;
	private String scribeCategory;
	private String encoding;

	private List<LogEntry> logEntries;

	private scribe.Client client;
	private TFramedTransport transport;

	public String getHostname() {
		return hostname;
	}

	public void setHostname(String hostname) {
		this.hostname = hostname;
	}

	public String getScribeHost() {
		return scribeHost;
	}

	public void setScribeHost(String scribeHost) {
		this.scribeHost = scribeHost;
	}

	public int getScribePort() {
		return scribePort;
	}

	public void setScribePort(int scribePort) {
		this.scribePort = scribePort;
	}

	public String getScribeCategory() {
		return scribeCategory;
	}

	public void setScribeCategory(String scribeCategory) {
		this.scribeCategory = scribeCategory;
	}
	

	public String getEncoding() {
		return encoding;
	}

	public void setEncoding(String encoding) {
		this.encoding = encoding;
	}

	/*
	 * Activates this Appender by opening a transport to the Scribe server.
	 */
	@Override
	public void activateOptions() {
		try {
			synchronized (this) {
				if (hostname == null) {
					try {
						hostname = InetAddress.getLocalHost()
								.getCanonicalHostName();
					} catch (UnknownHostException e) {
						// can't get hostname
					}
				}
		System.out.println(scribeHost + scribePort + scribeCategory + encoding);
				// Thrift boilerplate code
				logEntries = new ArrayList<LogEntry>(1);
				TSocket sock = new TSocket(new Socket(scribeHost, scribePort));
				transport = new TFramedTransport(sock);
				TBinaryProtocol protocol = new TBinaryProtocol(transport,
						false, false);
				client = new scribe.Client(protocol, protocol);
				// This is commented out because it was throwing Exceptions for
				// no good reason.
				// transport.open();
			}
		} catch (TTransportException e) {
			e.printStackTrace();
		} catch (UnknownHostException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		} catch (Exception e) {
			e.printStackTrace();
		}
	}

	/*
	 * Appends a log message to Scribe
	 */
	@Override
	public void append(LoggingEvent event) {
		synchronized (this) {
			try {
				String message = String.format("%s %s", hostname, layout
						.format(event));
				LogEntry entry = new LogEntry(scribeCategory, message);
				logEntries.add(entry);
				client.Log(logEntries);
			} catch (Exception e) {
				e.printStackTrace();
			} finally {
				logEntries.clear();
			}
		}
	}

	@Override
	public void close() {
		if (transport != null) {
			transport.close();
		}
	}

	@Override
	public boolean requiresLayout() {
		return true;
	}
}

 

 

   log4j.properties

 

#1 \u5b9a\u4e49\u4e86\u4e24\u4e2a\u8f93\u51fa\u7aef
log4j.rootLogger = DEBUG,CONSOLE,scribe
log4j.addivity.org.apache=true

log4j.appender.CONSOLE = org.apache.log4j.ConsoleAppender
log4j.appender.CONSOLE.layout = org.apache.log4j.PatternLayout
log4j.appender.CONSOLE.layout.ConversionPattern = %-4r [%t] %-5p %c - %m%n

log4j.logger.com.vmars= DEBUG, scribe
log4j.appender.scribe= com.logtest.AsyncScribeAppender
log4j.appender.scribe.encoading=utf-8
log4j.appender.scribe.hostname=scribe
log4j.appender.scribe.scribeHost=192.168.2.221
log4j.appender.scribe.scribePort=1463
log4j.appender.scribe.hostname=ninja
log4j.appender.scribe.scribeCategory=scribe
log4j.appender.scribe.layout=org.apache.log4j.PatternLayout
log4j.appender.scribe.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n

 

 

   测试客户端：

 

package com.logtest;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;


public class LogTest {
	
	private static Log log = LogFactory.getLog(LogTest.class);
	
	public static void main(String[] args) {
		log.error("this is a charactor test ");
		log.debug("这是中文测试");
		log.fatal("fatal error 致命错误！！");
	}
}

 

 

所有资源均在resources.rar 中。

评论

8 楼 san_yun 2013-11-28  

这个相当于应该写到本地磁盘的日志写到了远程服务器，性能很差把，日志是同步写还是异步写？

7 楼 di1984HIT 2013-02-18  

写的不错学习了

6 楼 woqidaideshi 2012-11-30  

我的为什么一直报错说是连接失败呢？connection time out？

5 楼 houzhaowei 2012-03-05  

vicegoytd 写道

是啊，这个scribe收集日志存放在哪里，在/tmp/scribetext 里没有啊

这目录在scribe里配

4 楼 vicegoytd 2012-03-02  

是啊，这个scribe收集日志存放在哪里，在/tmp/scribetext 里没有啊

3 楼 remexl 2011-04-20  

请问楼主，scribe收集的日志是放在哪个目录下面的？

2 楼 houzhaowei 2010-11-03  

uuuvvv 写道

请教lz，这样做到底有哪些好处呢？

scribe 是分布式日志系统 能更好的处理海量日志. facebook 就是用的scribe.

1 楼 uuuvvv 2010-11-03  

请教lz，这样做到底有哪些好处呢？