NoSql存儲日志數據之Spring+Logback+Hbase深度集成
關鍵詞：nosql, spring logback, logback hbase appender
技術框架：spring-data-hadoop, logback
一些比較重要的日志信息需要經常查看，比如用戶行為日志，報錯或關鍵業務日志數據然而同一系統多結點運行時這個工作會變的非常繁瑣。
本例借用Logback日志框架和Hbase數據庫來解決這一問題。
主要功能：
- 所有結點日志數據可通過配置同步到一個Hbase數據庫
- 與Spring整合，全局共享一個Hbase操作實例，動態為某日志添加Appender
- 存儲的日志數據可指定日志和日志級別，日志過濾
- Key-Value方式存儲，可指定Value的生成格式
Hbase的操作采用的是Spring-Hadoop中Hbase部分實現，沒有直接引用Spring-Hadoop而是只提取Hbase實現部分，原因是Hbase只是其中很小一部分又不想自己封裝，因此直接提取使用，封裝功能包括Hbase配置和HbaseTemplate等，相關源碼請參考Spring-Data-Hadoop下Hbase部分！
關於Logback沒有直接使用Logback的配置文件來配置HbaseAppender的原因是想與Spring整合使用Spring的Hbase-bean實例，不用單為Logback建一個Hbase實現等等。
使用相關配置
本文采用的NoSql數據庫為Hbase，並且只有Hbase的Logback Appender實現類，可參考實現其它類型的Appender。
Logback Nosql Appender工廠類
在這里指定用哪種類型的NoSql數據庫操作對像和對應的LOGBACK APPENDER操作類。
<beanid="logbackNosqlFactory"class="b2gonline.wap.logback.NosqlAppenderFactoryBean"><!--數據庫操作類--><propertyname="template"ref="hbaseTemplate"/><!--日志存儲表名--><propertyname="tbname"value="waplogdata"/><!--Logback Appender 類全路徑--><propertyname="appenderPaht"value="b2gonline.wap.logback.hbase.HbaseAppender"/></bean>Logback Appender配置
在這里配置LOGGER和APPENDER的對應關系和詳細的APPENDER配置，引用的APPENDER從上面配置的Logback Nosql Appender工廠類獲得。
<!--單個LOGGER配置--><beanid="hbaseappender1"class="b2gonline.wap.logback.SpringLogbackBean"lazy-init="false"><propertyname="appenderName"value="hbaseappender1"/><propertyname="logLevel"value="INFO"/><propertyname="filterLevel"value="INFO"/><propertyname="logName"value="logbackhbaseappendertest"/><propertyname="appender"ref="logbackNosqlFactory"/></bean><!--多個LOGGER配置--><beanid="hbaseappender2"class="b2gonline.wap.logback.SpringLogbackBean"lazy-init="false"><propertyname="appenderName"value="hbaseappender2"/><propertyname="filterLevel"value="WARN"/><propertyname="logName"><map><entrykey="logbackhbaseappendertest2"value="WARN"/><entrykey="logbackhbaseappendertest3"value="WARN"/></map></property><propertyname="appender"ref="logbackNosqlFactory"/></bean>JAVA代碼實現
NosqlAppenderFactoryBean
不同NOSQL數據庫實現的LOGBACK APPENDER工廠類
import b2gonline.wap.logback.hbase.HbaseAppender;import org.springframework.beans.factory.FactoryBean;import org.springframework.util.Assert;/** * NosqlAppenderFactoryBean工廠類，根據Nosql類型返回實例 */publicclassNosqlAppenderFactoryBeanimplementsFactoryBean<NoSqlAppender>{privateObject template;privateString tbname;privateString appenderPaht;/** * Appender類路徑，實例化不同類型Appender實例 * @param appender */publicvoid setAppenderPaht(String appender){this.appenderPaht = appender;}/** * 指定類型數據庫操作類 * @param template */publicvoid setTemplate(Object template){this.template = template;}/** * 數據存儲表名 * @param tbname */publicvoid setTbname(String tbname){this.tbname = tbname;}/** * 根據數據庫類型返回Appender實例 * * @return * @throws Exception */@OverridepublicNoSqlAppender getObject()throwsException{//校驗配置Assert.notNull(template);Assert.notNull(appenderPaht);Assert.notNull(tbname);//生成實例Class<?> appenderClass =Class.forName(appenderPaht);StringSCname= appenderClass.getSuperclass().getSimpleName();if(SCname.equals("NoSqlAppender")){NoSqlAppender hbaseAppender =(NoSqlAppender) appenderClass.newInstance(); hbaseAppender.setTemplate(template); hbaseAppender.setTbname(tbname);return hbaseAppender;}else{thrownewIllegalArgumentException(appenderPaht +"is not NoSqlAppender subclass!");}}@OverridepublicClass<?> getObjectType(){returnHbaseAppender.class;}@Overridepublicboolean isSingleton(){returnfalse;}}SpringLogbackBean
Logback Appender與Spring整合類，參考上面第二部分配置
import ch.qos.logback.classic.Level;import ch.qos.logback.classic.Logger;import ch.qos.logback.classic.LoggerContext;import ch.qos.logback.classic.filter.LevelFilter;import ch.qos.logback.core.spi.FilterReply;import org.slf4j.LoggerFactory;import org.springframework.beans.factory.InitializingBean;import org.springframework.util.Assert;import java.util.Map;/** * Logback Appender與Spring整合類 */publicclassSpringLogbackBeanimplementsInitializingBean{ org.slf4j.Logger _logger =LoggerFactory.getLogger(this.getClass());privateLevel logLevel =Level.INFO;privateString appenderName ="NoSqlAppender";privateNoSqlAppender appender;privateObject logName ="root";privateLevel filterLevel =Level.INFO;privateboolean useFilterLevel =true;privateboolean additiveAppender =true;privateString pattern ="%d{HH:mm:ss} [%thread] %-5level %logger{36} - %msg%n";/** * Appender 名稱 * * @param appenderName */publicvoid setAppenderName(String appenderName){this.appenderName = appenderName;}/** * 設置存儲的日志級別，默認是INFO * * @param logLevel */publicvoid setLogLevel(Level logLevel){this.logLevel = logLevel;}/** * 設置Logback appender * * @param appender */publicvoid setAppender(NoSqlAppender appender){this.appender = appender;}/** * 設置需要記錄的日志名，默認是root * * @param logName */publicvoid setLogName(Object logName){this.logName = logName;}/** * 使用過濾器過濾的日志級別，默認INFO * * @param filterLevel */publicvoid setFilterLevel(Level filterLevel){this.filterLevel = filterLevel;}/** * 是否累加Appender(繼承root appender)，默認 true * * @param additiveAppender */publicvoid setAdditiveAppender(boolean additiveAppender){this.additiveAppender = additiveAppender;}/** * 是否使用日志過濾，其它級別日志數據交給繼承來的APPENDER處理 * * @param useFilterLevel */publicvoid setUseFilterLevel(boolean useFilterLevel){this.useFilterLevel = useFilterLevel;}/** * 設置日志格式 * * @param pattern */publicvoid setPattern(String pattern){this.pattern = pattern;}@Overridepublicvoid afterPropertiesSet()throwsException{Assert.notNull(appender,"property `appender` must set!"); buildAppender();//多LOGGER支持配置if(logName.getClass().getSimpleName().equals("LinkedHashMap")){Map<String,String> loggers =(Map) logName;for(Map.Entry<String,String> log : loggers.entrySet()){Logger logger =(Logger)LoggerFactory.getLogger(log.getKey()); logger.setLevel(Level.toLevel(log.getValue())); logger.setAdditive(additiveAppender); logger.addAppender(appender); _logger.debug("Set appender: {} to logger: {} ", appender.getName(), log.getKey());}}//單個配置else{Logger logger =(Logger)LoggerFactory.getLogger(logName.toString());//將appender添加到指定logger logger.setLevel(logLevel); logger.setAdditive(additiveAppender); logger.addAppender(appender); _logger.debug("Set appender: {} to logger: {} ", appender.getName(), logName);}}privatevoid buildAppender(){LoggerContext loggerContext =(LoggerContext)LoggerFactory.getILoggerFactory();//啟用級別過濾，適用場景：把級別為 warn 是放入數據庫。if(useFilterLevel){LevelFilter levelFilter =newLevelFilter(); levelFilter.setContext(loggerContext); levelFilter.setLevel(filterLevel); levelFilter.setOnMatch(FilterReply.ACCEPT); levelFilter.setOnMismatch(FilterReply.DENY); levelFilter.start(); appender.addFilter(levelFilter);}//設置appender相關屬性 appender.setName(appenderName); appender.setPattern(pattern); appender.setContext(loggerContext); appender.start();}}NoSqlAppender
NoSql Appender 基礎類，不同NOSQL數據庫依賴此類實現不同APPENDER。
import ch.qos.logback.classic.PatternLayout;import ch.qos.logback.classic.spi.ILoggingEvent;import ch.qos.logback.core.UnsynchronizedAppenderBase;import ch.qos.logback.core.spi.LogbackLock;import org.springframework.util.Assert;/** * NoSql Appender 基礎類 * 子類需要實現generatedKey方法和指定存儲類實例 */abstractpublicclassNoSqlAppender<E>extendsUnsynchronizedAppenderBase<E>{//日志存儲protectedILogRepository logRepository;//日志表名protectedString tbname;//日志存儲格式protectedString pattern;//日志格式解析器protectedPatternLayout patternLayout;//Nosql操作類protectedObject template;/** * 日志存儲KEY生成 * * @param event * @return */protectedabstractString generatedKey(E event);/** * 使用指定的格式生成日志內容數據 * * @param event * @return */protectedString generatedValue(E event){return patternLayout.doLayout((ILoggingEvent) event);}/** * 日志表名 * * @param tbname */publicvoid setTbname(String tbname){this.tbname = tbname;}/** * 日志存儲 * * @param eventObject */@Overrideprotectedvoid append(E eventObject){if(!isStarted()){return;}try{String key = generatedKey(eventObject);String value = generatedValue(eventObject); logRepository.saveLog(key, value);}catch(Exception e){ addError(e.getMessage());}}/** * 初始化，patternLayout */@Overridepublicvoid start(){Assert.notNull(tbname,"tbname not null !"); patternLayout =newPatternLayout(); patternLayout.setPattern(pattern); patternLayout.setContext(context); patternLayout.setOutputPatternAsHeader(false); patternLayout.start();super.start();}@Overridepublicvoid stop(){super.stop();}/** * 日志格式 * * @param pattern */publicvoid setPattern(String pattern){this.pattern = pattern;}publicvoid setTemplate(Object template){this.template = template;}}ILogRepository
日志存儲接口，不同NOSQL數據庫提供統一保存方法。
/** * 日志存儲接口 */public interface ILogRepository<E>{/** * 保存日志，KEY-VALUE形式 * * @param key * @param value */publicvoid saveLog(String key,String value);}HbaseLogRepository
Hbase的日志存儲實現類
import b2gonline.wap.hbase.HbaseTemplate;import b2gonline.wap.hbase.TableCallback;import b2gonline.wap.logback.ILogRepository;import org.apache.hadoop.hbase.client.HTableInterface;import org.apache.hadoop.hbase.client.Put;import org.apache.hadoop.hbase.util.Bytes;/** * Hbase 日志存儲實現類 */publicclassHbaseLogRepositoryimplementsILogRepository<HbaseTemplate>{privateString tbname;privateHbaseTemplate hbaseTemplate;//日志列族publicstaticbyte[] CF_INFO =Bytes.toBytes("log");//日志列名privatebyte[] CF_CELL =Bytes.toBytes("data");//日志表名publicvoid setTbname(String tbname){this.tbname = tbname;}publicvoid setHbaseTemplate(HbaseTemplate hbaseTemplate){this.hbaseTemplate = hbaseTemplate;}/** * 日志數據存儲 * * @param key * @param value */@Overridepublicvoid saveLog(finalString key,String value){finalbyte[] bKey =Bytes.toBytes(key);finalbyte[] bValue =Bytes.toBytes(value); hbaseTemplate.execute(tbname,newTableCallback<Object>(){@OverridepublicObject doInTable(HTableInterface table)throwsThrowable{Put p =newPut(bKey); p.add(CF_INFO, CF_CELL, bValue); table.put(p);returnnull;}});}}HbaseAppender
Hbase的Appender實現類
import b2gonline.wap.hbase.HbaseTemplate;import b2gonline.wap.logback.NoSqlAppender;import ch.qos.logback.classic.spi.ILoggingEvent;import org.apache.hadoop.hbase.HColumnDescriptor;import org.apache.hadoop.hbase.HTableDescriptor;import org.apache.hadoop.hbase.client.HBaseAdmin;import org.springframework.beans.factory.FactoryBean;import org.springframework.beans.factory.InitializingBean;import org.springframework.util.Assert;import java.util.Random;/** * Hbase Appender 實現類 */publicclassHbaseAppenderextendsNoSqlAppender<ILoggingEvent>implementsInitializingBean{Random generator =newRandom();privateHBaseAdmin admin;/** * 初始化，HbaseLogRepository */@Overridepublicvoid start(){Assert.notNull(template,"hbaseTemplate not null !");try{ afterPropertiesSet();}catch(Exception e){ e.printStackTrace();}HbaseLogRepository repository =newHbaseLogRepository(); repository.setHbaseTemplate((HbaseTemplate) template); repository.setTbname(tbname); logRepository = repository;super.start();}/** * 生成記錄KEY，如果有必要也可以通過patternLayout生成 * * @param event * @return */@OverrideprotectedString generatedKey(ILoggingEvent event){//使用隨機數防止並發生成同名KEYint id = generator.nextInt(9999)+1000;StringBuilder sb =newStringBuilder(); sb.append(event.getLoggerName()); sb.append(event.getLevel()); sb.append(event.getThreadName()); sb.append(event.getTimeStamp()); sb.append("-"); sb.append(id);return sb.toString().toLowerCase().replaceAll(" ","");}/** * 沒有表自動創建 * * @throws Exception */@Overridepublicvoid afterPropertiesSet()throwsException{ admin =newHBaseAdmin(((HbaseTemplate) template).getConfiguration());if(!admin.tableExists(tbname)){HTableDescriptor tableDescriptor =newHTableDescriptor(tbname);HColumnDescriptor columnDescriptor =newHColumnDescriptor(HbaseLogRepository.CF_INFO); tableDescriptor.addFamily(columnDescriptor); admin.createTable(tableDescriptor);}}}關於HbaseTemplate
如上所說，Hbase操作類HbaseTemplate提取自SPRING-DATA-HADOOP，參考https://github.com/SpringSource/spring-hadoop/trunk/spring-hadoop-core/src/main/java/org/springframework/data/hadoop/hbase。
提取后會依賴ConfigurationUtils，源碼如下：
import org.springframework.util.Assert;import java.util.Enumeration;import java.util.Properties;publicclassConfigurationUtils{publicstaticvoid addProperties(org.apache.hadoop.conf.Configuration configuration,Properties properties){Assert.notNull(configuration,"A non-null configuration is required");if(properties !=null){Enumeration<?> props = properties.propertyNames();while(props.hasMoreElements()){String key = props.nextElement().toString(); configuration.set(key, properties.getProperty(key));}}}}提取后的Hbase連接配置：
<bean id="hbaseConfiguration"class="b2gonline.wap.hbase.HbaseConfigurationFactoryBean"><property name="zkPort" value="2181"/><property name="zkQuorum" value="hadoopmaster,hadoopnode1"/></bean><bean id="hbaseTemplate"class="b2gonline.wap.hbase.HbaseTemplate"><property name="configuration" ref="hbaseConfiguration"/></bean>最后
實現了日志數據統一存儲就還得有統一查看的功能，沒錯，下一步實現！