分析針對{歷史數據，每天的增量數據}
站長工具 seo 優化， pageRank
http://seo.chinaz.com/www.taobao.com
PV (page view ） UV(user view) 頁面訪問量，用戶訪問量
一針對增量數據進行分析。 設定淘寶300G/每天； 唯品會訪問量為17G
建立數據表時預留兩個字段，同樣程序功能也預留一個模塊。
一般 有指標和維度 。
埋點 ： 向后端大數據平台報告請求。
時間參數格式舉例：
js-sdk,java-sdk
startURL工具畫時序圖
node204 上安裝nginx(tengine)
1002 cd software/
1004 tar -zxvf tengine-2.1.0.tar.gz ## 解壓nginx
1005 cd tengine-2.1.0
1007 ./configure
1008 yum -y install gcc ## 安裝依賴 否則confugure報錯 CentOS nginx checking for C compiler ... not found
1009 yum -y install gcc-c++
1010 yum -y install openssl openssl-devel
1011 ./configure
1012 make && make install
1013 whereis nginx
1014 cd /usr/local/nginx/
1016 cd sbin/
1018 ./nginx ## 啟動nginx
1020 cd conf/
1021 ls
1022 cp nginx.conf nginx.conf.bak
1023 vi nginx.conf # 配置nginx
[root@node204 data]# cat /usr/local/nginx/conf/nginx.conf
#user nobody;
worker_processes 1;
#error_log logs/error.log;
#error_log logs/error.log notice;
#error_log logs/error.log info;
#pid logs/nginx.pid;
events {
worker_connections 1024;
}
# load modules compiled as Dynamic Shared Object (DSO)
#
#dso {
# load ngx_http_fastcgi_module.so;
# load ngx_http_rewrite_module.so;
#}
http {
include mime.types;
default_type application/octet-stream;
#log_format main '$remote_addr - $remote_user [$time_local] "$request" '
# '$status $body_bytes_sent "$http_referer" '
# '"$http_user_agent" "$http_x_forwarded_for"';
log_format my_format '$remote_addr^A$msec^A$http_host^A$request_uri';
#access_log logs/access.log main;
sendfile on;
#tcp_nopush on;
#keepalive_timeout 0;
keepalive_timeout 65;
#gzip on;
server {
listen 80;
server_name localhost;
#charset koi8-r;
#access_log logs/host.access.log main;
location / {
root html;
index index.html index.htm;
}
location = /log.gif {
default_type image/gif;
access_log /opt/data/access.log my_format;
}
#error_page 404 /404.html;
# redirect server error pages to the static page /50x.html
#
error_page 500 502 503 504 /50x.html;
location = /50x.html {
root html;
}
# proxy the PHP scripts to Apache listening on 127.0.0.1:80
#
#location ~ \.php$ {
# proxy_pass http://127.0.0.1;
#}
# pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000
#
#location ~ \.php$ {
# root html;
# fastcgi_pass 127.0.0.1:9000;
# fastcgi_index index.php;
# fastcgi_param SCRIPT_FILENAME /scripts$fastcgi_script_name;
# include fastcgi_params;
#}
# deny access to .htaccess files, if Apache's document root
# concurs with nginx's one
#
#location ~ /\.ht {
# deny all;
#}
}
# another virtual host using mix of IP-, name-, and port-based configuration
#
#server {
# listen 8000;
# listen somename:8080;
# server_name somename alias another.alias;
# location / {
# root html;
# index index.html index.htm;
# }
#}
# HTTPS server
#
#server {
# listen 443 ssl;
# server_name localhost;
# ssl_certificate cert.pem;
# ssl_certificate_key cert.key;
# ssl_session_cache shared:SSL:1m;
# ssl_session_timeout 5m;
# ssl_ciphers HIGH:!aNULL:!MD5;
# ssl_prefer_server_ciphers on;
# location / {
# root html;
# index index.html index.htm;
# }
#}
}
1024 mkdir -r /opt/data ## 創建日志存放的目錄
1025 mkdir /opt/data
1026 cd vi /etc/init.d/nginx ## 配置service nginx 服務 ## 注意pid和安裝目錄需要與nginx.conf保持一致。 否則會報錯
Centos7 配置systemctl的Nginx啟動服務，start一直卡着，stop不生效https://www.cnblogs.com/JaminXie/p/11322697.html
[root@node204 data]# cat /etc/init.d/nginx
#!/bin/sh
#
# nginx - this script starts and stops the nginx daemon
#
# chkconfig: - 85 15
# description: Nginx is an HTTP(S) server, HTTP(S) reverse \
# proxy and IMAP/POP3 proxy server
# processname: nginx
# config: /etc/nginx/nginx.conf
# config: /etc/sysconfig/nginx
# pidfile: /usr/local/nginx/logs/nginx.pid
# Source function library.
. /etc/rc.d/init.d/functions
# Source networking configuration.
. /etc/sysconfig/network
# Check that networking is up.
[ "$NETWORKING" = "no" ] && exit 0
nginx="/usr/local/nginx/sbin/nginx"
prog=$(basename $nginx)
NGINX_CONF_FILE="/usr/local/nginx/conf/nginx.conf"
[ -f /etc/sysconfig/nginx ] && . /etc/sysconfig/nginx
lockfile=/var/lock/subsys/nginx
make_dirs() {
# make required directories
user=`nginx -V 2>&1 | grep "configure arguments:" | sed 's/[^*]*--user=\([^ ]*\).*/\1/g' -`
options=`$nginx -V 2>&1 | grep 'configure arguments:'`
for opt in $options; do
if [ `echo $opt | grep '.*-temp-path'` ]; then
value=`echo $opt | cut -d "=" -f 2`
if [ ! -d "$value" ]; then
# echo "creating" $value
mkdir -p $value && chown -R $user $value
fi
fi
done
}
start() {
[ -x $nginx ] || exit 5
[ -f $NGINX_CONF_FILE ] || exit 6
make_dirs
echo -n $"Starting $prog: "
daemon $nginx -c $NGINX_CONF_FILE
retval=$?
echo
[ $retval -eq 0 ] && touch $lockfile
return $retval
}
stop() {
echo -n $"Stopping $prog: "
killproc $prog -QUIT
retval=$?
echo
[ $retval -eq 0 ] && rm -f $lockfile
return $retval
}
restart() {
configtest || return $?
stop
sleep 1
start
}
reload() {
configtest || return $?
echo -n $"Reloading $prog: "
killproc $nginx -HUP
RETVAL=$?
echo
}
force_reload() {
restart
}
configtest() {
$nginx -t -c $NGINX_CONF_FILE
}
rh_status() {
status $prog
}
rh_status_q() {
rh_status >/dev/null 2>&1
}
case "$1" in
start)
rh_status_q && exit 0
$1
;;
stop)
rh_status_q || exit 0
$1
;;
restart|configtest)
$1
;;
reload)
rh_status_q || exit 7
$1
;;
force-reload)
force_reload
;;
status)
rh_status
;;
condrestart|try-restart)
rh_status_q || exit 0
;;
*)
echo $"Usage: $0 {start|stop|status|restart|condrestart|try-restart|reload|force-reload|configtest}"
exit 2
esac
1028 chmod +x /etc/init.d/nginx
1029 service nginx restart
1097 systemctl daemon-reload
1098 service nginx stop
1099 service nginx start
1100 cd /opt/data/ ##從window上傳一張圖片log.gif到 /usr/local/nginx/html/下。
1102 tail -f access.log
## http://node204/log.gif?name=zhangsan&age=19 可以收到日志
js-sdk,java-sdk訪問方法：
analytics.js
(function() {
var CookieUtil = {
// get the cookie of the key is name
get : function(name) {
var cookieName = encodeURIComponent(name) + "=", cookieStart = document.cookie
.indexOf(cookieName), cookieValue = null;
if (cookieStart > -1) {
var cookieEnd = document.cookie.indexOf(";", cookieStart);
if (cookieEnd == -1) {
cookieEnd = document.cookie.length;
}
cookieValue = decodeURIComponent(document.cookie.substring(
cookieStart + cookieName.length, cookieEnd));
}
return cookieValue;
},
// set the name/value pair to browser cookie
set : function(name, value, expires, path, domain, secure) {
var cookieText = encodeURIComponent(name) + "="
+ encodeURIComponent(value);
if (expires) {
// set the expires time
var expiresTime = new Date();
expiresTime.setTime(expires);
cookieText += ";expires=" + expiresTime.toGMTString();
}
if (path) {
cookieText += ";path=" + path;
}
if (domain) {
cookieText += ";domain=" + domain;
}
if (secure) {
cookieText += ";secure";
}
document.cookie = cookieText;
},
setExt : function(name, value) {
this.set(name, value, new Date().getTime() + 315360000000, "/");
}
};
// 主體，其實就是tracker js
var tracker = {
// config
clientConfig : {
serverUrl : "http://node204/log.gif",
sessionTimeout : 360, // 360s -> 6min
maxWaitTime : 3600, // 3600s -> 60min -> 1h
ver : "1"
},
cookieExpiresTime : 315360000000, // cookie過期時間，10年
columns : {
// 發送到服務器的列名稱
eventName : "en",
version : "ver",
platform : "pl",
sdk : "sdk",
uuid : "u_ud",
memberId : "u_mid",
sessionId : "u_sd",
clientTime : "c_time",
language : "l",
userAgent : "b_iev",
resolution : "b_rst",
currentUrl : "p_url",
referrerUrl : "p_ref",
title : "tt",
orderId : "oid",
orderName : "on",
currencyAmount : "cua",
currencyType : "cut",
paymentType : "pt",
category : "ca",
action : "ac",
kv : "kv_",
duration : "du"
},
keys : {
pageView : "e_pv",
chargeRequestEvent : "e_crt",
launch : "e_l",
eventDurationEvent : "e_e",
sid : "bftrack_sid",
uuid : "bftrack_uuid",
mid : "bftrack_mid",
preVisitTime : "bftrack_previsit",
},
/**
* 獲取會話id
*/
getSid : function() {
return CookieUtil.get(this.keys.sid);
},
/**
* 保存會話id到cookie
*/
setSid : function(sid) {
if (sid) {
CookieUtil.setExt(this.keys.sid, sid);
}
},
/**
* 獲取uuid，從cookie中
*/
getUuid : function() {
return CookieUtil.get(this.keys.uuid);
},
/**
* 保存uuid到cookie
*/
setUuid : function(uuid) {
if (uuid) {
CookieUtil.setExt(this.keys.uuid, uuid);
}
},
/**
* 獲取memberID
*/
getMemberId : function() {
return CookieUtil.get(this.keys.mid);
},
/**
* 設置mid
*/
setMemberId : function(mid) {
if (mid) {
CookieUtil.setExt(this.keys.mid, mid);
}
},
startSession : function() {
// 加載js就觸發的方法
if (this.getSid()) {
// 會話id存在，表示uuid也存在
if (this.isSessionTimeout()) {
// 會話過期,產生新的會話
this.createNewSession();
} else {
// 會話沒有過期，更新最近訪問時間
this.updatePreVisitTime(new Date().getTime());
}
} else {
// 會話id不存在，表示uuid也不存在
this.createNewSession();
}
this.onPageView();
},
onLaunch : function() {
// 觸發launch事件
var launch = {};
launch[this.columns.eventName] = this.keys.launch; // 設置事件名稱
this.setCommonColumns(launch); // 設置公用columns
this.sendDataToServer(this.parseParam(launch)); // 最終發送編碼后的數據
},
onPageView : function() {
// 觸發page view事件
if (this.preCallApi()) {
var time = new Date().getTime();
var pageviewEvent = {};
pageviewEvent[this.columns.eventName] = this.keys.pageView;
pageviewEvent[this.columns.currentUrl] = window.location.href; // 設置當前url
pageviewEvent[this.columns.referrerUrl] = document.referrer; // 設置前一個頁面的url
pageviewEvent[this.columns.title] = document.title; // 設置title
this.setCommonColumns(pageviewEvent); // 設置公用columns
this.sendDataToServer(this.parseParam(pageviewEvent)); // 最終發送編碼后的數據
this.updatePreVisitTime(time);
}
},
onChargeRequest : function(orderId, name, currencyAmount, currencyType, paymentType) {
// 觸發訂單產生事件
if (this.preCallApi()) {
if (!orderId || !currencyType || !paymentType) {
this.log("訂單id、貨幣類型以及支付方式不能為空");
return;
}
if (typeof (currencyAmount) == "number") {
// 金額必須是數字
var time = new Date().getTime();
var chargeRequestEvent = {};
chargeRequestEvent[this.columns.eventName] = this.keys.chargeRequestEvent;
chargeRequestEvent[this.columns.orderId] = orderId;
chargeRequestEvent[this.columns.orderName] = name;
chargeRequestEvent[this.columns.currencyAmount] = currencyAmount;
chargeRequestEvent[this.columns.currencyType] = currencyType;
chargeRequestEvent[this.columns.paymentType] = paymentType;
this.setCommonColumns(chargeRequestEvent); // 設置公用columns
this.sendDataToServer(this.parseParam(chargeRequestEvent)); // 最終發送編碼后的數據
this.updatePreVisitTime(time);
} else {
this.log("訂單金額必須是數字");
return;
}
}
},
onEventDuration : function(category, action, map, duration) {
// 觸發event事件
if (this.preCallApi()) {
if (category && action) {
var time = new Date().getTime();
var event = {};
event[this.columns.eventName] = this.keys.eventDurationEvent;
event[this.columns.category] = category;
event[this.columns.action] = action;
if (map) {
for ( var k in map) {
if (k && map[k]) {
event[this.columns.kv + k] = map[k];
}
}
}
if (duration) {
event[this.columns.duration] = duration;
}
this.setCommonColumns(event); // 設置公用columns
this.sendDataToServer(this.parseParam(event)); // 最終發送編碼后的數據
this.updatePreVisitTime(time);
} else {
this.log("category和action不能為空");
}
}
},
/**
* 執行對外方法前必須執行的方法
*/
preCallApi : function() {
if (this.isSessionTimeout()) {
// 如果為true，表示需要新建
this.startSession();
} else {
this.updatePreVisitTime(new Date().getTime());
}
return true;
},
sendDataToServer : function(data) {
alert(data);
// 發送數據data到服務器，其中data是一個字符串
var that = this;
var i2 = new Image(1, 1);// <img src="url"></img>
i2.onerror = function() {
// 這里可以進行重試操作
};
i2.src = this.clientConfig.serverUrl + "?" + data;
},
/**
* 往data中添加發送到日志收集服務器的公用部分
*/
setCommonColumns : function(data) {
data[this.columns.version] = this.clientConfig.ver;
data[this.columns.platform] = "website";
data[this.columns.sdk] = "js";
data[this.columns.uuid] = this.getUuid(); // 設置用戶id
data[this.columns.memberId] = this.getMemberId(); // 設置會員id
data[this.columns.sessionId] = this.getSid(); // 設置sid
data[this.columns.clientTime] = new Date().getTime(); // 設置客戶端時間
data[this.columns.language] = window.navigator.language; // 設置瀏覽器語言
data[this.columns.userAgent] = window.navigator.userAgent; // 設置瀏覽器類型
data[this.columns.resolution] = screen.width + "*" + screen.height; // 設置瀏覽器分辨率
},
/**
* 創建新的會員，並判斷是否是第一次訪問頁面，如果是，進行launch事件的發送。
*/
createNewSession : function() {
var time = new Date().getTime(); // 獲取當前操作時間
// 1. 進行會話更新操作
var sid = this.generateId(); // 產生一個session id
this.setSid(sid);
this.updatePreVisitTime(time); // 更新最近訪問時間
// 2. 進行uuid查看操作
if (!this.getUuid()) {
// uuid不存在，先創建uuid，然后保存到cookie，最后觸發launch事件
var uuid = this.generateId(); // 產品uuid
this.setUuid(uuid);
this.onLaunch();
}
},
/**
* 參數編碼返回字符串
*/
parseParam : function(data) {
var params = "";
for ( var e in data) {
if (e && data[e]) {
params += encodeURIComponent(e) + "="
+ encodeURIComponent(data[e]) + "&";
}
}
if (params) {
return params.substring(0, params.length - 1);
} else {
return params;
}
},
/**
* 產生uuid
*/
generateId : function() {
var chars = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz';
var tmpid = [];
var r;
tmpid[8] = tmpid[13] = tmpid[18] = tmpid[23] = '-';
tmpid[14] = '4';
for (i = 0; i < 36; i++) {
if (!tmpid[i]) {
r = 0 | Math.random() * 16;
tmpid[i] = chars[(i == 19) ? (r & 0x3) | 0x8 : r];
}
}
return tmpid.join('');
},
/**
* 判斷這個會話是否過期，查看當前時間和最近訪問時間間隔時間是否小於this.clientConfig.sessionTimeout<br/>
* 如果是小於，返回false;否則返回true。
*/
isSessionTimeout : function() {
var time = new Date().getTime();
var preTime = CookieUtil.get(this.keys.preVisitTime);
if (preTime) {
// 最近訪問時間存在,那么進行區間判斷
return time - preTime > this.clientConfig.sessionTimeout * 1000;
}
return true;
},
/**
* 更新最近訪問時間
*/
updatePreVisitTime : function(time) {
CookieUtil.setExt(this.keys.preVisitTime, time);
},
/**
* 打印日志
*/
log : function(msg) {
console.log(msg);
},
};
// 對外暴露的方法名稱
window.__AE__ = {
startSession : function() {
tracker.startSession();
},
onPageView : function() {
tracker.onPageView();
},
onChargeRequest : function(orderId, name, currencyAmount, currencyType, paymentType) {
tracker.onChargeRequest(orderId, name, currencyAmount, currencyType, paymentType);
},
onEventDuration : function(category, action, map, duration) {
tracker.onEventDuration(category, action, map, duration);
},
setMemberId : function(mid) {
tracker.setMemberId(mid);
}
};
// 自動加載方法
var autoLoad = function() {
// 進行參數設置
var _aelog_ = _aelog_ || window._aelog_ || [];
var memberId = null;
for (i = 0; i < _aelog_.length; i++) {
_aelog_[i][0] === "memberId" && (memberId = _aelog_[i][1]);
}
// 根據是給定memberid，設置memberid的值
memberId && __AE__.setMemberId(memberId);
// 啟動session
__AE__.startSession();
};
autoLoad();
})();
demo.jsp
<%@ page contentType="text/html; charset=utf-8" pageEncoding="utf-8"%>
<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
<title>測試頁面1</title>
<script type="text/javascript" src="./js/analytics.js"></script>
</head>
<body>
測試頁面1<br/>
跳轉到:
<a href="demo.jsp">demo</a>
<a href="demo2.jsp">demo2</a>
<a href="demo3.jsp">demo3</a>
<a href="demo4.jsp">demo4</a>
</body>
</html>
demo2.jsp
<%@ page contentType="text/html; charset=utf-8" pageEncoding="utf-8"%>
<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
<title>測試頁面2</title>
<script type="text/javascript" src="./js/analytics.js"></script>
</head>
<body>
測試頁面2
<br/>
<label>orderid: 123456</label><br>
<label>orderName: 測試訂單123456</label><br/>
<label>currencyAmount: 524.01</label><br/>
<label>currencyType: RMB</label><br/>
<label>paymentType: alipay</label><br/>
<button onclick="__AE__.onChargeRequest('123456','測試訂單123456',524.01,'RMB','alipay')">觸發chargeRequest事件</button><br/>
跳轉到:
<a href="demo.jsp">demo</a>
<a href="demo2.jsp">demo2</a>
<a href="demo3.jsp">demo3</a>
<a href="demo4.jsp">demo4</a>
</body>
</html>
demo3.jsp
<%@ page contentType="text/html; charset=utf-8" pageEncoding="utf-8"%>
<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
<title>測試頁面3</title>
<script type="text/javascript" src="./js/analytics.js"></script>
</head>
<body>
測試頁面3<br/>
<label>category: event的category名稱</label><br/>
<label>action: event的action名稱</label><br/>
<label>map: {"key1":"value1", "key2":"value2"}</label><br/>
<label>duration: 1245</label><br/>
<button onclick="__AE__.onEventDuration('event的category名稱','event的action名稱', {'key1':'value1','key2':'value2'}, 1245)">觸發帶map和duration的事件</button><br/>
<button onclick="__AE__.onEventDuration('event的category名稱','event的action名稱')">觸發不帶map和duration的事件</button><br/>
跳轉到:
<a href="demo.jsp">demo</a>
<a href="demo2.jsp">demo2</a>
<a href="demo3.jsp">demo3</a>
<a href="demo4.jsp">demo4</a>
</body>
</html>
demo4.jsp
<%@ page contentType="text/html; charset=utf-8" pageEncoding="utf-8"%>
<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
<title>測試頁面4</title>
<script type="text/javascript">
(function(){
var _aelog_ = _aelog_ || window._aelog_ || [];
// 設置_aelog_相關屬性
_aelog_.push(["memberId","zhangsan"]);
window._aelog_ = _aelog_;
(function(){
var aejs = document.createElement('script');
aejs.type = 'text/javascript';
aejs.async = true;
aejs.src = './js/analytics.js';
var script = document.getElementsByTagName('script')[0];
script.parentNode.insertBefore(aejs, script);
})();
})();
</script>
</head>
<body>
測試頁面4<br/>
在本頁面設置memberid為zhangsan<br/>
跳轉到:
<a href="demo.jsp">demo</a>
<a href="demo2.jsp">demo2</a>
<a href="demo3.jsp">demo3</a>
<a href="demo4.jsp">demo4</a>
</body>
</html>
package com.sxt.client;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.HashMap;
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;
/**
* 分析引擎sdk java服務器端數據收集
*
* @author root
* @version 1.0
*
*/
public class AnalyticsEngineSDK {
// 日志打印對象
private static final Logger log = Logger.getGlobal();
// 請求url的主體部分
public static final String accessUrl = "http://node204/log.gif";
private static final String platformName = "java_server";
private static final String sdkName = "jdk";
private static final String version = "1";
/**
* 觸發訂單支付成功事件，發送事件數據到服務器
*
* @param orderId
* 訂單支付id
* @param memberId
* 訂單支付會員id
* @return 如果發送數據成功(加入到發送隊列中)，那么返回true；否則返回false(參數異常&添加到發送隊列失敗).
*/
public static boolean onChargeSuccess(String orderId, String memberId) {
try {
if (isEmpty(orderId) || isEmpty(memberId)) {
// 訂單id或者memberid為空
log.log(Level.WARNING, "訂單id和會員id不能為空");
return false;
}
// 代碼執行到這兒，表示訂單id和會員id都不為空。
Map<String, String> data = new HashMap<String, String>();
data.put("u_mid", memberId);
data.put("oid", orderId);
data.put("c_time", String.valueOf(System.currentTimeMillis()));
data.put("ver", version);
data.put("en", "e_cs");
data.put("pl", platformName);
data.put("sdk", sdkName);
// 創建url
String url = buildUrl(data);
// 發送url&將url加入到隊列
SendDataMonitor.addSendUrl(url);
return true;
} catch (Throwable e) {
log.log(Level.WARNING, "發送數據異常", e);
}
return false;
}
/**
* 觸發訂單退款事件，發送退款數據到服務器
*
* @param orderId
* 退款訂單id
* @param memberId
* 退款會員id
* @return 如果發送數據成功，返回true。否則返回false。
*/
public static boolean onChargeRefund(String orderId, String memberId) {
try {
if (isEmpty(orderId) || isEmpty(memberId)) {
// 訂單id或者memberid為空
log.log(Level.WARNING, "訂單id和會員id不能為空");
return false;
}
// 代碼執行到這兒，表示訂單id和會員id都不為空。
Map<String, String> data = new HashMap<String, String>();
data.put("u_mid", memberId);
data.put("oid", orderId);
data.put("c_time", String.valueOf(System.currentTimeMillis()));
data.put("ver", version);
data.put("en", "e_cr");
data.put("pl", platformName);
data.put("sdk", sdkName);
// 構建url
String url = buildUrl(data);
// 發送url&將url添加到隊列中
SendDataMonitor.addSendUrl(url);
return true;
} catch (Throwable e) {
log.log(Level.WARNING, "發送數據異常", e);
}
return false;
}
/**
* 根據傳入的參數構建url
*
* @param data
* @return
* @throws UnsupportedEncodingException
*/
private static String buildUrl(Map<String, String> data)
throws UnsupportedEncodingException {
StringBuilder sb = new StringBuilder();
sb.append(accessUrl).append("?");
for (Map.Entry<String, String> entry : data.entrySet()) {
if (isNotEmpty(entry.getKey()) && isNotEmpty(entry.getValue())) {
sb.append(entry.getKey().trim())
.append("=")
.append(URLEncoder.encode(entry.getValue().trim(), "utf-8"))
.append("&");
}
}
return sb.substring(0, sb.length() - 1);// 去掉最后&
}
/**
* 判斷字符串是否為空，如果為空，返回true。否則返回false。
*
* @param value
* @return
*/
private static boolean isEmpty(String value) {
return value == null || value.trim().isEmpty();
}
/**
* 判斷字符串是否非空，如果不是空，返回true。如果是空，返回false。
*
* @param value
* @return
*/
private static boolean isNotEmpty(String value) {
return !isEmpty(value);
}
}
package com.sxt.client;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.logging.Level;
import java.util.logging.Logger;
/**
* 發送url數據的監控者，用於啟動一個單獨的線程來發送數據
*
* @author root
*
*/
public class SendDataMonitor {
// 日志記錄對象
private static final Logger log = Logger.getGlobal();
// 隊列，用戶存儲發送url
private BlockingQueue<String> queue = new LinkedBlockingQueue<String>();
// 用於單列的一個類對象
private static SendDataMonitor monitor = null;
private SendDataMonitor() {
// 私有構造方法，進行單列模式的創建
}
/**
* 獲取單列的monitor對象實例
*
* @return
*/
public static SendDataMonitor getSendDataMonitor() {
if (monitor == null) {
synchronized (SendDataMonitor.class) {
if (monitor == null) {
monitor = new SendDataMonitor();
Thread thread = new Thread(new Runnable() {
@Override
public void run() {
// 線程中調用具體的處理方法
SendDataMonitor.monitor.run();
}
});
// 測試的時候，不設置為守護模式
// thread.setDaemon(true);
thread.start();
}
}
}
return monitor;
}
/**
* 添加一個url到隊列中去
*
* @param url
* @throws InterruptedException
*/
public static void addSendUrl(String url) throws InterruptedException {
getSendDataMonitor().queue.put(url);
}
/**
* 具體執行發送url的方法
*
*/
private void run() {
while (true) {
try {
String url = this.queue.take();
// 正式的發送url
HttpRequestUtil.sendData(url);
} catch (Throwable e) {
log.log(Level.WARNING, "發送url異常", e);
}
}
}
/**
* 內部類，用戶發送數據的http工具類
*
* @author root
*
*/
public static class HttpRequestUtil {
/**
* 具體發送url的方法
*
* @param url
* @throws IOException
*/
public static void sendData(String url) throws IOException {
HttpURLConnection con = null;
BufferedReader in = null;
try {
URL obj = new URL(url); // 創建url對象
con = (HttpURLConnection) obj.openConnection(); // 打開url連接
// 設置連接參數
con.setConnectTimeout(5000); // 連接過期時間
con.setReadTimeout(5000); // 讀取數據過期時間
con.setRequestMethod("GET"); // 設置請求類型為get
System.out.println("發送url:" + url);
// 發送連接請求
in = new BufferedReader(new InputStreamReader(
con.getInputStream()));
// TODO: 這里考慮是否可以
} finally {
try {
if (in != null) {
in.close();
}
} catch (Throwable e) {
// nothing
}
try {
con.disconnect();
} catch (Throwable e) {
// nothing
}
}
}
}
}
package com.sxt.log.test;
import java.util.Random;
import com.sxt.client.AnalyticsEngineSDK;
public class Test {
public static String day = "20160607";
static Random r = new Random();
public static void main(String[] args) {
AnalyticsEngineSDK.onChargeSuccess("orderid123", "zhangsan");
AnalyticsEngineSDK.onChargeRefund("orderid456", "lisi");
// try {
// String d = day
// + String.format(
// "%02d%02d%02d",
// new Object[] { r.nextInt(24), r.nextInt(60),
// r.nextInt(60) });
// System.out.println(d);
// Date date = new SimpleDateFormat("yyyyMMddhhmmss").parse(d);
// System.out.println(date);
// String datetime = date.getTime() + "";
// System.out.println(datetime);
// String prefix = datetime.substring(0, datetime.length() - 3);
// System.out.println(prefix + "." + r.nextInt(1000));
// System.out.println(new Date(1423634643000l));
// } catch (ParseException e) {
// e.printStackTrace();
// }
}
}
使用tomcat 8運行。訪問jsp; 能夠看到日志。
flume http://flume.apache.org/index.html
亦可以在命令行運行時的參數。
單節點配置
安裝flume 204上。 可以參考官網文檔，非常詳細
1007 tar -zxvf apache-flume-1.6.0-bin.tar.gz -C /opt/sxt/
1008 cd /opt/sxt/apache-flume-1.6.0-bin/conf/
1015 cp flume-env.sh.template flume-env.sh
1016 vi flume-env.sh
export JAVA_HOME=/usr/java/jdk1.8.0_221
1020 vi /etc/profile
export FLUME_HOME=/opt/sxt/apache-flume-1.6.0-bin
export PATH=$PATH:$JAVA_HOME/bin:$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$HIVE_HOME/bin:$HBASE_HOME/bin:$FLUME_HOME/bin
1021 source /etc/profile
1022 flume-ng
1023 flume-ng version ## 查看版本
1024 cd /opt/
1032 mkdir flumedir ## 配置flume
1033 cd flumedir/
1035 vi option
[root@node204 conf]# cat /opt/flumedir/option
# example.conf: A single-node Flume configuration
# Name the components on this agent
a1.sources = r1
a1.sinks = k1
a1.channels = c1
# Describe/configure the source
a1.sources.r1.type = netcat ## netcat方式獲取
a1.sources.r1.bind = node204
a1.sources.r1.port = 44444
# Describe the sink
a1.sinks.k1.type = logger
# Use a channel which buffers events in memory
a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100
# Bind the source and sink to the channel
a1.sources.r1.channels = c1
a1.sinks.k1.channel = c1
1036 flume-ng agent --conf-file option --name a1 -Dflume.root.logger=INFO,console ## 啟動flume
2019-08-27 04:41:00,537 INFO node.Application: Starting Source r1 2019-08-27 04:41:00,542 INFO source.NetcatSource: Source starting 2019-08-27 04:41:00,650 INFO source.NetcatSource: Created serverSocket:sun.nio.ch.ServerSocketChannelImpl[/192.168.112.204:44444]
node205上操作 telnet發送socket數據，查看flume204 是否接收到。 795 yum install -y telnet [root@node205 ~]# telnet node204 44444 Trying 192.168.112.204... Connected to node204. Escape character is '^]'. ## 分隔符] hello OK hi OK sxt OK nihao OK ##在node204 flume程序下可以看到telnet發送的信息。
多節點配置
AVRO 格式（通過RPC發送數據）
node204(左) node205 （右）
node204
scp -r apache-flume-1.6.0-bin root@node205:`pwd` ## 分發文件
cd /opt/flumedir/
1048 vi option
# example.conf: A single-node Flume configuration
# Name the components on this agent
a1.sources = r1
a1.sinks = k1
a1.channels = c1
# Describe/configure the source
a1.sources.r1.type = netcat
a1.sources.r1.bind = node204
a1.sources.r1.port = 44444
# Describe the sink
a1.sinks.k1.type = avro
a1.sinks.k1.hostname = node205
a1.sinks.k1.port = 10086
# Use a channel which buffers events in memory
a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100
# Bind the source and sink to the channel
a1.sources.r1.channels = c1
a1.sinks.k1.channel = c1
node205 配置 flume環境變量
cd /opt/flumedir/
1048 vi option2
# example.conf: A single-node Flume configuration
# Name the components on this agent
a1.sources = r1
a1.sinks = k1
a1.channels = c1
# Describe/configure the source
a1.sources.r1.type = avro
a1.sources.r1.bind = node205
a1.sources.r1.port = 10086
# Describe the sink
a1.sinks.k1.type = logger
# Use a channel which buffers events in memory
a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100
# Bind the source and sink to the channel
a1.sources.r1.channels = c1
a1.sinks.k1.channel = c1
## a1.sources.r1.bind = node205 表示我監聽本機端口獲取數據。
node205 服務端先啟動
flume-ng agent --conf-file option2 --name a1 -Dflume.root.logger=INFO,console
node204客戶端在啟動
flume-ng agent --conf-file option --name a1 -Dflume.root.logger=INFO,console
在node205上試用telnet發送消息，node205 flume conlose下能夠查看到消息。
2019-08-27 05:16:11,893 INFO sink.LoggerSink: Event: { headers:{} body: 68 65 6C 6C 6F 0D hello. }
2019-08-27 05:16:11,893 INFO sink.LoggerSink: Event: { headers:{} body: 6E 69 6E 68 61 6F 08 08 08 0D ninhao.... }
2019-08-27 05:16:16,169 INFO sink.LoggerSink: Event: { headers:{} body: 77 6F 6D 65 6E 0D women. }
## 配置讀取文件末尾Exec source ## tail 監控文件增量 [root@node204 ~]# cat /opt/flumedir/option3 # example.conf: A single-node Flume configuration # Name the components on this agent a1.sources = r1 a1.sinks = k1 a1.channels = c1 # Describe/configure the source a1.sources.r1.type = exec a1.sources.r1.command = tail -F /root/sxt.log # Describe the sink a1.sinks.k1.type = avro a1.sinks.k1.hostname = node205 a1.sinks.k1.port = 10086 # Use a channel which buffers events in memory a1.channels.c1.type = memory a1.channels.c1.capacity = 1000 a1.channels.c1.transactionCapacity = 100 # Bind the source and sink to the channel a1.sources.r1.channels = c1 a1.sinks.k1.channel = c1 [root@node204 ~]# vi sxt.log ## 在node205上也啟動option2 # flume-ng agent --conf-file option3 --name a1 -Dflume.root.logger=INFO,console [root@node204 ~]# echo 'sxt' >> sxt.log ## 此時，王文建最佳呢絨。node205的console會輸出sxt字樣。 [root@node204 ~]# echo 'sxt' >> sxt.log [root@node204 ~]# echo 'sxt' >> sxt.log [root@node204 ~]# echo 'sxt' >> sxt.log ## vi 操作 vi sxt.log sxt sxt sxt :.,$y ## 復制當前行到末尾行內容
配置文件夾作為source [root@node204 ~]# cat /opt/flumedir/option4 # Name the components on this agent a1.sources = r1 a1.sinks = k1 a1.channels = c1 # Describe/configure the source a1.sources.r1.type = spooldir a1.sources.r1.spoolDir = /root/abc/ a1.sources.r1.fileHeader = true ### 表示是否顯示文件名 # Describe the sink a1.sinks.k1.type = logger # Use a channel which buffers events in memory a1.channels.c1.type = memory a1.channels.c1.capacity = 1000 a1.channels.c1.transactionCapacity = 100 # Bind the source and sink to the channel a1.sources.r1.channels = c1 a1.sinks.k1.channel = c1 [root@node204 flumedir]# flume-ng agent --conf-file option4 --name a1 -Dflume.root.logger=INFO,console [root@node204 ~]# mkdir abc [root@node204 ~]# echo 1111 >> aaa [root@node204 ~]# echo 2222 >> aaa [root@node204 ~]# echo 3333 >> bbb [root@node204 ~]# echo 4444 >> bbb [root@node204 ~]# cp aaa abc/ [root@node204 ~]# cp bbb abc/ ##此時能夠在option4下看到文件內容 [root@node204 ~]# cd abc/ [root@node204 abc]# ls aaa.COMPLETED bbb.COMPLETED
sink 為hdfs [root@node204 ~]# cat /opt/flumedir/option5 # Name the components on this agent a1.sources = r1 a1.sinks = k1 a1.channels = c1 # Describe/configure the source a1.sources.r1.type = exec a1.sources.r1.command = tail -F /root/sxt.log # Describe the sink a1.sinks.k1.type = hdfs a1.sinks.k1.hdfs.path = /flume/events/%y-%m-%d/%H%M/%S a1.sinks.k1.hdfs.filePrefix = events- a1.sinks.k1.hdfs.round = true a1.sinks.k1.hdfs.roundValue = 10 a1.sinks.k1.hdfs.roundUnit = minute a1.sinks.k1.hdfs.useLocalTimeStamp =true # Use a channel which buffers events in memory a1.channels.c1.type = memory a1.channels.c1.capacity = 1000 a1.channels.c1.transactionCapacity = 100 # Bind the source and sink to the channel a1.sources.r1.channels = c1 a1.sinks.k1.channel = c1
[root@node204 flumedir]# flume-ng agent --conf-file option5 --name a1 -Dflume.root.logger=INFO,console ## 會在hdfs上創建目錄和文件。 （必須啟動hdfs）
[root@node204 ~]# echo 'hello sxt ni hao' >> sxt.log ## 繼續最佳文件內容，hdfs數據增加。 [root@node204 ~]# echo 'hello sxt ni hao' >> sxt.log [root@node204 ~]# echo 'hello sxt ni hao' >> sxt.log [root@node204 ~]# echo 'hello sxt ni hao' >> sxt.log
[root@node204 ~]# cat aaa | tee a.log ## 一個source 多個sink；可以再用linux實現。 1111 2222 [root@node204 ~]# cat a.log 1111 2222
kafka 與flume一般組對使用
ETL 程序和操作
node204配置：
[root@node204 ~]# service nginx start [root@node204 ~]# cat /opt/flumedir/option6 a1.sources = r1 a1.sinks = k1 a1.channels = c1 a1.sources.r1.type = exec a1.sources.r1.command = tail -F /opt/data/access.log a1.sinks.k1.type=hdfs a1.sinks.k1.hdfs.path=hdfs://mycluster/log/%Y%m%d a1.sinks.k1.hdfs.rollCount=0 a1.sinks.k1.hdfs.rollInterval=0 a1.sinks.k1.hdfs.rollSize=10240 a1.sinks.k1.hdfs.idleTimeout=5 a1.sinks.k1.hdfs.fileType=DataStream a1.sinks.k1.hdfs.useLocalTimeStamp=true a1.sinks.k1.hdfs.callTimeout=40000 a1.channels.c1.type = memory a1.channels.c1.capacity = 1000 a1.channels.c1.transactionCapacity = 100 a1.sources.r1.channels = c1 a1.sinks.k1.channel = c1 ## 使用eclipse 運行BIG_DATA_LOG2 項目，訪問jsp頁面觸發埋點上報信息到nginx ,可以看到數據被上傳到hdfs 。 tail -f /opt/data/access.log 查看訪問的數據。
關於ETL在此處的作用 過濾臟數據（沒有?或en參數的請求過濾掉） 解析ip 為地域 解析瀏覽器。 存儲到hbase rowid 使用CRC32編碼（將時間戳，事件，用戶信息帶入）
使用工具類： 解析瀏覽器
package com.sxt.etl.util;
import java.io.IOException;
import cz.mallat.uasparser.OnlineUpdater;
import cz.mallat.uasparser.UASparser;
/**
* 解析瀏覽器的user agent的工具類，內部就是調用這個uasparser jar文件
*
* @author root
*
*/
public class UserAgentUtil {
static UASparser uasParser = null;
// static 代碼塊, 初始化uasParser對象
static {
try {
uasParser = new UASparser(OnlineUpdater.getVendoredInputStream());
} catch (IOException e) {
e.printStackTrace();
}
}
/**
* 解析瀏覽器的user agent字符串，返回UserAgentInfo對象。<br/>
* 如果user agent為空，返回null。如果解析失敗，也直接返回null。
*
* @param userAgent
* 要解析的user agent字符串
* @return 返回具體的值
*/
public static UserAgentInfo analyticUserAgent(String userAgent) {
UserAgentInfo result = null;
if (!(userAgent == null || userAgent.trim().isEmpty())) {
// 此時userAgent不為null，而且不是由全部空格組成的
try {
cz.mallat.uasparser.UserAgentInfo info = null;
info = uasParser.parse(userAgent);
result = new UserAgentInfo();
result.setBrowserName(info.getUaFamily());
result.setBrowserVersion(info.getBrowserVersionInfo());
result.setOsName(info.getOsFamily());
result.setOsVersion(info.getOsName());
} catch (IOException e) {
// 出現異常，將返回值設置為null
result = null;
}
}
return result;
}
/**
* 內部解析后的瀏覽器信息model對象
*
* @author root
*
*/
public static class UserAgentInfo {
private String browserName; // 瀏覽器名稱
private String browserVersion; // 瀏覽器版本號
private String osName; // 操作系統名稱
private String osVersion; // 操作系統版本號
public String getBrowserName() {
return browserName;
}
public void setBrowserName(String browserName) {
this.browserName = browserName;
}
public String getBrowserVersion() {
return browserVersion;
}
public void setBrowserVersion(String browserVersion) {
this.browserVersion = browserVersion;
}
public String getOsName() {
return osName;
}
public void setOsName(String osName) {
this.osName = osName;
}
public String getOsVersion() {
return osVersion;
}
public void setOsVersion(String osVersion) {
this.osVersion = osVersion;
}
@Override
public String toString() {
return "UserAgentInfo [browserName=" + browserName + ", browserVersion=" + browserVersion + ", osName="
+ osName + ", osVersion=" + osVersion + "]";
}
}
}
ip地址. qqwry.dat
package com.sxt.etl.util.ip;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.io.UnsupportedEncodingException;
import java.nio.ByteOrder;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.util.ArrayList;
import java.util.Hashtable;
import java.util.List;
/**
* * 用來讀取QQwry.dat文件，以根據ip獲得好友位置，QQwry.dat的格式是 一. 文件頭，共8字節 1. 第一個起始IP的絕對偏移， 4字節
* 2. 最后一個起始IP的絕對偏移， 4字節 二. "結束地址/國家/區域"記錄區 四字節ip地址后跟的每一條記錄分成兩個部分 1. 國家記錄 2.
* 地區記錄 但是地區記錄是不一定有的。而且國家記錄和地區記錄都有兩種形式 1. 以0結束的字符串 2. 4個字節，一個字節可能為0x1或0x2 a.
* 為0x1時，表示在絕對偏移后還跟着一個區域的記錄，注意是絕對偏移之后，而不是這四個字節之后 b. 為0x2時，表示在絕對偏移后沒有區域記錄
* 不管為0x1還是0x2，后三個字節都是實際國家名的文件內絕對偏移
* 如果是地區記錄，0x1和0x2的含義不明，但是如果出現這兩個字節，也肯定是跟着3個字節偏移，如果不是 則為0結尾字符串 三.
* "起始地址/結束地址偏移"記錄區 1. 每條記錄7字節，按照起始地址從小到大排列 a. 起始IP地址，4字節 b. 結束ip地址的絕對偏移，3字節
*
* 注意，這個文件里的ip地址和所有的偏移量均采用little-endian格式，而java是采用 big-endian格式的，要注意轉換
*
* @author root
*/
public class IPSeeker {
// 一些固定常量，比如記錄長度等等
private static final int IP_RECORD_LENGTH = 7;
private static final byte AREA_FOLLOWED = 0x01;
private static final byte NO_AREA = 0x2;
// 用來做為cache，查詢一個ip時首先查看cache，以減少不必要的重復查找
private Hashtable ipCache;
// 隨機文件訪問類
private RandomAccessFile ipFile;
// 內存映射文件
private MappedByteBuffer mbb;
// 單一模式實例
private static IPSeeker instance = null;
// 起始地區的開始和結束的絕對偏移
private long ipBegin, ipEnd;
// 為提高效率而采用的臨時變量
private IPLocation loc;
private byte[] buf;
private byte[] b4;
private byte[] b3;
/** */
/**
* 私有構造函數
*/
protected IPSeeker() {
ipCache = new Hashtable();
loc = new IPLocation();
buf = new byte[100];
b4 = new byte[4];
b3 = new byte[3];
try {
String ipFilePath = IPSeeker.class.getResource("/qqwry.dat")
.getFile();
ipFile = new RandomAccessFile(ipFilePath, "r");
} catch (FileNotFoundException e) {
System.out.println("IP地址信息文件沒有找到，IP顯示功能將無法使用");
ipFile = null;
}
// 如果打開文件成功，讀取文件頭信息
if (ipFile != null) {
try {
ipBegin = readLong4(0);
ipEnd = readLong4(4);
if (ipBegin == -1 || ipEnd == -1) {
ipFile.close();
ipFile = null;
}
} catch (IOException e) {
System.out.println("IP地址信息文件格式有錯誤，IP顯示功能將無法使用");
ipFile = null;
}
}
}
/** */
/**
* @return 單一實例
*/
public static IPSeeker getInstance() {
if (instance == null) {
instance = new IPSeeker();
}
return instance;
}
/** */
/**
* 給定一個地點的不完全名字，得到一系列包含s子串的IP范圍記錄
*
* @param s
* 地點子串
* @return 包含IPEntry類型的List
*/
public List getIPEntriesDebug(String s) {
List ret = new ArrayList();
long endOffset = ipEnd + 4;
for (long offset = ipBegin + 4; offset <= endOffset; offset += IP_RECORD_LENGTH) {
// 讀取結束IP偏移
long temp = readLong3(offset);
// 如果temp不等於-1，讀取IP的地點信息
if (temp != -1) {
IPLocation loc = getIPLocation(temp);
// 判斷是否這個地點里面包含了s子串，如果包含了，添加這個記錄到List中，如果沒有，繼續
if (loc.country.indexOf(s) != -1 || loc.area.indexOf(s) != -1) {
IPEntry entry = new IPEntry();
entry.country = loc.country;
entry.area = loc.area;
// 得到起始IP
readIP(offset - 4, b4);
entry.beginIp = IPSeekerUtils.getIpStringFromBytes(b4);
// 得到結束IP
readIP(temp, b4);
entry.endIp = IPSeekerUtils.getIpStringFromBytes(b4);
// 添加該記錄
ret.add(entry);
}
}
}
return ret;
}
/** */
/**
* 給定一個地點的不完全名字，得到一系列包含s子串的IP范圍記錄
*
* @param s
* 地點子串
* @return 包含IPEntry類型的List
*/
public List getIPEntries(String s) {
List ret = new ArrayList();
try {
// 映射IP信息文件到內存中
if (mbb == null) {
FileChannel fc = ipFile.getChannel();
mbb = fc.map(FileChannel.MapMode.READ_ONLY, 0, ipFile.length());
mbb.order(ByteOrder.LITTLE_ENDIAN);
}
int endOffset = (int) ipEnd;
for (int offset = (int) ipBegin + 4; offset <= endOffset; offset += IP_RECORD_LENGTH) {
int temp = readInt3(offset);
if (temp != -1) {
IPLocation loc = getIPLocation(temp);
// 判斷是否這個地點里面包含了s子串，如果包含了，添加這個記錄到List中，如果沒有，繼續
if (loc.country.indexOf(s) != -1
|| loc.area.indexOf(s) != -1) {
IPEntry entry = new IPEntry();
entry.country = loc.country;
entry.area = loc.area;
// 得到起始IP
readIP(offset - 4, b4);
entry.beginIp = IPSeekerUtils.getIpStringFromBytes(b4);
// 得到結束IP
readIP(temp, b4);
entry.endIp = IPSeekerUtils.getIpStringFromBytes(b4);
// 添加該記錄
ret.add(entry);
}
}
}
} catch (IOException e) {
System.out.println(e.getMessage());
}
return ret;
}
/** */
/**
* 從內存映射文件的offset位置開始的3個字節讀取一個int
*
* @param offset
* @return
*/
private int readInt3(int offset) {
mbb.position(offset);
return mbb.getInt() & 0x00FFFFFF;
}
/** */
/**
* 從內存映射文件的當前位置開始的3個字節讀取一個int
*
* @return
*/
private int readInt3() {
return mbb.getInt() & 0x00FFFFFF;
}
/** */
/**
* 根據IP得到國家名
*
* @param ip
* ip的字節數組形式
* @return 國家名字符串
*/
public String getCountry(byte[] ip) {
// 檢查ip地址文件是否正常
if (ipFile == null)
return "錯誤的IP數據庫文件";
// 保存ip，轉換ip字節數組為字符串形式
String ipStr = IPSeekerUtils.getIpStringFromBytes(ip);
// 先檢查cache中是否已經包含有這個ip的結果，沒有再搜索文件
if (ipCache.containsKey(ipStr)) {
IPLocation loc = (IPLocation) ipCache.get(ipStr);
return loc.country;
} else {
IPLocation loc = getIPLocation(ip);
ipCache.put(ipStr, loc.getCopy());
return loc.country;
}
}
/** */
/**
* 根據IP得到國家名
*
* @param ip
* IP的字符串形式
* @return 國家名字符串
*/
public String getCountry(String ip) {
return getCountry(IPSeekerUtils.getIpByteArrayFromString(ip));
}
/** */
/**
* 根據IP得到地區名
*
* @param ip
* ip的字節數組形式
* @return 地區名字符串
*/
public String getArea(byte[] ip) {
// 檢查ip地址文件是否正常
if (ipFile == null)
return "錯誤的IP數據庫文件";
// 保存ip，轉換ip字節數組為字符串形式
String ipStr = IPSeekerUtils.getIpStringFromBytes(ip);
// 先檢查cache中是否已經包含有這個ip的結果，沒有再搜索文件
if (ipCache.containsKey(ipStr)) {
IPLocation loc = (IPLocation) ipCache.get(ipStr);
return loc.area;
} else {
IPLocation loc = getIPLocation(ip);
ipCache.put(ipStr, loc.getCopy());
return loc.area;
}
}
/**
* 根據IP得到地區名
*
* @param ip
* IP的字符串形式
* @return 地區名字符串
*/
public String getArea(String ip) {
return getArea(IPSeekerUtils.getIpByteArrayFromString(ip));
}
/** */
/**
* 根據ip搜索ip信息文件，得到IPLocation結構，所搜索的ip參數從類成員ip中得到
*
* @param ip
* 要查詢的IP
* @return IPLocation結構
*/
public IPLocation getIPLocation(byte[] ip) {
IPLocation info = null;
long offset = locateIP(ip);
if (offset != -1)
info = getIPLocation(offset);
if (info == null) {
info = new IPLocation();
info.country = "未知國家";
info.area = "未知地區";
}
return info;
}
/**
* 從offset位置讀取4個字節為一個long，因為java為big-endian格式，所以沒辦法 用了這么一個函數來做轉換
*
* @param offset
* @return 讀取的long值，返回-1表示讀取文件失敗
*/
private long readLong4(long offset) {
long ret = 0;
try {
ipFile.seek(offset);
ret |= (ipFile.readByte() & 0xFF);
ret |= ((ipFile.readByte() << 8) & 0xFF00);
ret |= ((ipFile.readByte() << 16) & 0xFF0000);
ret |= ((ipFile.readByte() << 24) & 0xFF000000);
return ret;
} catch (IOException e) {
return -1;
}
}
/**
* 從offset位置讀取3個字節為一個long，因為java為big-endian格式，所以沒辦法 用了這么一個函數來做轉換
*
* @param offset
* @return 讀取的long值，返回-1表示讀取文件失敗
*/
private long readLong3(long offset) {
long ret = 0;
try {
ipFile.seek(offset);
ipFile.readFully(b3);
ret |= (b3[0] & 0xFF);
ret |= ((b3[1] << 8) & 0xFF00);
ret |= ((b3[2] << 16) & 0xFF0000);
return ret;
} catch (IOException e) {
return -1;
}
}
/**
* 從當前位置讀取3個字節轉換成long
*
* @return
*/
private long readLong3() {
long ret = 0;
try {
ipFile.readFully(b3);
ret |= (b3[0] & 0xFF);
ret |= ((b3[1] << 8) & 0xFF00);
ret |= ((b3[2] << 16) & 0xFF0000);
return ret;
} catch (IOException e) {
return -1;
}
}
/**
* 從offset位置讀取四個字節的ip地址放入ip數組中，讀取后的ip為big-endian格式，但是
* 文件中是little-endian形式，將會進行轉換
*
* @param offset
* @param ip
*/
private void readIP(long offset, byte[] ip) {
try {
ipFile.seek(offset);
ipFile.readFully(ip);
byte temp = ip[0];
ip[0] = ip[3];
ip[3] = temp;
temp = ip[1];
ip[1] = ip[2];
ip[2] = temp;
} catch (IOException e) {
System.out.println(e.getMessage());
}
}
/**
* 從offset位置讀取四個字節的ip地址放入ip數組中，讀取后的ip為big-endian格式，但是
* 文件中是little-endian形式，將會進行轉換
*
* @param offset
* @param ip
*/
private void readIP(int offset, byte[] ip) {
mbb.position(offset);
mbb.get(ip);
byte temp = ip[0];
ip[0] = ip[3];
ip[3] = temp;
temp = ip[1];
ip[1] = ip[2];
ip[2] = temp;
}
/**
* 把類成員ip和beginIp比較，注意這個beginIp是big-endian的
*
* @param ip
* 要查詢的IP
* @param beginIp
* 和被查詢IP相比較的IP
* @return 相等返回0，ip大於beginIp則返回1，小於返回-1。
*/
private int compareIP(byte[] ip, byte[] beginIp) {
for (int i = 0; i < 4; i++) {
int r = compareByte(ip[i], beginIp[i]);
if (r != 0)
return r;
}
return 0;
}
/**
* 把兩個byte當作無符號數進行比較
*
* @param b1
* @param b2
* @return 若b1大於b2則返回1，相等返回0，小於返回-1
*/
private int compareByte(byte b1, byte b2) {
if ((b1 & 0xFF) > (b2 & 0xFF)) // 比較是否大於
return 1;
else if ((b1 ^ b2) == 0)// 判斷是否相等
return 0;
else
return -1;
}
/**
* 這個方法將根據ip的內容，定位到包含這個ip國家地區的記錄處，返回一個絕對偏移 方法使用二分法查找。
*
* @param ip
* 要查詢的IP
* @return 如果找到了，返回結束IP的偏移，如果沒有找到，返回-1
*/
private long locateIP(byte[] ip) {
long m = 0;
int r;
// 比較第一個ip項
readIP(ipBegin, b4);
r = compareIP(ip, b4);
if (r == 0)
return ipBegin;
else if (r < 0)
return -1;
// 開始二分搜索
for (long i = ipBegin, j = ipEnd; i < j;) {
m = getMiddleOffset(i, j);
readIP(m, b4);
r = compareIP(ip, b4);
// log.debug(Utils.getIpStringFromBytes(b));
if (r > 0)
i = m;
else if (r < 0) {
if (m == j) {
j -= IP_RECORD_LENGTH;
m = j;
} else
j = m;
} else
return readLong3(m + 4);
}
// 如果循環結束了，那么i和j必定是相等的，這個記錄為最可能的記錄，但是並非
// 肯定就是，還要檢查一下，如果是，就返回結束地址區的絕對偏移
m = readLong3(m + 4);
readIP(m, b4);
r = compareIP(ip, b4);
if (r <= 0)
return m;
else
return -1;
}
/**
* 得到begin偏移和end偏移中間位置記錄的偏移
*
* @param begin
* @param end
* @return
*/
private long getMiddleOffset(long begin, long end) {
long records = (end - begin) / IP_RECORD_LENGTH;
records >>= 1;
if (records == 0)
records = 1;
return begin + records * IP_RECORD_LENGTH;
}
/**
* 給定一個ip國家地區記錄的偏移，返回一個IPLocation結構
*
* @param offset
* @return
*/
private IPLocation getIPLocation(long offset) {
try {
// 跳過4字節ip
ipFile.seek(offset + 4);
// 讀取第一個字節判斷是否標志字節
byte b = ipFile.readByte();
if (b == AREA_FOLLOWED) {
// 讀取國家偏移
long countryOffset = readLong3();
// 跳轉至偏移處
ipFile.seek(countryOffset);
// 再檢查一次標志字節，因為這個時候這個地方仍然可能是個重定向
b = ipFile.readByte();
if (b == NO_AREA) {
loc.country = readString(readLong3());
ipFile.seek(countryOffset + 4);
} else
loc.country = readString(countryOffset);
// 讀取地區標志
loc.area = readArea(ipFile.getFilePointer());
} else if (b == NO_AREA) {
loc.country = readString(readLong3());
loc.area = readArea(offset + 8);
} else {
loc.country = readString(ipFile.getFilePointer() - 1);
loc.area = readArea(ipFile.getFilePointer());
}
return loc;
} catch (IOException e) {
return null;
}
}
/**
* @param offset
* @return
*/
private IPLocation getIPLocation(int offset) {
// 跳過4字節ip
mbb.position(offset + 4);
// 讀取第一個字節判斷是否標志字節
byte b = mbb.get();
if (b == AREA_FOLLOWED) {
// 讀取國家偏移
int countryOffset = readInt3();
// 跳轉至偏移處
mbb.position(countryOffset);
// 再檢查一次標志字節，因為這個時候這個地方仍然可能是個重定向
b = mbb.get();
if (b == NO_AREA) {
loc.country = readString(readInt3());
mbb.position(countryOffset + 4);
} else
loc.country = readString(countryOffset);
// 讀取地區標志
loc.area = readArea(mbb.position());
} else if (b == NO_AREA) {
loc.country = readString(readInt3());
loc.area = readArea(offset + 8);
} else {
loc.country = readString(mbb.position() - 1);
loc.area = readArea(mbb.position());
}
return loc;
}
/**
* 從offset偏移開始解析后面的字節，讀出一個地區名
*
* @param offset
* @return 地區名字符串
* @throws IOException
*/
private String readArea(long offset) throws IOException {
ipFile.seek(offset);
byte b = ipFile.readByte();
if (b == 0x01 || b == 0x02) {
long areaOffset = readLong3(offset + 1);
if (areaOffset == 0)
return "未知地區";
else
return readString(areaOffset);
} else
return readString(offset);
}
/**
* @param offset
* @return
*/
private String readArea(int offset) {
mbb.position(offset);
byte b = mbb.get();
if (b == 0x01 || b == 0x02) {
int areaOffset = readInt3();
if (areaOffset == 0)
return "未知地區";
else
return readString(areaOffset);
} else
return readString(offset);
}
/**
* 從offset偏移處讀取一個以0結束的字符串
*
* @param offset
* @return 讀取的字符串，出錯返回空字符串
*/
private String readString(long offset) {
try {
ipFile.seek(offset);
int i;
for (i = 0, buf[i] = ipFile.readByte(); buf[i] != 0; buf[++i] = ipFile
.readByte())
;
if (i != 0)
return IPSeekerUtils.getString(buf, 0, i, "GBK");
} catch (IOException e) {
System.out.println(e.getMessage());
}
return "";
}
/**
* 從內存映射文件的offset位置得到一個0結尾字符串
*
* @param offset
* @return
*/
private String readString(int offset) {
try {
mbb.position(offset);
int i;
for (i = 0, buf[i] = mbb.get(); buf[i] != 0; buf[++i] = mbb.get())
;
if (i != 0)
return IPSeekerUtils.getString(buf, 0, i, "GBK");
} catch (IllegalArgumentException e) {
System.out.println(e.getMessage());
}
return "";
}
public String getAddress(String ip) {
String country = getCountry(ip).equals(" CZ88.NET") ? ""
: getCountry(ip);
String area = getArea(ip).equals(" CZ88.NET") ? "" : getArea(ip);
String address = country + " " + area;
return address.trim();
}
/**
* * 用來封裝ip相關信息，目前只有兩個字段，ip所在的國家和地區
*
*
* @author swallow
*/
public class IPLocation {
public String country;
public String area;
public IPLocation() {
country = area = "";
}
public IPLocation getCopy() {
IPLocation ret = new IPLocation();
ret.country = country;
ret.area = area;
return ret;
}
}
/**
* 一條IP范圍記錄，不僅包括國家和區域，也包括起始IP和結束IP *
*
*
* @author root
*/
public class IPEntry {
public String beginIp;
public String endIp;
public String country;
public String area;
public IPEntry() {
beginIp = endIp = country = area = "";
}
public String toString() {
return this.area + " " + this.country + "IP Χ:" + this.beginIp
+ "-" + this.endIp;
}
}
/**
* 操作工具類
*
* @author root
*
*/
public static class IPSeekerUtils {
/**
* 從ip的字符串形式得到字節數組形式
*
* @param ip
* 字符串形式的ip
* @return 字節數組形式的ip
*/
public static byte[] getIpByteArrayFromString(String ip) {
byte[] ret = new byte[4];
java.util.StringTokenizer st = new java.util.StringTokenizer(ip,
".");
try {
ret[0] = (byte) (Integer.parseInt(st.nextToken()) & 0xFF);
ret[1] = (byte) (Integer.parseInt(st.nextToken()) & 0xFF);
ret[2] = (byte) (Integer.parseInt(st.nextToken()) & 0xFF);
ret[3] = (byte) (Integer.parseInt(st.nextToken()) & 0xFF);
} catch (Exception e) {
System.out.println(e.getMessage());
}
return ret;
}
/**
* 對原始字符串進行編碼轉換，如果失敗，返回原始的字符串
*
* @param s
* 原始字符串
* @param srcEncoding
* 源編碼方式
* @param destEncoding
* 目標編碼方式
* @return 轉換編碼后的字符串，失敗返回原始字符串
*/
public static String getString(String s, String srcEncoding,
String destEncoding) {
try {
return new String(s.getBytes(srcEncoding), destEncoding);
} catch (UnsupportedEncodingException e) {
return s;
}
}
/**
* 根據某種編碼方式將字節數組轉換成字符串
*
* @param b
* 字節數組
* @param encoding
* 編碼方式
* @return 如果encoding不支持，返回一個缺省編碼的字符串
*/
public static String getString(byte[] b, String encoding) {
try {
return new String(b, encoding);
} catch (UnsupportedEncodingException e) {
return new String(b);
}
}
/**
* 根據某種編碼方式將字節數組轉換成字符串
*
* @param b
* 字節數組
* @param offset
* 要轉換的起始位置
* @param len
* 要轉換的長度
* @param encoding
* 編碼方式
* @return 如果encoding不支持，返回一個缺省編碼的字符串
*/
public static String getString(byte[] b, int offset, int len,
String encoding) {
try {
return new String(b, offset, len, encoding);
} catch (UnsupportedEncodingException e) {
return new String(b, offset, len);
}
}
/**
* @param ip
* ip的字節數組形式
* @return 字符串形式的ip
*/
public static String getIpStringFromBytes(byte[] ip) {
StringBuffer sb = new StringBuffer();
sb.append(ip[0] & 0xFF);
sb.append('.');
sb.append(ip[1] & 0xFF);
sb.append('.');
sb.append(ip[2] & 0xFF);
sb.append('.');
sb.append(ip[3] & 0xFF);
return sb.toString();
}
}
/**
* 獲取全部ip地址集合列表
*
* @return
*/
public List<String> getAllIp() {
List<String> list = new ArrayList<String>();
byte[] buf = new byte[4];
for (long i = ipBegin; i < ipEnd; i += IP_RECORD_LENGTH) {
try {
this.readIP(this.readLong3(i + 4), buf); // 讀取ip，最終ip放到buf中
String ip = IPSeekerUtils.getIpStringFromBytes(buf);
list.add(ip);
} catch (Exception e) {
// nothing
}
}
return list;
}
}
測試時，jdk版本選擇為1.7可以運行老師提供的源碼
由於ETL 只需要過濾和解析，不涉及到聚合，直接寫入到hbase.一次不需要reduce.
[root@node204 ~]# hadoop-daemon.sh stop namenode ## 切換active standby
配置程序的 hdfs 和zookeeper
@Override
public void setConf(Configuration conf) {
conf.set("fs.defaultFS", "hdfs://node203:8020");
// conf.set("yarn.resourcemanager.hostname", "node3");
conf.set("hbase.zookeeper.quorum", "node204,node205,node206");
this.conf = HBaseConfiguration.create(conf);
}
run main方法，並且通過runconfuguration配置參數 -d hdfs 上flume創建的文件目錄。如 -d 2019-08-28 用-風格的日期
