Nginx 中用Shell简单分析access.log日志-ITB运维部落—http://www.itbcn.cn—ITB运维技术交流之家平台
记录工作点滴
分享运维知识

Nginx 中用Shell简单分析access.log日志

查找访问频率最高的 URL 和次数:

cat access.log | awk -F ‘^A’ ‘{print $10}’ | sort | uniq -c

查找当前日志文件 500 错误的访问:

cat access.log | awk -F ‘^A’ ‘{if($5 == 500) print $0}’

查找当前日志文件 500 错误的数量:

cat access.log | awk -F ‘^A’ ‘{if($5 == 500) print $0}’ | wc -l

查找某一分钟内 500 错误访问的数量:

cat access.log | awk -F ‘^A’ ‘{if($5 == 500) print $0}’ | grep ’09:00’ | wc-l

查找耗时超过 1s 的慢请求:

tail -f access.log | awk -F ‘^A’ ‘{if($6>1) print $0}’

假如只想查看某些位:

tail -f access.log | awk -F ‘^A’ ‘{if($6>1) print $3″|”$4}’

查找 502 错误最多的 URL:

cat access.log | awk -F ‘^A’ ‘{if($5==502) print $11}’ | sort | uniq -c

查找 200 空白页

cat access.log | awk -F ‘^A’ ‘{if($5==200 && $8 < 100) print $3″|”$4″|”$11″|”$6}’

统计IP访问量(独立ip访问数量)

awk '{print $1}' access.log | sort -n | uniq | wc -l

查看某一时间段的IP访问量(4-5点)

grep "07/Apr/2017:0[4-5]" access.log | awk '{print $1}' | sort | uniq -c| sort -nr | wc -l  

查看访问最频繁的前100个IP

awk '{print $1}' access.log | sort -n |uniq -c | sort -rn | head -n 100

查看访问100次以上的IP

awk '{print $1}' access.log | sort -n |uniq -c |awk '{if($1 >100) print $0}'|sort -rn

查询某个IP的详细访问情况,按访问频率排序

grep '127.0.01' access.log |awk '{print $7}'|sort |uniq -c |sort -rn |head -n 100

页面访问统计

查看访问最频的页面(TOP100)

awk '{print $7}' access.log | sort |uniq -c | sort -rn | head -n 100

查看访问最频的页面([排除php页面】(TOP100)

grep -v ".php"  access.log | awk '{print $7}' | sort |uniq -c | sort -rn | head -n 100 

查看页面访问次数超过100次的页面

cat access.log | cut -d ' ' -f 7 | sort |uniq -c | awk '{if ($1 > 100) print $0}' | less

查看最近1000条记录,访问量最高的页面

tail -1000 access.log |awk '{print $7}'|sort|uniq -c|sort -nr|less

每秒请求量统计

统计每秒的请求数,top100的时间点(精确到秒)

awk '{print $4}' access.log |cut -c 14-21|sort|uniq -c|sort -nr|head -n 100

每分钟请求量统计

统计每分钟的请求数,top100的时间点(精确到分钟)

awk '{print $4}' access.log |cut -c 14-18|sort|uniq -c|sort -nr|head -n 100

每小时请求量统计

统计每小时的请求数,top100的时间点(精确到小时)

awk '{print $4}' access.log |cut -c 14-15|sort|uniq -c|sort -nr|head -n 100

性能分析

在nginx log中最后一个字段加入$request_time

列出传输时间超过 3 秒的页面,显示前20条

cat access.log|awk '($NF > 3){print $7}'|sort -n|uniq -c|sort -nr|head -20

列出php页面请求时间超过3秒的页面,并统计其出现的次数,显示前100条

cat access.log|awk '($NF > 1 &&  $7~/\.php/){print $7}'|sort -n|uniq -c|sort -nr|head -100

蜘蛛抓取统计

统计蜘蛛抓取次数

grep 'Baiduspider' access.log |wc -l

统计蜘蛛抓取404的次数

grep 'Baiduspider' access.log |grep '404' | wc -l

TCP连接统计

查看当前TCP连接数

netstat -tan | grep "ESTABLISHED" | grep ":80" | wc -l

用tcpdump嗅探80端口的访问看看谁最高

tcpdump -i eth0 -tnn dst port 80 -c 1000 | awk -F"." '{print $1"."$2"."$3"."$4}' | sort | uniq -c | sort -nr

1,对日志的统计

1).根据访问IP统计UV

awk '{print $1}'  access.log|sort | uniq -c |wc -l

2).统计访问URL统计PV

awk '{print $7}' access.log|wc -l

3).查询访问最频繁的URL

awk '{print $7}' access.log|sort | uniq -c |sort -n -k 1 -r|more

4).查询访问最频繁的IP

awk '{print $1}' access.log|sort | uniq -c |sort -n -k 1 -r|more

5).根据时间段统计查看日志

cat  access.log| sed -n '/14\/Mar\/2018:21/,/14\/Mar\/2018:22/p'|more

 

2,利用zabbix进行监控

1,zabbix 配置文件添加

vim /etc/zabbix/zabbix_server.conf

1
2
3
4
5
6
7
8
<span style="color:#656e6a;">### Option: AlertScriptsPath
#       Full path to location of custom alert scripts.
#       Default depends on compilation options.
#
# Mandatory: no
# Default:
# AlertScriptsPath=${datadir}/zabbix/alertscripts
</span><span style="color:#ff0000;">AlertScriptsPath=/usr/local/zabbix/sbin</span,<br><br>

2,vim /usr/local/zabbix/sbin/nginx-status.py

复制代码
import urllib2,base64,sys,getopt,re
 
def Usage():
    print "Usage: Nginx-Status.py -h 127.0.0.1 -p 80 -a [active|accepted|handled|requests|reading|writting|waiting]"
    sys.exit(2)
 
def main():
    if len(sys.argv) < 6:
        Usage()
 
    try:
        opts,args = getopt.getopt(sys.argv[1:], "h:p:a:")
        Dict = dict(opts)
    except getopt.GetoptError:
        Usage()
 
    Nginx_url = "http://" + Dict['-h'] + ":" + Dict['-p'] + "/nginx_status"
    Nginx_req = urllib2.Request(Nginx_url)
    Nginx_res = urllib2.urlopen(Nginx_req)
    Output_key = re.findall(r'\d{1,8}', Nginx_res.read())
 
    if ( Dict['-a'] == "active"):
            print Output_key[0]
    elif ( Dict['-a'] == "accepted"):
            print Output_key[1]
    elif ( Dict['-a'] == "handled"):
            print Output_key[2]
    elif ( Dict['-a'] == "requests"):
            print Output_key[3]
    elif ( Dict['-a'] == "reading"):
            print Output_key[4]
    elif ( Dict['-a'] == "writting"):
            print Output_key[5]
    elif ( Dict['-a'] == "waiting"):
            print Output_key[6]
    else:
            print "unknown!!"
            sys.exit(1)
 
if __name__ == '__main__':
    main()
复制代码

3,nginx_status_templates.xml

复制代码
<?xml version="1.0" encoding="UTF-8"?>
<zabbix_export>
    <version>2.0</version>
    <date>2015-11-03T01:37:04Z</date>
    <groups>
        <group>
            <name>nginx-status</name>
        </group>
    </groups>
    <templates>
        <template>
            <template>nginx-status</template>
            <name>nginx-status</name>
            <description/>
            <groups>
                <group>
                    <name>nginx-status</name>
                </group>
            </groups>
            <applications/>
            <items>
                <item>
                    <name>nginx-accepted</name>
                    <type>10</type>
                    <snmp_community/>
                    <multiplier>0</multiplier>
                    <snmp_oid/>
                    <key>nginx-status.py["-h","{HOST.IP}","-p","80","-a","accepted"]</key>
                    <delay>30</delay>
                    <history>90</history>
                    <trends>365</trends>
                    <status>0</status>
                    <value_type>3</value_type>
                    <allowed_hosts/>
                    <units/>
                    <delta>0</delta>
                    <snmpv3_contextname/>
                    <snmpv3_securityname/>
                    <snmpv3_securitylevel>0</snmpv3_securitylevel>
                    <snmpv3_authprotocol>0</snmpv3_authprotocol>
                    <snmpv3_authpassphrase/>
                    <snmpv3_privprotocol>0</snmpv3_privprotocol>
                    <snmpv3_privpassphrase/>
                    <formula>1</formula>
                    <delay_flex/>
                    <params/>
                    <ipmi_sensor/>
                    <data_type>0</data_type>
                    <authtype>0</authtype>
                    <username/>
                    <password/>
                    <publickey/>
                    <privatekey/>
                    <port/>
                    <description/>
                    <inventory_link>0</inventory_link>
                    <applications/>
                    <valuemap/>
                    <logtimefmt/>
                </item>
                <item>
                    <name>nginx-active</name>
                    <type>10</type>
                    <snmp_community/>
                    <multiplier>0</multiplier>
                    <snmp_oid/>
                    <key>nginx-status.py["-h","{HOST.IP}","-p","80","-a","active"]</key>
                    <delay>30</delay>
                    <history>90</history>
                    <trends>365</trends>
                    <status>0</status>
                    <value_type>3</value_type>
                    <allowed_hosts/>
                    <units/>
                    <delta>0</delta>
                    <snmpv3_contextname/>
                    <snmpv3_securityname/>
                    <snmpv3_securitylevel>0</snmpv3_securitylevel>
                    <snmpv3_authprotocol>0</snmpv3_authprotocol>
                    <snmpv3_authpassphrase/>
                    <snmpv3_privprotocol>0</snmpv3_privprotocol>
                    <snmpv3_privpassphrase/>
                    <formula>1</formula>
                    <delay_flex/>
                    <params/>
                    <ipmi_sensor/>
                    <data_type>0</data_type>
                    <authtype>0</authtype>
                    <username/>
                    <password/>
                    <publickey/>
                    <privatekey/>
                    <port/>
                    <description/>
                    <inventory_link>0</inventory_link>
                    <applications/>
                    <valuemap/>
                    <logtimefmt/>
                </item>
                <item>
                    <name>nginx-handled</name>
                    <type>10</type>
                    <snmp_community/>
                    <multiplier>0</multiplier>
                    <snmp_oid/>
                    <key>nginx-status.py["-h","{HOST.IP}","-p","80","-a","handled"]</key>
                    <delay>30</delay>
                    <history>90</history>
                    <trends>365</trends>
                    <status>0</status>
                    <value_type>3</value_type>
                    <allowed_hosts/>
                    <units/>
                    <delta>0</delta>
                    <snmpv3_contextname/>
                    <snmpv3_securityname/>
                    <snmpv3_securitylevel>0</snmpv3_securitylevel>
                    <snmpv3_authprotocol>0</snmpv3_authprotocol>
                    <snmpv3_authpassphrase/>
                    <snmpv3_privprotocol>0</snmpv3_privprotocol>
                    <snmpv3_privpassphrase/>
                    <formula>1</formula>
                    <delay_flex/>
                    <params/>
                    <ipmi_sensor/>
                    <data_type>0</data_type>
                    <authtype>0</authtype>
                    <username/>
                    <password/>
                    <publickey/>
                    <privatekey/>
                    <port/>
                    <description/>
                    <inventory_link>0</inventory_link>
                    <applications/>
                    <valuemap/>
                    <logtimefmt/>
                </item>
                <item>
                    <name>nginx-reading</name>
                    <type>10</type>
                    <snmp_community/>
                    <multiplier>0</multiplier>
                    <snmp_oid/>
                    <key>nginx-status.py["-h","{HOST.IP}","-p","80","-a","reading"]</key>
                    <delay>30</delay>
                    <history>90</history>
                    <trends>365</trends>
                    <status>0</status>
                    <value_type>3</value_type>
                    <allowed_hosts/>
                    <units/>
                    <delta>0</delta>
                    <snmpv3_contextname/>
                    <snmpv3_securityname/>
                    <snmpv3_securitylevel>0</snmpv3_securitylevel>
                    <snmpv3_authprotocol>0</snmpv3_authprotocol>
                    <snmpv3_authpassphrase/>
                    <snmpv3_privprotocol>0</snmpv3_privprotocol>
                    <snmpv3_privpassphrase/>
                    <formula>1</formula>
                    <delay_flex/>
                    <params/>
                    <ipmi_sensor/>
                    <data_type>0</data_type>
                    <authtype>0</authtype>
                    <username/>
                    <password/>
                    <publickey/>
                    <privatekey/>
                    <port/>
                    <description/>
                    <inventory_link>0</inventory_link>
                    <applications/>
                    <valuemap/>
                    <logtimefmt/>
                </item>
                <item>
                    <name>nginx-requests</name>
                    <type>10</type>
                    <snmp_community/>
                    <multiplier>0</multiplier>
                    <snmp_oid/>
                    <key>nginx-status.py["-h","{HOST.IP}","-p","80","-a","requests"]</key>
                    <delay>30</delay>
                    <history>90</history>
                    <trends>365</trends>
                    <status>0</status>
                    <value_type>3</value_type>
                    <allowed_hosts/>
                    <units/>
                    <delta>0</delta>
                    <snmpv3_contextname/>
                    <snmpv3_securityname/>
                    <snmpv3_securitylevel>0</snmpv3_securitylevel>
                    <snmpv3_authprotocol>0</snmpv3_authprotocol>
                    <snmpv3_authpassphrase/>
                    <snmpv3_privprotocol>0</snmpv3_privprotocol>
                    <snmpv3_privpassphrase/>
                    <formula>1</formula>
                    <delay_flex/>
                    <params/>
                    <ipmi_sensor/>
                    <data_type>0</data_type>
                    <authtype>0</authtype>
                    <username/>
                    <password/>
                    <publickey/>
                    <privatekey/>
                    <port/>
                    <description/>
                    <inventory_link>0</inventory_link>
                    <applications/>
                    <valuemap/>
                    <logtimefmt/>
                </item>
                <item>
                    <name>nginx-waiting</name>
                    <type>10</type>
                    <snmp_community/>
                    <multiplier>0</multiplier>
                    <snmp_oid/>
                    <key>nginx-status.py["-h","{HOST.IP}","-p","80","-a","waiting"]</key>
                    <delay>30</delay>
                    <history>90</history>
                    <trends>365</trends>
                    <status>0</status>
                    <value_type>3</value_type>
                    <allowed_hosts/>
                    <units/>
                    <delta>0</delta>
                    <snmpv3_contextname/>
                    <snmpv3_securityname/>
                    <snmpv3_securitylevel>0</snmpv3_securitylevel>
                    <snmpv3_authprotocol>0</snmpv3_authprotocol>
                    <snmpv3_authpassphrase/>
                    <snmpv3_privprotocol>0</snmpv3_privprotocol>
                    <snmpv3_privpassphrase/>
                    <formula>1</formula>
                    <delay_flex/>
                    <params/>
                    <ipmi_sensor/>
                    <data_type>0</data_type>
                    <authtype>0</authtype>
                    <username/>
                    <password/>
                    <publickey/>
                    <privatekey/>
                    <port/>
                    <description/>
                    <inventory_link>0</inventory_link>
                    <applications/>
                    <valuemap/>
                    <logtimefmt/>
                </item>
                <item>
                    <name>nginx-writting</name>
                    <type>10</type>
                    <snmp_community/>
                    <multiplier>0</multiplier>
                    <snmp_oid/>
                    <key>nginx-status.py["-h","{HOST.IP}","-p","80","-a","writting"]</key>
                    <delay>30</delay>
                    <history>90</history>
                    <trends>365</trends>
                    <status>0</status>
                    <value_type>3</value_type>
                    <allowed_hosts/>
                    <units/>
                    <delta>0</delta>
                    <snmpv3_contextname/>
                    <snmpv3_securityname/>
                    <snmpv3_securitylevel>0</snmpv3_securitylevel>
                    <snmpv3_authprotocol>0</snmpv3_authprotocol>
                    <snmpv3_authpassphrase/>
                    <snmpv3_privprotocol>0</snmpv3_privprotocol>
                    <snmpv3_privpassphrase/>
                    <formula>1</formula>
                    <delay_flex/>
                    <params/>
                    <ipmi_sensor/>
                    <data_type>0</data_type>
                    <authtype>0</authtype>
                    <username/>
                    <password/>
                    <publickey/>
                    <privatekey/>
                    <port/>
                    <description/>
                    <inventory_link>0</inventory_link>
                    <applications/>
                    <valuemap/>
                    <logtimefmt/>
                </item>
            </items>
            <discovery_rules/>
            <macros/>
            <templates/>
            <screens/>
        </template>
    </templates>
    <graphs>
        <graph>
            <name>nginx-clients-status</name>
            <width>900</width>
            <height>200</height>
            <yaxismin>0.0000</yaxismin>
            <yaxismax>100.0000</yaxismax>
            <show_work_period>1</show_work_period>
            <show_triggers>1</show_triggers>
            <type>0</type>
            <show_legend>1</show_legend>
            <show_3d>0</show_3d>
            <percent_left>0.0000</percent_left>
            <percent_right>0.0000</percent_right>
            <ymin_type_1>0</ymin_type_1>
            <ymax_type_1>0</ymax_type_1>
            <ymin_item_1>0</ymin_item_1>
            <ymax_item_1>0</ymax_item_1>
            <graph_items>
                <graph_item>
                    <sortorder>0</sortorder>
                    <drawtype>0</drawtype>
                    <color>00C800</color>
                    <yaxisside>0</yaxisside>
                    <calc_fnc>2</calc_fnc>
                    <type>0</type>
                    <item>
                        <host>nginx-status</host>
                        <key>nginx-status.py["-h","{HOST.IP}","-p","80","-a","active"]</key>
                    </item>
                </graph_item>
                <graph_item>
                    <sortorder>1</sortorder>
                    <drawtype>0</drawtype>
                    <color>C80000</color>
                    <yaxisside>0</yaxisside>
                    <calc_fnc>2</calc_fnc>
                    <type>0</type>
                    <item>
                        <host>nginx-status</host>
                        <key>nginx-status.py["-h","{HOST.IP}","-p","80","-a","reading"]</key>
                    </item>
                </graph_item>
                <graph_item>
                    <sortorder>2</sortorder>
                    <drawtype>0</drawtype>
                    <color>0000C8</color>
                    <yaxisside>0</yaxisside>
                    <calc_fnc>2</calc_fnc>
                    <type>0</type>
                    <item>
                        <host>nginx-status</host>
                        <key>nginx-status.py["-h","{HOST.IP}","-p","80","-a","waiting"]</key>
                    </item>
                </graph_item>
                <graph_item>
                    <sortorder>3</sortorder>
                    <drawtype>0</drawtype>
                    <color>C800C8</color>
                    <yaxisside>0</yaxisside>
                    <calc_fnc>2</calc_fnc>
                    <type>0</type>
                    <item>
                        <host>nginx-status</host>
                        <key>nginx-status.py["-h","{HOST.IP}","-p","80","-a","writting"]</key>
                    </item>
                </graph_item>
            </graph_items>
        </graph>
        <graph>
            <name>nginx-socket-status</name>
            <width>900</width>
            <height>200</height>
            <yaxismin>0.0000</yaxismin>
            <yaxismax>100.0000</yaxismax>
            <show_work_period>1</show_work_period>
            <show_triggers>1</show_triggers>
            <type>0</type>
            <show_legend>1</show_legend>
            <show_3d>0</show_3d>
            <percent_left>0.0000</percent_left>
            <percent_right>0.0000</percent_right>
            <ymin_type_1>0</ymin_type_1>
            <ymax_type_1>0</ymax_type_1>
            <ymin_item_1>0</ymin_item_1>
            <ymax_item_1>0</ymax_item_1>
            <graph_items>
                <graph_item>
                    <sortorder>0</sortorder>
                    <drawtype>0</drawtype>
                    <color>00C800</color>
                    <yaxisside>0</yaxisside>
                    <calc_fnc>2</calc_fnc>
                    <type>0</type>
                    <item>
                        <host>nginx-status</host>
                        <key>nginx-status.py["-h","{HOST.IP}","-p","80","-a","handled"]</key>
                    </item>
                </graph_item>
                <graph_item>
                    <sortorder>1</sortorder>
                    <drawtype>0</drawtype>
                    <color>0000C8</color>
                    <yaxisside>0</yaxisside>
                    <calc_fnc>2</calc_fnc>
                    <type>0</type>
                    <item>
                        <host>nginx-status</host>
                        <key>nginx-status.py["-h","{HOST.IP}","-p","80","-a","requests"]</key>
                    </item>
                </graph_item>
                <graph_item>
                    <sortorder>2</sortorder>
                    <drawtype>0</drawtype>
                    <color>C800C8</color>
                    <yaxisside>0</yaxisside>
                    <calc_fnc>2</calc_fnc>
                    <type>0</type>
                    <item>
                        <host>nginx-status</host>
                        <key>nginx-status.py["-h","{HOST.IP}","-p","80","-a","accepted"]</key>
                    </item>
                </graph_item>
            </graph_items>
        </graph>
    </graphs>
</zabbix_export>
复制代码
未经允许不得转载:ITB运维部落—http://www.itbcn.cn—ITB运维技术交流之家平台 » Nginx 中用Shell简单分析access.log日志

如果文章对你有帮助,欢迎点击上方按钮打赏作者

评论 抢沙发

  • 昵称 (必填)
  • 邮箱 (必填)
  • 网址