网站流量分析

goaccess --log-format CADDY localhost.log -o /path/to/you/webroot/goaccess.html

goaccess --log-format CADDY localhost.log

goaccess --log-format CADDY localhost.log  -o /path/to/you/webroot/goaccess.html --real-time-html

{
	"level": "info",
	"ts": 1683809163.1758618,
	"logger": "http.log.access.log1",
	"msg": "handled request",
	"request": {
		"remote_ip": "172.69.67.51",
		"remote_port": "63594",
		"proto": "HTTP/2.0",
		"method": "HEAD",
		"host": "youerning.top",
		"uri": "/",
		"headers": {
			"Cdn-Loop": ["cloudflare"],
			"Cache-Control": ["no-cache"],
			"Cf-Ipcountry": ["US"],
			"X-Forwarded-For": ["216.144.248.23"],
			"X-Forwarded-Proto": ["https"],
			"Accept": ["text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"],
			"Referer": ["https://youerning.top"],
			"Accept-Encoding": ["gzip"],
			"Accept-Language": ["en-US,en;q=0.8"],
			"Cf-Connecting-Ip": ["216.144.248.23"],
			"Cf-Ray": ["7c5a85c508a6aa34-DFW"],
			"Content-Length": ["0"],
			"Cf-Visitor": ["{\"scheme\":\"https\"}"],
			"User-Agent": ["Mozilla/5.0+(compatible; UptimeRobot/2.0; http://www.uptimerobot.com/)"]
		},
		"tls": {
			"proto": "h2",
			"server_name": "youerning.top"
		}
	},
	"user_id": "",
	"duration": 0.001344724,
	"size": 0,
	"status": 200,
	"resp_headers": {
		"Content-Length": ["30986"],
		"Server": ["Caddy"],
		"Alt-Svc": ["h3=\":443\"; ma=2592000"],
		"Etag": ["\"ruhtxgnwq\""],
		"Content-Type": ["text/html; charset=utf-8"],
		"Last-Modified": ["Thu, 11 May 2023 12:12:52 GMT"],
		"Accept-Ranges": ["bytes"]
	}
}

import json
import time
from urllib.parse import urlparse
from collections import Counter
from datetime import datetime, timedelta

from ua_parser.user_agent_parser import Parse as ua_parse

LAST_HOURS = 24 * 7
now = datetime.now()
analysis_start_time = now - timedelta(hours=LAST_HOURS)

user_agents = Counter()
# client_ips = Counter()
# hosts = Counter()
uris = Counter()
referers = Counter()
# statuses = Counter()
# durations = Counter()

with open(sys.argv[1], encoding="utf8") as rf:
    while True:
        ts = None
        line = rf.readline()
        if not line:
            break
        # 处理format没有指定为json时的日志
        if line.startswith("2"):
            start_index = line.find("{")
            if start_index == -1:
                print(f"日志格式错误(无法找到'{{'起始位置位置): {line}")
                continue
            ts_index = line.find(".")
            if ts_index == -1:
                print(f"日志格式错误(解析时间格式): {line}")
                continue

            try:
                # 2023/05/09 09:06:17.323
                ts = time.strptime(line[:ts_index], "%Y/%m/%d %H:%M:%S")
                ts = datetime(ts.tm_year, ts.tm_mon, ts.tm_mday, ts.tm_hour+8, ts.tm_min, ts.tm_sec, 0)
            except Exception as exc:
                print("错误的时间格式: ", exc, line)
                continue

            line = line[start_index:]
        try:
            record = json.loads(line)
        except Exception as exc:
            print(f"日志格式错误(解析json格式失败): {line}", exc)

        if not ts:
            ts = datetime.fromtimestamp(record["ts"])

        if ts< analysis_start_time:
            continue
        
        request: dict = record.get("request")
        if request.get("headers").get("User-Agent"):
            user_agent = request.get("headers").get("User-Agent")[0]
        else:
            user_agent = "Unknow"
        referer = request.get("headers").get("Referer")
        if referer:
            referer = referer[0]
            try:
                referers.update([urlparse(referer).netloc])
            except Exception as exc:
                print("解析referer失败:", referer, exc)

        user_agnet = ua_parse(user_agent).get("user_agent").get("family")
        user_agents.update([user_agent])
        # client_ips.update([request.get("remote_ip")])
        # hosts.update([request.get("host")])
        uris.update([request.get("uri")])
        # statuses.update([record.get("status")])
        # durations.update([record.get("duration")])


print("user agent: \n%s" % "\n".join([f"{n[0]}|{n[1]}" for n in user_agents.most_common(5)]))
print("uris: \n%s" % "\n".join([f"{n[0]}|{n[1]}" for n in uris.most_common(5)]))
print("referers: \n%s" % "\n".join([f"{n[0]}|{n[1]}" for n in referers.most_common(5)]))

user agent: 
Edge/12.246|1631
Mozilla/5.0+(compatible; UptimeRobot/2.0; http://www.uptimerobot.com/)|1201
Chrome/74.0.3729.169|608
Chrome/104.0.0.0/537.36|440
Firefox/112.0|275

uris: 
/|1664
/favicon.ico|90
/js/menu.js|80
/css/style.css|76
/page/2/|35
referers: 
youerning.top|3569
www.google.com|11
blog.51cto.com|3
youerning.top:8080|2
baidu.com|2