如何用Logstash 收集 IIS 日志？

日志样例


查看 IIS 日志配置，选择格式为 W3C（默认字段设置）保存生效。

1. [backcolor=transparent]2016[backcolor=transparent]-[backcolor=transparent]02[backcolor=transparent]-[backcolor=transparent]25[backcolor=transparent] [backcolor=transparent]01[backcolor=transparent]:[backcolor=transparent]27[backcolor=transparent]:[backcolor=transparent]04[backcolor=transparent] [backcolor=transparent]112.74[backcolor=transparent].[backcolor=transparent]74.124[backcolor=transparent] GET [backcolor=transparent]/[backcolor=transparent]goods[backcolor=transparent]/[backcolor=transparent]list[backcolor=transparent]/[backcolor=transparent]0[backcolor=transparent]/[backcolor=transparent]1.html[backcolor=transparent] [backcolor=transparent]-[backcolor=transparent] [backcolor=transparent]80[backcolor=transparent] [backcolor=transparent]-[backcolor=transparent] [backcolor=transparent]66.249[backcolor=transparent].[backcolor=transparent]65.102[backcolor=transparent] [backcolor=transparent]Mozilla[backcolor=transparent]/[backcolor=transparent]5.0[backcolor=transparent]+([backcolor=transparent]compatible[backcolor=transparent];+[backcolor=transparent]Googlebot[backcolor=transparent]/[backcolor=transparent]2.1[backcolor=transparent];++[backcolor=transparent]http[backcolor=transparent]:[backcolor=transparent]//www.google.com/bot.html) 404 0 2 703

1. [backcolor=transparent]input [backcolor=transparent]{
2. [backcolor=transparent]  file [backcolor=transparent]{
3. [backcolor=transparent]    type [backcolor=transparent]=>[backcolor=transparent] [backcolor=transparent]"iis_log_1"
4. [backcolor=transparent]    path [backcolor=transparent]=>[backcolor=transparent] [backcolor=transparent][[backcolor=transparent]"C:/inetpub/logs/LogFiles/W3SVC1/*.log"[backcolor=transparent]]
5. [backcolor=transparent]    start_position [backcolor=transparent]=>[backcolor=transparent] [backcolor=transparent]"beginning"
6. [backcolor=transparent]  [backcolor=transparent]}
7. [backcolor=transparent]}
9. [backcolor=transparent]filter [backcolor=transparent]{
10. [backcolor=transparent]  [backcolor=transparent]if[backcolor=transparent] [backcolor=transparent][[backcolor=transparent]type[backcolor=transparent]][backcolor=transparent] [backcolor=transparent]==[backcolor=transparent] [backcolor=transparent]"iis_log_1"[backcolor=transparent] [backcolor=transparent]{
11. [backcolor=transparent]  [backcolor=transparent]#ignore log comments
12. [backcolor=transparent]  [backcolor=transparent]if[backcolor=transparent] [backcolor=transparent][[backcolor=transparent]message[backcolor=transparent]][backcolor=transparent] [backcolor=transparent]=~[backcolor=transparent] [backcolor=transparent]"^#"[backcolor=transparent] [backcolor=transparent]{
13. [backcolor=transparent]    drop [backcolor=transparent]{}
14. [backcolor=transparent]  [backcolor=transparent]}
16. [backcolor=transparent]  grok [backcolor=transparent]{
17. [backcolor=transparent]    [backcolor=transparent]# check that fields match your IIS log settings
18. [backcolor=transparent]    match [backcolor=transparent]=>[backcolor=transparent] [backcolor=transparent][[backcolor=transparent]"message"[backcolor=transparent],[backcolor=transparent] [backcolor=transparent]"%{TIMESTAMP_ISO8601:log_timestamp} %{IPORHOST:site} %{WORD:method} %{URIPATH:page} %{NOTSPACE:querystring} %{NUMBER:port} %{NOTSPACE:username} %{IPORHOST:clienthost} %{NOTSPACE:useragent} %{NUMBER:response} %{NUMBER:subresponse} %{NUMBER:scstatus} %{NUMBER:time_taken}"[backcolor=transparent]]
19. [backcolor=transparent]  [backcolor=transparent]}
21. [backcolor=transparent]    date [backcolor=transparent]{
22. [backcolor=transparent]    match [backcolor=transparent]=>[backcolor=transparent] [backcolor=transparent][[backcolor=transparent] [backcolor=transparent]"log_timestamp"[backcolor=transparent],[backcolor=transparent] [backcolor=transparent]"YYYY-MM-dd HH:mm:ss"[backcolor=transparent] [backcolor=transparent]]
23. [backcolor=transparent]      timezone [backcolor=transparent]=>[backcolor=transparent] [backcolor=transparent]"Etc/UTC"
24. [backcolor=transparent]  [backcolor=transparent]}[backcolor=transparent]    
26. [backcolor=transparent]  useragent [backcolor=transparent]{
27. [backcolor=transparent]    source[backcolor=transparent]=>[backcolor=transparent] [backcolor=transparent]"useragent"
28. [backcolor=transparent]    prefix[backcolor=transparent]=>[backcolor=transparent] [backcolor=transparent]"browser"
29. [backcolor=transparent]  [backcolor=transparent]}
31. [backcolor=transparent]  mutate [backcolor=transparent]{
32. [backcolor=transparent]    remove_field [backcolor=transparent]=>[backcolor=transparent] [backcolor=transparent][[backcolor=transparent] [backcolor=transparent]"log_timestamp"[backcolor=transparent]]
33. [backcolor=transparent]  [backcolor=transparent]}
34. [backcolor=transparent]  [backcolor=transparent]}
35. [backcolor=transparent]}
37. [backcolor=transparent]output [backcolor=transparent]{
38. [backcolor=transparent]  [backcolor=transparent]if[backcolor=transparent] [backcolor=transparent][[backcolor=transparent]type[backcolor=transparent]][backcolor=transparent] [backcolor=transparent]==[backcolor=transparent] [backcolor=transparent]"iis_log_1"[backcolor=transparent] [backcolor=transparent]{
39. [backcolor=transparent]  logservice [backcolor=transparent]{
40. [backcolor=transparent]        codec [backcolor=transparent]=>[backcolor=transparent] [backcolor=transparent]"json"
41. [backcolor=transparent]        endpoint [backcolor=transparent]=>[backcolor=transparent] [backcolor=transparent]"***"
42. [backcolor=transparent]        project [backcolor=transparent]=>[backcolor=transparent] [backcolor=transparent]"***"
43. [backcolor=transparent]        logstore [backcolor=transparent]=>[backcolor=transparent] [backcolor=transparent]"***"
44. [backcolor=transparent]        topic [backcolor=transparent]=>[backcolor=transparent] [backcolor=transparent]""
45. [backcolor=transparent]        source [backcolor=transparent]=>[backcolor=transparent] [backcolor=transparent]""
46. [backcolor=transparent]        access_key_id [backcolor=transparent]=>[backcolor=transparent] [backcolor=transparent]"***"
47. [backcolor=transparent]        access_key_secret [backcolor=transparent]=>[backcolor=transparent] [backcolor=transparent]"***"
48. [backcolor=transparent]        max_send_retry [backcolor=transparent]=>[backcolor=transparent] [backcolor=transparent]10
49. [backcolor=transparent]    [backcolor=transparent]}
50. [backcolor=transparent]    [backcolor=transparent]}
51. [backcolor=transparent]}

注意：

• 配置文件格式必须以 UTF-8 无 BOM 格式编码，可以通过notepad++修改文件编码格式。
• path 填写文件路径时请使用UNIX模式的分隔符，如：C:/test/multiline/*.log，否则无法支持模糊匹配。
• type 字段需要统一修改并在该文件内保持一致，如果单台机器存在多个 Logstash 配置文件，需要保证各配置 type 字段唯一，否则会导致数据处理的错乱。