Образовательный проект «SnakeProject» Михаила Козлова

⇒ FreeBSD and Nix ⇐

CISCO

Voice(Asterisk\Cisco)

Microsoft

Powershell

Python

SQL\T-SQL

1С

Общая

WEB Разработка

ORACLE SQL \ JAVA

Мото

Стрельба, пневматика, оружие

Саморазвитие и психология

Graylog2 Nginx json log

Graylog2 Nginx json log

Нас не устраивает предыдущий вариант, хотим больше полей

Мне конкретно нужно было поле $host для graylog

Устанавливаем этот пак:

https://github.com/petestorey26/graylog-content-pack-nginx-json/blob/master/content_pack.json

Или копипастим:

{ "name": "nginx-json", "description": "This content pack supports nginx sending JSON formatted log messages over syslog, allowing for arbitrary addition of fields without having to add regex extractors, as well as faster performance on the graylog side as it has reduced numbers of regexes running against the input compared with the original graylog version. It then blanks the "json" field used as an intermediate step, and reduces the message field to just show the request path, reducing the storage requirements.", "category": "nginx", "inputs": [ { "id": "5a4bc0be12522a5ae7bf6fb4", "title": "nginx error_log", "configuration": { "override_source": "", "allow_override_date": true, "recv_buffer_size": 1048576, "port": 12302, "bind_address": "0.0.0.0" }, "static_fields": { "from_nginx": "true", "nginx_error": "true" }, "type": "org.graylog2.inputs.syslog.udp.SyslogUDPInput", "global": true, "extractors": [ { "title": "Timestamp", "type": "REGEX", "cursor_strategy": "COPY", "target_field": "timestamp", "source_field": "message", "configuration": { "regex_value": "^.*:\s(\d\d\d\d/\d\d/\d\d\s\d\d:\d\d:\d\d)\s.*$" }, "converters": [ { "type": "DATE", "configuration": { "date_format": "yyyy/MM/dd HH:mm:ss " } } ], "condition_type": "NONE", "condition_value": "", "order": 0 }, { "title": "server", "type": "REGEX", "cursor_strategy": "COPY", "target_field": "server", "source_field": "message", "configuration": { "regex_value": "server:\s(.+?)(,|$)" }, "converters": [], "condition_type": "STRING", "condition_value": "server", "order": 0 }, { "title": "remote_addr/client", "type": "REGEX", "cursor_strategy": "COPY", "target_field": "remote_addr", "source_field": "message", "configuration": { "regex_value": "client:\s(.+?)(,|$)" }, "converters": [], "condition_type": "STRING", "condition_value": "client", "order": 0 }, { "title": "host", "type": "REGEX", "cursor_strategy": "COPY", "target_field": "host", "source_field": "message", "configuration": { "regex_value": "host:\s"(.+?)"(,|$)" }, "converters": [], "condition_type": "STRING", "condition_value": "host", "order": 0 }, { "title": "request_path/request", "type": "REGEX", "cursor_strategy": "COPY", "target_field": "request_path", "source_field": "message", "configuration": { "regex_value": "request:\s"(.+?)"(,|$)" }, "converters": [], "condition_type": "STRING", "condition_value": "request", "order": 0 }, { "title": "request_verb", "type": "REGEX", "cursor_strategy": "COPY", "target_field": "request_verb", "source_field": "message", "configuration": { "regex_value": "request:\s"(GET|HEAD|POST|PUT|DELETE|TRACE|OPTIONS|CONNECT|PATCH).+"(,|$)" }, "converters": [], "condition_type": "STRING", "condition_value": "request", "order": 0 } ] }, { "id": "5a4bc0be12522a5ae7bf6fb3", "title": "nginx access_log", "configuration": { "expand_structured_data": true, "recv_buffer_size": 1048576, "port": 12301, "override_source": "", "force_rdns": false, "allow_override_date": true, "bind_address": "0.0.0.0", "store_full_message": false }, "static_fields": { "from_nginx": "true", "nginx_access": "true" }, "type": "org.graylog2.inputs.syslog.udp.SyslogUDPInput", "global": true, "extractors": [ { "title": "Get JSON from syslog message", "type": "REGEX", "cursor_strategy": "COPY", "target_field": "json", "source_field": "message", "configuration": { "regex_value": "nginx:\s+(.*)" }, "converters": [], "condition_type": "NONE", "condition_value": "", "order": 0 }, { "title": "Extract JSON fields", "type": "JSON", "cursor_strategy": "COPY", "target_field": "", "source_field": "json", "configuration": { "flatten": true, "list_separator": ", ", "kv_separator": "=", "key_prefix": "", "key_separator": "_", "replace_key_whitespace": false, "key_whitespace_replacement": "_" }, "converters": [], "condition_type": "NONE", "condition_value": "", "order": 1 }, { "title": "Empty JSON field", "type": "REGEX_REPLACE", "cursor_strategy": "COPY", "target_field": "json", "source_field": "json", "configuration": { "replacement": "-", "regex": ".*" }, "converters": [], "condition_type": "NONE", "condition_value": "", "order": 2 }, { "title": "Reduced message to path", "type": "REGEX_REPLACE", "cursor_strategy": "COPY", "target_field": "message", "source_field": "message", "configuration": { "replacement": "$1", "regex": ".*request": "(.*?)".*" }, "converters": [], "condition_type": "NONE", "condition_value": "", "order": 3 } ] } ], "streams": [ { "id": "547b2a2dd4c6c10b4f1b93ce", "title": "nginx HTTP 404s", "description": "All requests that were answered with a HTTP 404 by nginx", "disabled": false, "matching_type": "AND", "stream_rules": [ { "type": "EXACT", "field": "from_nginx", "value": "true", "inverted": false, "description": "" }, { "type": "EXACT", "field": "response_status", "value": "404", "inverted": false, "description": "" } ], "outputs": [], "default_stream": false }, { "id": "5445733cd4c6d7d480b5f48b", "title": "nginx errors", "description": "All requests that were logged into the nginx error_log", "disabled": false, "matching_type": "AND", "stream_rules": [ { "type": "EXACT", "field": "nginx_error", "value": "true", "inverted": false, "description": "" } ], "outputs": [], "default_stream": false }, { "id": "5445736fd4c6d7d480b5f4c2", "title": "nginx requests", "description": "All requests that were logged into the nginx access_log", "disabled": false, "matching_type": "AND", "stream_rules": [ { "type": "EXACT", "field": "nginx_access", "value": "true", "inverted": false, "description": "" } ], "outputs": [], "default_stream": false }, { "id": "547b2a77d4c6c10b4f1b941f", "title": "nginx HTTP 5XXs", "description": "All requests that were answered with a HTTP code in the 500 range by nginx", "disabled": false, "matching_type": "AND", "stream_rules": [ { "type": "EXACT", "field": "from_nginx", "value": "true", "inverted": false, "description": "" }, { "type": "GREATER", "field": "response_status", "value": "499", "inverted": false, "description": "" } ], "outputs": [], "default_stream": false }, { "id": "547b2ad4d4c6c10b4f1b9485", "title": "nginx HTTP 4XXs", "description": "All requests that were answered with a HTTP code in the 400 range by nginx", "disabled": false, "matching_type": "AND", "stream_rules": [ { "type": "EXACT", "field": "from_nginx", "value": "true", "inverted": false, "description": "" }, { "type": "GREATER", "field": "response_status", "value": "399", "inverted": false, "description": "" }, { "type": "SMALLER", "field": "response_status", "value": "500", "inverted": false, "description": "" } ], "outputs": [], "default_stream": false }, { "id": "547b29b6d4c6c10b4f1b934d", "title": "nginx", "description": "All requests that were logged into the nginx access_log or nginx_error_log", "disabled": false, "matching_type": "AND", "stream_rules": [ { "type": "EXACT", "field": "from_nginx", "value": "true", "inverted": false, "description": "" } ], "outputs": [], "default_stream": false } ], "outputs": [], "dashboards": [ { "title": "nginx overview", "description": "Overview of requests handled by nginx", "dashboard_widgets": [ { "description": "Response codes last hour", "type": "QUICKVALUES", "cache_time": 10, "configuration": { "timerange": { "type": "relative", "range": 3600 }, "field": "response_status", "stream_id": "5445736fd4c6d7d480b5f4c2", "query": "*" }, "col": 3, "row": 4, "height": 3, "width": 1 }, { "description": "Response codes last 24h", "type": "QUICKVALUES", "cache_time": 10, "configuration": { "timerange": { "type": "relative", "range": 86400 }, "field": "response_status", "stream_id": "5445736fd4c6d7d480b5f4c2", "query": "*" }, "col": 2, "row": 4, "height": 3, "width": 1 }, { "description": "Requests last 24h", "type": "SEARCH_RESULT_CHART", "cache_time": 10, "configuration": { "interval": "minute", "timerange": { "type": "relative", "range": 86400 }, "stream_id": "5445736fd4c6d7d480b5f4c2", "query": "*" }, "col": 2, "row": 1, "height": 1, "width": 2 }, { "description": "Requests last 24h", "type": "STREAM_SEARCH_RESULT_COUNT", "cache_time": 10, "configuration": { "timerange": { "type": "relative", "range": 86400 }, "stream_id": "5445736fd4c6d7d480b5f4c2", "query": "*" }, "col": 1, "row": 1, "height": 1, "width": 1 }, { "description": "HTTP versions last 24h", "type": "QUICKVALUES", "cache_time": 300, "configuration": { "timerange": { "type": "relative", "range": 86400 }, "field": "http_version", "stream_id": "5445736fd4c6d7d480b5f4c2", "query": "*" }, "col": 1, "row": 4, "height": 3, "width": 1 }, { "description": "HTTP 5XXs last 24h", "type": "STREAM_SEARCH_RESULT_COUNT", "cache_time": 10, "configuration": { "timerange": { "type": "relative", "range": 86400 }, "stream_id": "547b2a77d4c6c10b4f1b941f", "query": "*" }, "col": 1, "row": 3, "height": 1, "width": 1 }, { "description": "HTTP 4XXs last 24h", "type": "STREAM_SEARCH_RESULT_COUNT", "cache_time": 10, "configuration": { "timerange": { "type": "relative", "range": 86400 }, "stream_id": "547b2ad4d4c6c10b4f1b9485", "query": "*" }, "col": 1, "row": 2, "height": 1, "width": 1 }, { "description": "HTTP 4XXs last 24h", "type": "SEARCH_RESULT_CHART", "cache_time": 10, "configuration": { "interval": "minute", "timerange": { "type": "relative", "range": 86400 }, "stream_id": "547b2ad4d4c6c10b4f1b9485", "query": "*" }, "col": 2, "row": 2, "height": 1, "width": 2 }, { "description": "HTTP 5XXs last 24h", "type": "SEARCH_RESULT_CHART", "cache_time": 10, "configuration": { "interval": "minute", "timerange": { "type": "relative", "range": 86400 }, "stream_id": "547b2a77d4c6c10b4f1b941f", "query": "*" }, "col": 2, "row": 3, "height": 1, "width": 2 }, { "description": "Map of requests", "type": "org.graylog.plugins.map.widget.strategy.MapWidgetStrategy", "cache_time": 10, "configuration": { "timerange": { "type": "relative", "range": 300 }, "field": "remote_addr_geolocation", "stream_id": "547b29b6d4c6c10b4f1b934d", "query": "" }, "col": 4, "row": 2, "height": 2, "width": 2 } ] } ], "grok_patterns": [], "lookup_tables": [], "lookup_caches": [], "lookup_data_adapters": [] }

В файле nginx.conf:

log_format graylog2_json escape=json '{ "timestamp": "$time_iso8601", ' '"remote_addr": "$remote_addr", ' '"remote_user": "$remote_user", ' '"body_bytes_sent": $body_bytes_sent, ' '"request_time": $request_time, ' '"status": $status, ' '"request": "$request", ' '"request_method": "$request_method", ' '"host": "$host",' '"upstream_cache_status": "$upstream_cache_status",' '"upstream_addr": "$upstream_addr",' '"http_x_forwarded_for": "$http_x_forwarded_for",' '"http_referrer": "$http_referer", ' '"http_user_agent": "$http_user_agent" }';

В секции server:

access_log syslog:server=2.9.0.1:514 graylog2_json; error_log syslog:server=2.9.0.1:515;

Комментарии пользователей