Add analytics (#118)
* ELK stack to analyze logs from client and server * Short-live container to configure kibana. Import kibana dashboards from export.json on startup. * Client and server logging is working throw python-logstash * Minor polishing of events and dashboards. * Default is discover, new visualizations. * Make comments more readable inside logger.js. * Added a path for backups.main
Nikita Manovich
7 years ago
committed by
GitHub
parent
e4638685cd
commit
c096c2a600
@ -0,0 +1,64 @@
|
|||||||
|
version: '2.3'
|
||||||
|
services:
|
||||||
|
cvat_elasticsearch:
|
||||||
|
container_name: cvat_elasticsearch
|
||||||
|
image: cvat_elasticsearch
|
||||||
|
networks:
|
||||||
|
default:
|
||||||
|
aliases:
|
||||||
|
- elasticsearch
|
||||||
|
build:
|
||||||
|
context: ./analytics/elasticsearch
|
||||||
|
args:
|
||||||
|
ELK_VERSION: 6.4.0
|
||||||
|
restart: always
|
||||||
|
|
||||||
|
cvat_kibana:
|
||||||
|
container_name: cvat_kibana
|
||||||
|
image: cvat_kibana
|
||||||
|
networks:
|
||||||
|
default:
|
||||||
|
aliases:
|
||||||
|
- kibana
|
||||||
|
build:
|
||||||
|
context: ./analytics/kibana
|
||||||
|
args:
|
||||||
|
ELK_VERSION: 6.4.0
|
||||||
|
ports:
|
||||||
|
- "5601:5601"
|
||||||
|
depends_on: ['cvat_elasticsearch']
|
||||||
|
restart: always
|
||||||
|
|
||||||
|
cvat_kibana_setup:
|
||||||
|
container_name: cvat_kibana_setup
|
||||||
|
image: cvat
|
||||||
|
volumes: ['./analytics/kibana:/home/django/kibana:ro']
|
||||||
|
depends_on: ['cvat']
|
||||||
|
working_dir: '/home/django'
|
||||||
|
entrypoint: ['bash', 'wait-for-it.sh', 'elasticsearch:9200', '-t', '0', '--',
|
||||||
|
'/bin/bash', 'wait-for-it.sh', 'kibana:5601', '-t', '0', '--',
|
||||||
|
'/usr/bin/python3', 'kibana/setup.py', 'kibana/export.json']
|
||||||
|
environment:
|
||||||
|
no_proxy: elasticsearch,kibana,${no_proxy}
|
||||||
|
|
||||||
|
cvat_logstash:
|
||||||
|
container_name: cvat_logstash
|
||||||
|
image: cvat_logstash
|
||||||
|
networks:
|
||||||
|
default:
|
||||||
|
aliases:
|
||||||
|
- logstash
|
||||||
|
build:
|
||||||
|
context: ./analytics/logstash
|
||||||
|
args:
|
||||||
|
ELK_VERSION: 6.4.0
|
||||||
|
http_proxy: ${http_proxy}
|
||||||
|
https_proxy: ${https_proxy}
|
||||||
|
depends_on: ['cvat_elasticsearch']
|
||||||
|
restart: always
|
||||||
|
|
||||||
|
cvat:
|
||||||
|
environment:
|
||||||
|
DJANGO_LOG_SERVER_HOST: "logstash"
|
||||||
|
DJANGO_LOG_SERVER_PORT: 5000
|
||||||
|
no_proxy: logstash,${no_proxy}
|
||||||
@ -0,0 +1,3 @@
|
|||||||
|
ARG ELK_VERSION
|
||||||
|
FROM docker.elastic.co/elasticsearch/elasticsearch-oss:${ELK_VERSION}
|
||||||
|
COPY --chown=elasticsearch:elasticsearch elasticsearch.yml /usr/share/elasticsearch/config/
|
||||||
@ -0,0 +1,3 @@
|
|||||||
|
http.host: 0.0.0.0
|
||||||
|
script.painless.regex.enabled: true
|
||||||
|
path.repo: ["/usr/share/elasticsearch/backup"]
|
||||||
@ -0,0 +1,5 @@
|
|||||||
|
ARG ELK_VERSION
|
||||||
|
FROM docker.elastic.co/kibana/kibana-oss:${ELK_VERSION}
|
||||||
|
COPY kibana.yml /usr/share/kibana/config/
|
||||||
|
|
||||||
|
|
||||||
@ -0,0 +1,198 @@
|
|||||||
|
[
|
||||||
|
{
|
||||||
|
"_id": "3ade53d0-c23e-11e8-8e1b-758ef07f6de8",
|
||||||
|
"_type": "visualization",
|
||||||
|
"_source": {
|
||||||
|
"title": "Timeline for exceptions",
|
||||||
|
"visState": "{\"title\":\"Timeline for exceptions\",\"type\":\"histogram\",\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{},\"type\":\"category\"}],\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"legendPosition\":\"right\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Exceptions\"},\"drawLinesBetweenPoints\":true,\"mode\":\"stacked\",\"show\":\"true\",\"showCircles\":true,\"type\":\"histogram\",\"valueAxis\":\"ValueAxis-1\"}],\"times\":[],\"type\":\"histogram\",\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"mode\":\"normal\",\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Exceptions\"},\"type\":\"value\"}]},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum_bucket\",\"schema\":\"metric\",\"params\":{\"customBucket\":{\"id\":\"1-bucket\",\"enabled\":true,\"type\":\"filters\",\"schema\":\"bucketAgg\",\"params\":{\"filters\":[{\"input\":{\"query\":\"event:\\\"Send exception\\\"\"},\"label\":\"\"}]}},\"customMetric\":{\"id\":\"1-metric\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metricAgg\",\"params\":{\"customLabel\":\"Exceptions\"}},\"customLabel\":\"Exceptions\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Time\"}}]}",
|
||||||
|
"uiStateJSON": "{}",
|
||||||
|
"description": "",
|
||||||
|
"version": 1,
|
||||||
|
"kibanaSavedObjectMeta": {
|
||||||
|
"searchSourceJSON": "{\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"_meta": {
|
||||||
|
"savedObjectVersion": 2
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"_id": "7e8996e0-c23d-11e8-8e1b-758ef07f6de8",
|
||||||
|
"_type": "dashboard",
|
||||||
|
"_source": {
|
||||||
|
"optionsJSON": "{\"darkTheme\":false,\"hidePanelTitles\":false,\"useMargins\":true}",
|
||||||
|
"timeRestore": false,
|
||||||
|
"description": "",
|
||||||
|
"hits": 0,
|
||||||
|
"kibanaSavedObjectMeta": {
|
||||||
|
"searchSourceJSON": "{\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[]}"
|
||||||
|
},
|
||||||
|
"panelsJSON": "[{\"embeddableConfig\":{},\"gridData\":{\"x\":0,\"y\":21,\"w\":48,\"h\":13,\"i\":\"1\"},\"id\":\"3ade53d0-c23e-11e8-8e1b-758ef07f6de8\",\"panelIndex\":\"1\",\"type\":\"visualization\",\"version\":\"6.4.0\"},{\"embeddableConfig\":{},\"gridData\":{\"x\":0,\"y\":34,\"w\":48,\"h\":27,\"i\":\"2\"},\"id\":\"9397f350-c23e-11e8-8e1b-758ef07f6de8\",\"panelIndex\":\"2\",\"type\":\"search\",\"version\":\"6.4.0\"},{\"embeddableConfig\":{},\"gridData\":{\"x\":0,\"y\":0,\"w\":24,\"h\":21,\"i\":\"3\"},\"id\":\"1ec6a660-c244-11e8-8e1b-758ef07f6de8\",\"panelIndex\":\"3\",\"type\":\"visualization\",\"version\":\"6.4.0\"},{\"embeddableConfig\":{},\"gridData\":{\"x\":24,\"y\":0,\"w\":24,\"h\":21,\"i\":\"4\"},\"id\":\"65918380-c244-11e8-8e1b-758ef07f6de8\",\"panelIndex\":\"4\",\"type\":\"visualization\",\"version\":\"6.4.0\"}]",
|
||||||
|
"title": "Monitoring",
|
||||||
|
"version": 1
|
||||||
|
},
|
||||||
|
"_meta": {
|
||||||
|
"savedObjectVersion": 2
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"_id": "31ac2d60-c25b-11e8-8e1b-758ef07f6de8",
|
||||||
|
"_type": "visualization",
|
||||||
|
"_source": {
|
||||||
|
"title": "List of users",
|
||||||
|
"visState": "{\"title\":\"List of users\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showMetricsAtAllLevels\":false,\"showPartialRows\":false,\"showTotal\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"userid.keyword\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"_key\",\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"User\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"sum_bucket\",\"schema\":\"metric\",\"params\":{\"customBucket\":{\"id\":\"3-bucket\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"bucketAgg\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},\"customMetric\":{\"id\":\"3-metric\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metricAgg\",\"params\":{}},\"customLabel\":\"Activity\"}},{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"working time\",\"customLabel\":\"Working Time (h)\"}}]}",
|
||||||
|
"uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
|
||||||
|
"description": "",
|
||||||
|
"version": 1,
|
||||||
|
"kibanaSavedObjectMeta": {
|
||||||
|
"searchSourceJSON": "{\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[]}"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"_meta": {
|
||||||
|
"savedObjectVersion": 2
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"_id": "1ec6a660-c244-11e8-8e1b-758ef07f6de8",
|
||||||
|
"_type": "visualization",
|
||||||
|
"_source": {
|
||||||
|
"title": "Duration of events",
|
||||||
|
"visState": "{\"title\":\"Duration of events\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showMetricsAtAllLevels\":false,\"showPartialRows\":false,\"showTotal\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"event.keyword\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"_key\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Action\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"avg\",\"schema\":\"metric\",\"params\":{\"field\":\"duration\",\"customLabel\":\"\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"min\",\"schema\":\"metric\",\"params\":{\"field\":\"duration\",\"customLabel\":\"\"}},{\"id\":\"5\",\"enabled\":true,\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"duration\"}}]}",
|
||||||
|
"uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
|
||||||
|
"description": "",
|
||||||
|
"version": 1,
|
||||||
|
"kibanaSavedObjectMeta": {
|
||||||
|
"searchSourceJSON": "{\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[{\"$state\":{\"store\":\"appState\"},\"exists\":{\"field\":\"duration\"},\"meta\":{\"alias\":null,\"disabled\":false,\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"key\":\"duration\",\"negate\":false,\"type\":\"exists\",\"value\":\"exists\"}}]}"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"_meta": {
|
||||||
|
"savedObjectVersion": 2
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"_id": "ec510550-c238-11e8-8e1b-758ef07f6de8",
|
||||||
|
"_type": "index-pattern",
|
||||||
|
"_source": {
|
||||||
|
"fields": "[{\"name\":\"@timestamp\",\"type\":\"date\",\"count\":2,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"@version\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"@version.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"application\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"application.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"box count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"duration\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"event\",\"type\":\"string\",\"count\":2,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"event.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"frame count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"object count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"points count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"polygon count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"polyline count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"task\",\"type\":\"string\",\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"task.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"timestamp\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"track count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"userid\",\"type\":\"string\",\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"userid.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"working time\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true}]",
|
||||||
|
"fieldFormatMap": "{\"duration\":{\"id\":\"duration\",\"params\":{\"inputFormat\":\"milliseconds\",\"outputFormat\":\"asSeconds\"}},\"working time\":{\"id\":\"duration\",\"params\":{\"inputFormat\":\"milliseconds\",\"outputFormat\":\"asHours\"}}}",
|
||||||
|
"title": "cvat*",
|
||||||
|
"timeFieldName": "@timestamp"
|
||||||
|
},
|
||||||
|
"_meta": {
|
||||||
|
"savedObjectVersion": 2
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"_id": "65918380-c244-11e8-8e1b-758ef07f6de8",
|
||||||
|
"_type": "visualization",
|
||||||
|
"_source": {
|
||||||
|
"title": "Number of events",
|
||||||
|
"visState": "{\"title\":\"Number of events\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showMetricsAtAllLevels\":false,\"showPartialRows\":false,\"showTotal\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"event.keyword\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Action\"}}]}",
|
||||||
|
"uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
|
||||||
|
"description": "",
|
||||||
|
"version": 1,
|
||||||
|
"kibanaSavedObjectMeta": {
|
||||||
|
"searchSourceJSON": "{\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[]}"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"_meta": {
|
||||||
|
"savedObjectVersion": 2
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"_id": "d92524b0-c25c-11e8-8e1b-758ef07f6de8",
|
||||||
|
"_type": "visualization",
|
||||||
|
"_source": {
|
||||||
|
"title": "Activity of users",
|
||||||
|
"visState": "{\"title\":\"Activity of users\",\"type\":\"metrics\",\"params\":{\"id\":\"61ca57f0-469d-11e7-af02-69e470af7417\",\"type\":\"timeseries\",\"series\":[{\"id\":\"61ca57f1-469d-11e7-af02-69e470af7417\",\"color\":\"#68BC00\",\"split_mode\":\"terms\",\"metrics\":[{\"id\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"type\":\"count\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"label\":\"User\",\"terms_field\":\"userid.keyword\",\"terms_size\":\"100\"}],\"time_field\":\"@timestamp\",\"index_pattern\":\"cvat*\",\"interval\":\"auto\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"axis_scale\":\"normal\",\"show_legend\":1,\"show_grid\":1},\"aggs\":[]}",
|
||||||
|
"uiStateJSON": "{}",
|
||||||
|
"description": "",
|
||||||
|
"version": 1,
|
||||||
|
"kibanaSavedObjectMeta": {
|
||||||
|
"searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"_meta": {
|
||||||
|
"savedObjectVersion": 2
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"_id": "9397f350-c23e-11e8-8e1b-758ef07f6de8",
|
||||||
|
"_type": "search",
|
||||||
|
"_source": {
|
||||||
|
"title": "Table with exceptions",
|
||||||
|
"description": "",
|
||||||
|
"hits": 0,
|
||||||
|
"columns": [
|
||||||
|
"task",
|
||||||
|
"type",
|
||||||
|
"userid",
|
||||||
|
"stack"
|
||||||
|
],
|
||||||
|
"sort": [
|
||||||
|
"@timestamp",
|
||||||
|
"desc"
|
||||||
|
],
|
||||||
|
"version": 1,
|
||||||
|
"kibanaSavedObjectMeta": {
|
||||||
|
"searchSourceJSON": "{\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"highlightAll\":true,\"version\":true,\"query\":{\"language\":\"lucene\",\"query\":\"event:\\\"Send exception\\\"\"},\"filter\":[]}"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"_meta": {
|
||||||
|
"savedObjectVersion": 2
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"_id": "b6339c70-c7d4-11e8-a035-258d2bd7d91f",
|
||||||
|
"_type": "visualization",
|
||||||
|
"_source": {
|
||||||
|
"title": "Working calendar",
|
||||||
|
"visState": "{\"title\":\"Working calendar\",\"type\":\"heatmap\",\"params\":{\"type\":\"heatmap\",\"addTooltip\":true,\"addLegend\":true,\"enableHover\":false,\"legendPosition\":\"right\",\"times\":[],\"colorsNumber\":4,\"colorSchema\":\"Green to Red\",\"setColorRange\":false,\"colorsRange\":[],\"invertColors\":true,\"percentageMode\":false,\"valueAxes\":[{\"show\":false,\"id\":\"ValueAxis-1\",\"type\":\"value\",\"scale\":{\"type\":\"linear\",\"defaultYExtents\":false},\"labels\":{\"show\":false,\"rotate\":0,\"overwriteColor\":false,\"color\":\"#555\"}}]},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"userid.keyword\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"_key\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Users\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"d\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Date\"}}]}",
|
||||||
|
"uiStateJSON": "{\"vis\":{\"defaultColors\":{\"0 - 8\":\"rgb(165,0,38)\",\"8 - 15\":\"rgb(249,142,82)\",\"15 - 23\":\"rgb(255,255,190)\",\"23 - 30\":\"rgb(135,203,103)\"}}}",
|
||||||
|
"description": "",
|
||||||
|
"version": 1,
|
||||||
|
"kibanaSavedObjectMeta": {
|
||||||
|
"searchSourceJSON": "{\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"_meta": {
|
||||||
|
"savedObjectVersion": 2
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"_id": "22250a40-c25d-11e8-8e1b-758ef07f6de8",
|
||||||
|
"_type": "dashboard",
|
||||||
|
"_source": {
|
||||||
|
"title": "Managment",
|
||||||
|
"hits": 0,
|
||||||
|
"description": "",
|
||||||
|
"panelsJSON": "[{\"embeddableConfig\":{},\"gridData\":{\"x\":0,\"y\":0,\"w\":24,\"h\":21,\"i\":\"1\"},\"id\":\"31ac2d60-c25b-11e8-8e1b-758ef07f6de8\",\"panelIndex\":\"1\",\"type\":\"visualization\",\"version\":\"6.4.0\"},{\"embeddableConfig\":{},\"gridData\":{\"x\":24,\"y\":0,\"w\":24,\"h\":21,\"i\":\"2\"},\"id\":\"543f6260-c25c-11e8-8e1b-758ef07f6de8\",\"panelIndex\":\"2\",\"type\":\"visualization\",\"version\":\"6.4.0\"},{\"embeddableConfig\":{},\"gridData\":{\"x\":24,\"y\":21,\"w\":24,\"h\":16,\"i\":\"3\"},\"id\":\"d92524b0-c25c-11e8-8e1b-758ef07f6de8\",\"panelIndex\":\"3\",\"type\":\"visualization\",\"version\":\"6.4.0\"},{\"gridData\":{\"x\":0,\"y\":21,\"w\":24,\"h\":16,\"i\":\"4\"},\"version\":\"6.4.0\",\"panelIndex\":\"4\",\"type\":\"visualization\",\"id\":\"b6339c70-c7d4-11e8-a035-258d2bd7d91f\",\"embeddableConfig\":{}}]",
|
||||||
|
"optionsJSON": "{\"darkTheme\":false,\"hidePanelTitles\":false,\"useMargins\":true}",
|
||||||
|
"version": 1,
|
||||||
|
"timeRestore": false,
|
||||||
|
"kibanaSavedObjectMeta": {
|
||||||
|
"searchSourceJSON": "{\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[]}"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"_meta": {
|
||||||
|
"savedObjectVersion": 2
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"_id": "543f6260-c25c-11e8-8e1b-758ef07f6de8",
|
||||||
|
"_type": "visualization",
|
||||||
|
"_source": {
|
||||||
|
"title": "Working day",
|
||||||
|
"visState": "{\"title\":\"Working day\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"min\",\"schema\":\"metric\",\"params\":{\"field\":\"@timestamp\",\"customLabel\":\"Start\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"split\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"d\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"_\",\"row\":true}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"userid.keyword\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"User\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"@timestamp\",\"customLabel\":\"End\"}}]}",
|
||||||
|
"uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
|
||||||
|
"description": "",
|
||||||
|
"version": 1,
|
||||||
|
"kibanaSavedObjectMeta": {
|
||||||
|
"searchSourceJSON": "{\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"_meta": {
|
||||||
|
"savedObjectVersion": 2
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
@ -0,0 +1,4 @@
|
|||||||
|
server.host: 0.0.0.0
|
||||||
|
elasticsearch.url: http://elasticsearch:9200
|
||||||
|
elasticsearch.requestHeadersWhitelist: [ cookie, authorization, x-forwarded-user ]
|
||||||
|
kibana.defaultAppId: "discover"
|
||||||
@ -0,0 +1,40 @@
|
|||||||
|
#/usr/bin/env python
|
||||||
|
|
||||||
|
import os
|
||||||
|
import argparse
|
||||||
|
import requests
|
||||||
|
import json
|
||||||
|
|
||||||
|
def import_resources(host, port, cfg_file):
|
||||||
|
with open(cfg_file, 'r') as f:
|
||||||
|
for saved_object in json.load(f):
|
||||||
|
_id = saved_object["_id"]
|
||||||
|
_type = saved_object["_type"]
|
||||||
|
_doc = saved_object["_source"]
|
||||||
|
import_saved_object(host, port, _type, _id, _doc)
|
||||||
|
|
||||||
|
def import_saved_object(host, port, _type, _id, data):
|
||||||
|
saved_objects_api = "http://{}:{}/api/saved_objects/{}/{}".format(
|
||||||
|
host, port, _type, _id)
|
||||||
|
request = requests.get(saved_objects_api)
|
||||||
|
if request.status_code == 404:
|
||||||
|
print("Creating {} as {}".format(_type, _id))
|
||||||
|
request = requests.post(saved_objects_api, json={"attributes": data},
|
||||||
|
headers={'kbn-xsrf': 'true'})
|
||||||
|
else:
|
||||||
|
print("Updating {} named {}".format(_type, _id))
|
||||||
|
request = requests.put(saved_objects_api, json={"attributes": data},
|
||||||
|
headers={'kbn-xsrf': 'true'})
|
||||||
|
request.raise_for_status()
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
parser = argparse.ArgumentParser(description='import Kibana 6.x resources',
|
||||||
|
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
|
||||||
|
parser.add_argument('export_file', metavar='FILE',
|
||||||
|
help='JSON export file with resources')
|
||||||
|
parser.add_argument('-p', '--port', metavar='PORT', default=5601, type=int,
|
||||||
|
help='port of Kibana instance')
|
||||||
|
parser.add_argument('-H', '--host', metavar='HOST', default='kibana',
|
||||||
|
help='host of Kibana instance')
|
||||||
|
args = parser.parse_args()
|
||||||
|
import_resources(args.host, args.port, args.export_file)
|
||||||
@ -0,0 +1,7 @@
|
|||||||
|
ARG ELK_VERSION
|
||||||
|
FROM docker.elastic.co/logstash/logstash-oss:${ELK_VERSION}
|
||||||
|
RUN logstash-plugin install logstash-input-http logstash-filter-aggregate \
|
||||||
|
logstash-filter-prune logstash-output-email
|
||||||
|
|
||||||
|
COPY logstash.conf /usr/share/logstash/pipeline/
|
||||||
|
EXPOSE 5000
|
||||||
@ -0,0 +1,85 @@
|
|||||||
|
input {
|
||||||
|
tcp {
|
||||||
|
port => 5000
|
||||||
|
codec => json
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
filter {
|
||||||
|
if [logger_name] =~ /cvat.client/ {
|
||||||
|
# 1. Decode the event from json in 'message' field
|
||||||
|
# 2. Remove unnecessary field from it
|
||||||
|
# 3. Type it as client
|
||||||
|
json {
|
||||||
|
source => "message"
|
||||||
|
}
|
||||||
|
|
||||||
|
date {
|
||||||
|
match => ["timestamp", "UNIX", "UNIX_MS"]
|
||||||
|
remove_field => "timestamp"
|
||||||
|
}
|
||||||
|
|
||||||
|
if [event] == "Send exception" {
|
||||||
|
aggregate {
|
||||||
|
task_id => "%{userid}_%{application}_%{message}_%{filename}_%{line}"
|
||||||
|
code => "
|
||||||
|
require 'time'
|
||||||
|
|
||||||
|
map['userid'] ||= event.get('userid');
|
||||||
|
map['application'] ||= event.get('application');
|
||||||
|
map['message'] ||= event.get('message');
|
||||||
|
map['filename'] ||= event.get('filename');
|
||||||
|
map['line'] ||= event.get('line');
|
||||||
|
map['task'] ||= event.get('task');
|
||||||
|
|
||||||
|
map['error_count'] ||= 0;
|
||||||
|
map['error_count'] += 1;
|
||||||
|
|
||||||
|
map['aggregated_message'] ||= '';
|
||||||
|
time = Time.strptime(event.get('timestamp').to_s,'%Q').localtime('+03:00')
|
||||||
|
map['aggregated_message'] += time.to_s + '\n' + event.get('stack') + '\n\n\n';"
|
||||||
|
|
||||||
|
timeout => 3600
|
||||||
|
timeout_tags => ['send_email_notification']
|
||||||
|
push_map_as_event_on_timeout => true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
prune {
|
||||||
|
blacklist_names => ["level", "host", "logger_name", "message", "path",
|
||||||
|
"port", "stack_info"]
|
||||||
|
}
|
||||||
|
|
||||||
|
mutate {
|
||||||
|
replace => { "type" => "client" }
|
||||||
|
}
|
||||||
|
} else if [logger_name] =~ /cvat.server/ {
|
||||||
|
# 1. Remove unnecessary field from it
|
||||||
|
# 2. Type it as server
|
||||||
|
prune {
|
||||||
|
blacklist_names => ["host", "port"]
|
||||||
|
}
|
||||||
|
|
||||||
|
mutate {
|
||||||
|
replace => { "type" => "server" }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
output {
|
||||||
|
stdout {
|
||||||
|
codec => rubydebug
|
||||||
|
}
|
||||||
|
|
||||||
|
if [type] == "client" {
|
||||||
|
elasticsearch {
|
||||||
|
hosts => ["elasticsearch:9200"]
|
||||||
|
index => "cvat.client"
|
||||||
|
}
|
||||||
|
} else if [type] == "server" {
|
||||||
|
elasticsearch {
|
||||||
|
hosts => ["elasticsearch:9200"]
|
||||||
|
index => "cvat.server"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -1,5 +0,0 @@
|
|||||||
|
|
||||||
# Copyright (C) 2018 Intel Corporation
|
|
||||||
#
|
|
||||||
# SPDX-License-Identifier: MIT
|
|
||||||
|
|
||||||
@ -1,9 +0,0 @@
|
|||||||
|
|
||||||
# Copyright (C) 2018 Intel Corporation
|
|
||||||
#
|
|
||||||
# SPDX-License-Identifier: MIT
|
|
||||||
|
|
||||||
from django.contrib import admin
|
|
||||||
|
|
||||||
# Register your models here.
|
|
||||||
|
|
||||||
@ -1,11 +0,0 @@
|
|||||||
|
|
||||||
# Copyright (C) 2018 Intel Corporation
|
|
||||||
#
|
|
||||||
# SPDX-License-Identifier: MIT
|
|
||||||
|
|
||||||
from django.apps import AppConfig
|
|
||||||
|
|
||||||
|
|
||||||
class LogProxyConfig(AppConfig):
|
|
||||||
name = 'log_proxy'
|
|
||||||
|
|
||||||
@ -1,5 +0,0 @@
|
|||||||
|
|
||||||
# Copyright (C) 2018 Intel Corporation
|
|
||||||
#
|
|
||||||
# SPDX-License-Identifier: MIT
|
|
||||||
|
|
||||||
@ -1,9 +0,0 @@
|
|||||||
|
|
||||||
# Copyright (C) 2018 Intel Corporation
|
|
||||||
#
|
|
||||||
# SPDX-License-Identifier: MIT
|
|
||||||
|
|
||||||
from django.db import models
|
|
||||||
|
|
||||||
# Create your models here.
|
|
||||||
|
|
||||||
@ -1,91 +0,0 @@
|
|||||||
|
|
||||||
# Copyright (C) 2018 Intel Corporation
|
|
||||||
#
|
|
||||||
# SPDX-License-Identifier: MIT
|
|
||||||
|
|
||||||
from django.conf import settings
|
|
||||||
import os
|
|
||||||
import logging
|
|
||||||
import requests
|
|
||||||
import json
|
|
||||||
from urllib.parse import urlparse
|
|
||||||
from enum import Enum
|
|
||||||
from cvat.apps.engine.models import Job, Task
|
|
||||||
|
|
||||||
from requests.adapters import HTTPAdapter
|
|
||||||
from urllib3.util.retry import Retry
|
|
||||||
|
|
||||||
class ClientLoggerStorage:
|
|
||||||
def __init__(self):
|
|
||||||
self._storage = dict()
|
|
||||||
self._formatter = logging.Formatter('%(message)s')
|
|
||||||
|
|
||||||
def __getitem__(self, tid):
|
|
||||||
if tid not in self._storage:
|
|
||||||
self._storage[tid] = self._create_client_logger(tid)
|
|
||||||
return self._storage[tid]
|
|
||||||
|
|
||||||
def _create_client_logger(self, tid):
|
|
||||||
task = self._get_task(tid)
|
|
||||||
logger = logging.getLogger(name='client_annotation_logger_{}'.format(tid))
|
|
||||||
logger.setLevel(logging.INFO)
|
|
||||||
handler = logging.FileHandler(filename=task.get_client_log_path())
|
|
||||||
handler.setFormatter(self._formatter)
|
|
||||||
logger.addHandler(handler)
|
|
||||||
return logger
|
|
||||||
|
|
||||||
def _get_task(self, tid):
|
|
||||||
try:
|
|
||||||
return Task.objects.get(pk=tid)
|
|
||||||
except Exception:
|
|
||||||
raise Exception('Key must be task indentificator')
|
|
||||||
|
|
||||||
class ClientLogProxy():
|
|
||||||
class _HandlerType(Enum):
|
|
||||||
FILE = 1
|
|
||||||
HTTP = 2
|
|
||||||
|
|
||||||
def __init__(self):
|
|
||||||
self._client_logger = ClientLoggerStorage()
|
|
||||||
def file_log_handler(tid, messages):
|
|
||||||
for event in messages:
|
|
||||||
self._client_logger[tid].info(json.dumps(event))
|
|
||||||
|
|
||||||
self._handlers = {self._HandlerType.FILE: file_log_handler}
|
|
||||||
|
|
||||||
log_server_url = os.environ.get('DJANGO_LOG_SERVER_URL')
|
|
||||||
|
|
||||||
def create_retry_session(retries=3, session=None, backoff_factor=0.3):
|
|
||||||
session = session or requests.Session()
|
|
||||||
retry = Retry(total=retries, backoff_factor=backoff_factor)
|
|
||||||
adapter = HTTPAdapter(max_retries=retry)
|
|
||||||
session.mount('http://', adapter)
|
|
||||||
session.mount('https://', adapter)
|
|
||||||
return session
|
|
||||||
|
|
||||||
if log_server_url:
|
|
||||||
parse_result = urlparse(log_server_url)
|
|
||||||
|
|
||||||
if parse_result.scheme and 'http' not in parse_result.scheme:
|
|
||||||
raise Exception('unsuported annotation log destination')
|
|
||||||
|
|
||||||
def http_log_handler(taskID, messages):
|
|
||||||
r = create_retry_session().post(url=log_server_url, json=messages, verify=False)
|
|
||||||
r.raise_for_status()
|
|
||||||
|
|
||||||
self._handlers[self._HandlerType.HTTP] = http_log_handler
|
|
||||||
|
|
||||||
def push_logs(self, jid, logs):
|
|
||||||
taskID = self._get_task_id(jid)
|
|
||||||
|
|
||||||
for handler in self._handlers.values():
|
|
||||||
handler(taskID, logs)
|
|
||||||
|
|
||||||
def _get_task_id(self, jid):
|
|
||||||
try:
|
|
||||||
job = Job.objects.select_related("segment__task").get(id=jid)
|
|
||||||
return job.segment.task.id
|
|
||||||
except:
|
|
||||||
raise Exception('Key must be job indentificator')
|
|
||||||
|
|
||||||
client_log_proxy = ClientLogProxy()
|
|
||||||
@ -1,9 +0,0 @@
|
|||||||
|
|
||||||
# Copyright (C) 2018 Intel Corporation
|
|
||||||
#
|
|
||||||
# SPDX-License-Identifier: MIT
|
|
||||||
|
|
||||||
from django.test import TestCase
|
|
||||||
|
|
||||||
# Create your tests here.
|
|
||||||
|
|
||||||
@ -1,12 +0,0 @@
|
|||||||
|
|
||||||
# Copyright (C) 2018 Intel Corporation
|
|
||||||
#
|
|
||||||
# SPDX-License-Identifier: MIT
|
|
||||||
|
|
||||||
from django.urls import path
|
|
||||||
from . import views
|
|
||||||
|
|
||||||
urlpatterns = [
|
|
||||||
path('exception/<int:jid>', views.exception_receiver),
|
|
||||||
]
|
|
||||||
|
|
||||||
@ -1,25 +0,0 @@
|
|||||||
|
|
||||||
# Copyright (C) 2018 Intel Corporation
|
|
||||||
#
|
|
||||||
# SPDX-License-Identifier: MIT
|
|
||||||
|
|
||||||
from django.http import HttpResponse, HttpResponseBadRequest
|
|
||||||
from django.contrib.auth.decorators import permission_required
|
|
||||||
from .proxy_logger import client_log_proxy
|
|
||||||
from cvat.apps.authentication.decorators import login_required
|
|
||||||
|
|
||||||
|
|
||||||
import json
|
|
||||||
|
|
||||||
# Create your views here.
|
|
||||||
@login_required()
|
|
||||||
@permission_required('engine.view_task', raise_exception=True)
|
|
||||||
def exception_receiver(request, jid):
|
|
||||||
data = json.loads(request.body.decode('utf-8'))
|
|
||||||
try:
|
|
||||||
if 'exceptions' in data:
|
|
||||||
client_log_proxy.push_logs(jid, data['exceptions'])
|
|
||||||
except Exception as e:
|
|
||||||
return HttpResponseBadRequest(str(e))
|
|
||||||
|
|
||||||
return HttpResponse()
|
|
||||||
Loading…
Reference in New Issue