Add analytics (#118)
* ELK stack to analyze logs from client and server * Short-live container to configure kibana. Import kibana dashboards from export.json on startup. * Client and server logging is working throw python-logstash * Minor polishing of events and dashboards. * Default is discover, new visualizations. * Make comments more readable inside logger.js. * Added a path for backups.main
Nikita Manovich
7 years ago
committed by
GitHub
parent
e4638685cd
commit
c096c2a600
@ -0,0 +1,64 @@
|
||||
version: '2.3'
|
||||
services:
|
||||
cvat_elasticsearch:
|
||||
container_name: cvat_elasticsearch
|
||||
image: cvat_elasticsearch
|
||||
networks:
|
||||
default:
|
||||
aliases:
|
||||
- elasticsearch
|
||||
build:
|
||||
context: ./analytics/elasticsearch
|
||||
args:
|
||||
ELK_VERSION: 6.4.0
|
||||
restart: always
|
||||
|
||||
cvat_kibana:
|
||||
container_name: cvat_kibana
|
||||
image: cvat_kibana
|
||||
networks:
|
||||
default:
|
||||
aliases:
|
||||
- kibana
|
||||
build:
|
||||
context: ./analytics/kibana
|
||||
args:
|
||||
ELK_VERSION: 6.4.0
|
||||
ports:
|
||||
- "5601:5601"
|
||||
depends_on: ['cvat_elasticsearch']
|
||||
restart: always
|
||||
|
||||
cvat_kibana_setup:
|
||||
container_name: cvat_kibana_setup
|
||||
image: cvat
|
||||
volumes: ['./analytics/kibana:/home/django/kibana:ro']
|
||||
depends_on: ['cvat']
|
||||
working_dir: '/home/django'
|
||||
entrypoint: ['bash', 'wait-for-it.sh', 'elasticsearch:9200', '-t', '0', '--',
|
||||
'/bin/bash', 'wait-for-it.sh', 'kibana:5601', '-t', '0', '--',
|
||||
'/usr/bin/python3', 'kibana/setup.py', 'kibana/export.json']
|
||||
environment:
|
||||
no_proxy: elasticsearch,kibana,${no_proxy}
|
||||
|
||||
cvat_logstash:
|
||||
container_name: cvat_logstash
|
||||
image: cvat_logstash
|
||||
networks:
|
||||
default:
|
||||
aliases:
|
||||
- logstash
|
||||
build:
|
||||
context: ./analytics/logstash
|
||||
args:
|
||||
ELK_VERSION: 6.4.0
|
||||
http_proxy: ${http_proxy}
|
||||
https_proxy: ${https_proxy}
|
||||
depends_on: ['cvat_elasticsearch']
|
||||
restart: always
|
||||
|
||||
cvat:
|
||||
environment:
|
||||
DJANGO_LOG_SERVER_HOST: "logstash"
|
||||
DJANGO_LOG_SERVER_PORT: 5000
|
||||
no_proxy: logstash,${no_proxy}
|
||||
@ -0,0 +1,3 @@
|
||||
ARG ELK_VERSION
|
||||
FROM docker.elastic.co/elasticsearch/elasticsearch-oss:${ELK_VERSION}
|
||||
COPY --chown=elasticsearch:elasticsearch elasticsearch.yml /usr/share/elasticsearch/config/
|
||||
@ -0,0 +1,3 @@
|
||||
http.host: 0.0.0.0
|
||||
script.painless.regex.enabled: true
|
||||
path.repo: ["/usr/share/elasticsearch/backup"]
|
||||
@ -0,0 +1,5 @@
|
||||
ARG ELK_VERSION
|
||||
FROM docker.elastic.co/kibana/kibana-oss:${ELK_VERSION}
|
||||
COPY kibana.yml /usr/share/kibana/config/
|
||||
|
||||
|
||||
@ -0,0 +1,198 @@
|
||||
[
|
||||
{
|
||||
"_id": "3ade53d0-c23e-11e8-8e1b-758ef07f6de8",
|
||||
"_type": "visualization",
|
||||
"_source": {
|
||||
"title": "Timeline for exceptions",
|
||||
"visState": "{\"title\":\"Timeline for exceptions\",\"type\":\"histogram\",\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{},\"type\":\"category\"}],\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"legendPosition\":\"right\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Exceptions\"},\"drawLinesBetweenPoints\":true,\"mode\":\"stacked\",\"show\":\"true\",\"showCircles\":true,\"type\":\"histogram\",\"valueAxis\":\"ValueAxis-1\"}],\"times\":[],\"type\":\"histogram\",\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"mode\":\"normal\",\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Exceptions\"},\"type\":\"value\"}]},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum_bucket\",\"schema\":\"metric\",\"params\":{\"customBucket\":{\"id\":\"1-bucket\",\"enabled\":true,\"type\":\"filters\",\"schema\":\"bucketAgg\",\"params\":{\"filters\":[{\"input\":{\"query\":\"event:\\\"Send exception\\\"\"},\"label\":\"\"}]}},\"customMetric\":{\"id\":\"1-metric\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metricAgg\",\"params\":{\"customLabel\":\"Exceptions\"}},\"customLabel\":\"Exceptions\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Time\"}}]}",
|
||||
"uiStateJSON": "{}",
|
||||
"description": "",
|
||||
"version": 1,
|
||||
"kibanaSavedObjectMeta": {
|
||||
"searchSourceJSON": "{\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}"
|
||||
}
|
||||
},
|
||||
"_meta": {
|
||||
"savedObjectVersion": 2
|
||||
}
|
||||
},
|
||||
{
|
||||
"_id": "7e8996e0-c23d-11e8-8e1b-758ef07f6de8",
|
||||
"_type": "dashboard",
|
||||
"_source": {
|
||||
"optionsJSON": "{\"darkTheme\":false,\"hidePanelTitles\":false,\"useMargins\":true}",
|
||||
"timeRestore": false,
|
||||
"description": "",
|
||||
"hits": 0,
|
||||
"kibanaSavedObjectMeta": {
|
||||
"searchSourceJSON": "{\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[]}"
|
||||
},
|
||||
"panelsJSON": "[{\"embeddableConfig\":{},\"gridData\":{\"x\":0,\"y\":21,\"w\":48,\"h\":13,\"i\":\"1\"},\"id\":\"3ade53d0-c23e-11e8-8e1b-758ef07f6de8\",\"panelIndex\":\"1\",\"type\":\"visualization\",\"version\":\"6.4.0\"},{\"embeddableConfig\":{},\"gridData\":{\"x\":0,\"y\":34,\"w\":48,\"h\":27,\"i\":\"2\"},\"id\":\"9397f350-c23e-11e8-8e1b-758ef07f6de8\",\"panelIndex\":\"2\",\"type\":\"search\",\"version\":\"6.4.0\"},{\"embeddableConfig\":{},\"gridData\":{\"x\":0,\"y\":0,\"w\":24,\"h\":21,\"i\":\"3\"},\"id\":\"1ec6a660-c244-11e8-8e1b-758ef07f6de8\",\"panelIndex\":\"3\",\"type\":\"visualization\",\"version\":\"6.4.0\"},{\"embeddableConfig\":{},\"gridData\":{\"x\":24,\"y\":0,\"w\":24,\"h\":21,\"i\":\"4\"},\"id\":\"65918380-c244-11e8-8e1b-758ef07f6de8\",\"panelIndex\":\"4\",\"type\":\"visualization\",\"version\":\"6.4.0\"}]",
|
||||
"title": "Monitoring",
|
||||
"version": 1
|
||||
},
|
||||
"_meta": {
|
||||
"savedObjectVersion": 2
|
||||
}
|
||||
},
|
||||
{
|
||||
"_id": "31ac2d60-c25b-11e8-8e1b-758ef07f6de8",
|
||||
"_type": "visualization",
|
||||
"_source": {
|
||||
"title": "List of users",
|
||||
"visState": "{\"title\":\"List of users\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showMetricsAtAllLevels\":false,\"showPartialRows\":false,\"showTotal\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"userid.keyword\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"_key\",\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"User\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"sum_bucket\",\"schema\":\"metric\",\"params\":{\"customBucket\":{\"id\":\"3-bucket\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"bucketAgg\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},\"customMetric\":{\"id\":\"3-metric\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metricAgg\",\"params\":{}},\"customLabel\":\"Activity\"}},{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"working time\",\"customLabel\":\"Working Time (h)\"}}]}",
|
||||
"uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
|
||||
"description": "",
|
||||
"version": 1,
|
||||
"kibanaSavedObjectMeta": {
|
||||
"searchSourceJSON": "{\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[]}"
|
||||
}
|
||||
},
|
||||
"_meta": {
|
||||
"savedObjectVersion": 2
|
||||
}
|
||||
},
|
||||
{
|
||||
"_id": "1ec6a660-c244-11e8-8e1b-758ef07f6de8",
|
||||
"_type": "visualization",
|
||||
"_source": {
|
||||
"title": "Duration of events",
|
||||
"visState": "{\"title\":\"Duration of events\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showMetricsAtAllLevels\":false,\"showPartialRows\":false,\"showTotal\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"event.keyword\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"_key\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Action\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"avg\",\"schema\":\"metric\",\"params\":{\"field\":\"duration\",\"customLabel\":\"\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"min\",\"schema\":\"metric\",\"params\":{\"field\":\"duration\",\"customLabel\":\"\"}},{\"id\":\"5\",\"enabled\":true,\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"duration\"}}]}",
|
||||
"uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
|
||||
"description": "",
|
||||
"version": 1,
|
||||
"kibanaSavedObjectMeta": {
|
||||
"searchSourceJSON": "{\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[{\"$state\":{\"store\":\"appState\"},\"exists\":{\"field\":\"duration\"},\"meta\":{\"alias\":null,\"disabled\":false,\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"key\":\"duration\",\"negate\":false,\"type\":\"exists\",\"value\":\"exists\"}}]}"
|
||||
}
|
||||
},
|
||||
"_meta": {
|
||||
"savedObjectVersion": 2
|
||||
}
|
||||
},
|
||||
{
|
||||
"_id": "ec510550-c238-11e8-8e1b-758ef07f6de8",
|
||||
"_type": "index-pattern",
|
||||
"_source": {
|
||||
"fields": "[{\"name\":\"@timestamp\",\"type\":\"date\",\"count\":2,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"@version\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"@version.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"application\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"application.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"box count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"duration\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"event\",\"type\":\"string\",\"count\":2,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"event.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"frame count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"object count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"points count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"polygon count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"polyline count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"task\",\"type\":\"string\",\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"task.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"timestamp\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"track count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"userid\",\"type\":\"string\",\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"userid.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"working time\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true}]",
|
||||
"fieldFormatMap": "{\"duration\":{\"id\":\"duration\",\"params\":{\"inputFormat\":\"milliseconds\",\"outputFormat\":\"asSeconds\"}},\"working time\":{\"id\":\"duration\",\"params\":{\"inputFormat\":\"milliseconds\",\"outputFormat\":\"asHours\"}}}",
|
||||
"title": "cvat*",
|
||||
"timeFieldName": "@timestamp"
|
||||
},
|
||||
"_meta": {
|
||||
"savedObjectVersion": 2
|
||||
}
|
||||
},
|
||||
{
|
||||
"_id": "65918380-c244-11e8-8e1b-758ef07f6de8",
|
||||
"_type": "visualization",
|
||||
"_source": {
|
||||
"title": "Number of events",
|
||||
"visState": "{\"title\":\"Number of events\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showMetricsAtAllLevels\":false,\"showPartialRows\":false,\"showTotal\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"event.keyword\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Action\"}}]}",
|
||||
"uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
|
||||
"description": "",
|
||||
"version": 1,
|
||||
"kibanaSavedObjectMeta": {
|
||||
"searchSourceJSON": "{\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[]}"
|
||||
}
|
||||
},
|
||||
"_meta": {
|
||||
"savedObjectVersion": 2
|
||||
}
|
||||
},
|
||||
{
|
||||
"_id": "d92524b0-c25c-11e8-8e1b-758ef07f6de8",
|
||||
"_type": "visualization",
|
||||
"_source": {
|
||||
"title": "Activity of users",
|
||||
"visState": "{\"title\":\"Activity of users\",\"type\":\"metrics\",\"params\":{\"id\":\"61ca57f0-469d-11e7-af02-69e470af7417\",\"type\":\"timeseries\",\"series\":[{\"id\":\"61ca57f1-469d-11e7-af02-69e470af7417\",\"color\":\"#68BC00\",\"split_mode\":\"terms\",\"metrics\":[{\"id\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"type\":\"count\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"label\":\"User\",\"terms_field\":\"userid.keyword\",\"terms_size\":\"100\"}],\"time_field\":\"@timestamp\",\"index_pattern\":\"cvat*\",\"interval\":\"auto\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"axis_scale\":\"normal\",\"show_legend\":1,\"show_grid\":1},\"aggs\":[]}",
|
||||
"uiStateJSON": "{}",
|
||||
"description": "",
|
||||
"version": 1,
|
||||
"kibanaSavedObjectMeta": {
|
||||
"searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}"
|
||||
}
|
||||
},
|
||||
"_meta": {
|
||||
"savedObjectVersion": 2
|
||||
}
|
||||
},
|
||||
{
|
||||
"_id": "9397f350-c23e-11e8-8e1b-758ef07f6de8",
|
||||
"_type": "search",
|
||||
"_source": {
|
||||
"title": "Table with exceptions",
|
||||
"description": "",
|
||||
"hits": 0,
|
||||
"columns": [
|
||||
"task",
|
||||
"type",
|
||||
"userid",
|
||||
"stack"
|
||||
],
|
||||
"sort": [
|
||||
"@timestamp",
|
||||
"desc"
|
||||
],
|
||||
"version": 1,
|
||||
"kibanaSavedObjectMeta": {
|
||||
"searchSourceJSON": "{\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"highlightAll\":true,\"version\":true,\"query\":{\"language\":\"lucene\",\"query\":\"event:\\\"Send exception\\\"\"},\"filter\":[]}"
|
||||
}
|
||||
},
|
||||
"_meta": {
|
||||
"savedObjectVersion": 2
|
||||
}
|
||||
},
|
||||
{
|
||||
"_id": "b6339c70-c7d4-11e8-a035-258d2bd7d91f",
|
||||
"_type": "visualization",
|
||||
"_source": {
|
||||
"title": "Working calendar",
|
||||
"visState": "{\"title\":\"Working calendar\",\"type\":\"heatmap\",\"params\":{\"type\":\"heatmap\",\"addTooltip\":true,\"addLegend\":true,\"enableHover\":false,\"legendPosition\":\"right\",\"times\":[],\"colorsNumber\":4,\"colorSchema\":\"Green to Red\",\"setColorRange\":false,\"colorsRange\":[],\"invertColors\":true,\"percentageMode\":false,\"valueAxes\":[{\"show\":false,\"id\":\"ValueAxis-1\",\"type\":\"value\",\"scale\":{\"type\":\"linear\",\"defaultYExtents\":false},\"labels\":{\"show\":false,\"rotate\":0,\"overwriteColor\":false,\"color\":\"#555\"}}]},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"userid.keyword\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"_key\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Users\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"d\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Date\"}}]}",
|
||||
"uiStateJSON": "{\"vis\":{\"defaultColors\":{\"0 - 8\":\"rgb(165,0,38)\",\"8 - 15\":\"rgb(249,142,82)\",\"15 - 23\":\"rgb(255,255,190)\",\"23 - 30\":\"rgb(135,203,103)\"}}}",
|
||||
"description": "",
|
||||
"version": 1,
|
||||
"kibanaSavedObjectMeta": {
|
||||
"searchSourceJSON": "{\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}"
|
||||
}
|
||||
},
|
||||
"_meta": {
|
||||
"savedObjectVersion": 2
|
||||
}
|
||||
},
|
||||
{
|
||||
"_id": "22250a40-c25d-11e8-8e1b-758ef07f6de8",
|
||||
"_type": "dashboard",
|
||||
"_source": {
|
||||
"title": "Managment",
|
||||
"hits": 0,
|
||||
"description": "",
|
||||
"panelsJSON": "[{\"embeddableConfig\":{},\"gridData\":{\"x\":0,\"y\":0,\"w\":24,\"h\":21,\"i\":\"1\"},\"id\":\"31ac2d60-c25b-11e8-8e1b-758ef07f6de8\",\"panelIndex\":\"1\",\"type\":\"visualization\",\"version\":\"6.4.0\"},{\"embeddableConfig\":{},\"gridData\":{\"x\":24,\"y\":0,\"w\":24,\"h\":21,\"i\":\"2\"},\"id\":\"543f6260-c25c-11e8-8e1b-758ef07f6de8\",\"panelIndex\":\"2\",\"type\":\"visualization\",\"version\":\"6.4.0\"},{\"embeddableConfig\":{},\"gridData\":{\"x\":24,\"y\":21,\"w\":24,\"h\":16,\"i\":\"3\"},\"id\":\"d92524b0-c25c-11e8-8e1b-758ef07f6de8\",\"panelIndex\":\"3\",\"type\":\"visualization\",\"version\":\"6.4.0\"},{\"gridData\":{\"x\":0,\"y\":21,\"w\":24,\"h\":16,\"i\":\"4\"},\"version\":\"6.4.0\",\"panelIndex\":\"4\",\"type\":\"visualization\",\"id\":\"b6339c70-c7d4-11e8-a035-258d2bd7d91f\",\"embeddableConfig\":{}}]",
|
||||
"optionsJSON": "{\"darkTheme\":false,\"hidePanelTitles\":false,\"useMargins\":true}",
|
||||
"version": 1,
|
||||
"timeRestore": false,
|
||||
"kibanaSavedObjectMeta": {
|
||||
"searchSourceJSON": "{\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[]}"
|
||||
}
|
||||
},
|
||||
"_meta": {
|
||||
"savedObjectVersion": 2
|
||||
}
|
||||
},
|
||||
{
|
||||
"_id": "543f6260-c25c-11e8-8e1b-758ef07f6de8",
|
||||
"_type": "visualization",
|
||||
"_source": {
|
||||
"title": "Working day",
|
||||
"visState": "{\"title\":\"Working day\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"min\",\"schema\":\"metric\",\"params\":{\"field\":\"@timestamp\",\"customLabel\":\"Start\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"split\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"d\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"_\",\"row\":true}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"userid.keyword\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"User\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"@timestamp\",\"customLabel\":\"End\"}}]}",
|
||||
"uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
|
||||
"description": "",
|
||||
"version": 1,
|
||||
"kibanaSavedObjectMeta": {
|
||||
"searchSourceJSON": "{\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}"
|
||||
}
|
||||
},
|
||||
"_meta": {
|
||||
"savedObjectVersion": 2
|
||||
}
|
||||
}
|
||||
]
|
||||
@ -0,0 +1,4 @@
|
||||
server.host: 0.0.0.0
|
||||
elasticsearch.url: http://elasticsearch:9200
|
||||
elasticsearch.requestHeadersWhitelist: [ cookie, authorization, x-forwarded-user ]
|
||||
kibana.defaultAppId: "discover"
|
||||
@ -0,0 +1,40 @@
|
||||
#/usr/bin/env python
|
||||
|
||||
import os
|
||||
import argparse
|
||||
import requests
|
||||
import json
|
||||
|
||||
def import_resources(host, port, cfg_file):
|
||||
with open(cfg_file, 'r') as f:
|
||||
for saved_object in json.load(f):
|
||||
_id = saved_object["_id"]
|
||||
_type = saved_object["_type"]
|
||||
_doc = saved_object["_source"]
|
||||
import_saved_object(host, port, _type, _id, _doc)
|
||||
|
||||
def import_saved_object(host, port, _type, _id, data):
|
||||
saved_objects_api = "http://{}:{}/api/saved_objects/{}/{}".format(
|
||||
host, port, _type, _id)
|
||||
request = requests.get(saved_objects_api)
|
||||
if request.status_code == 404:
|
||||
print("Creating {} as {}".format(_type, _id))
|
||||
request = requests.post(saved_objects_api, json={"attributes": data},
|
||||
headers={'kbn-xsrf': 'true'})
|
||||
else:
|
||||
print("Updating {} named {}".format(_type, _id))
|
||||
request = requests.put(saved_objects_api, json={"attributes": data},
|
||||
headers={'kbn-xsrf': 'true'})
|
||||
request.raise_for_status()
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser(description='import Kibana 6.x resources',
|
||||
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
|
||||
parser.add_argument('export_file', metavar='FILE',
|
||||
help='JSON export file with resources')
|
||||
parser.add_argument('-p', '--port', metavar='PORT', default=5601, type=int,
|
||||
help='port of Kibana instance')
|
||||
parser.add_argument('-H', '--host', metavar='HOST', default='kibana',
|
||||
help='host of Kibana instance')
|
||||
args = parser.parse_args()
|
||||
import_resources(args.host, args.port, args.export_file)
|
||||
@ -0,0 +1,7 @@
|
||||
ARG ELK_VERSION
|
||||
FROM docker.elastic.co/logstash/logstash-oss:${ELK_VERSION}
|
||||
RUN logstash-plugin install logstash-input-http logstash-filter-aggregate \
|
||||
logstash-filter-prune logstash-output-email
|
||||
|
||||
COPY logstash.conf /usr/share/logstash/pipeline/
|
||||
EXPOSE 5000
|
||||
@ -0,0 +1,85 @@
|
||||
input {
|
||||
tcp {
|
||||
port => 5000
|
||||
codec => json
|
||||
}
|
||||
}
|
||||
|
||||
filter {
|
||||
if [logger_name] =~ /cvat.client/ {
|
||||
# 1. Decode the event from json in 'message' field
|
||||
# 2. Remove unnecessary field from it
|
||||
# 3. Type it as client
|
||||
json {
|
||||
source => "message"
|
||||
}
|
||||
|
||||
date {
|
||||
match => ["timestamp", "UNIX", "UNIX_MS"]
|
||||
remove_field => "timestamp"
|
||||
}
|
||||
|
||||
if [event] == "Send exception" {
|
||||
aggregate {
|
||||
task_id => "%{userid}_%{application}_%{message}_%{filename}_%{line}"
|
||||
code => "
|
||||
require 'time'
|
||||
|
||||
map['userid'] ||= event.get('userid');
|
||||
map['application'] ||= event.get('application');
|
||||
map['message'] ||= event.get('message');
|
||||
map['filename'] ||= event.get('filename');
|
||||
map['line'] ||= event.get('line');
|
||||
map['task'] ||= event.get('task');
|
||||
|
||||
map['error_count'] ||= 0;
|
||||
map['error_count'] += 1;
|
||||
|
||||
map['aggregated_message'] ||= '';
|
||||
time = Time.strptime(event.get('timestamp').to_s,'%Q').localtime('+03:00')
|
||||
map['aggregated_message'] += time.to_s + '\n' + event.get('stack') + '\n\n\n';"
|
||||
|
||||
timeout => 3600
|
||||
timeout_tags => ['send_email_notification']
|
||||
push_map_as_event_on_timeout => true
|
||||
}
|
||||
}
|
||||
|
||||
prune {
|
||||
blacklist_names => ["level", "host", "logger_name", "message", "path",
|
||||
"port", "stack_info"]
|
||||
}
|
||||
|
||||
mutate {
|
||||
replace => { "type" => "client" }
|
||||
}
|
||||
} else if [logger_name] =~ /cvat.server/ {
|
||||
# 1. Remove unnecessary field from it
|
||||
# 2. Type it as server
|
||||
prune {
|
||||
blacklist_names => ["host", "port"]
|
||||
}
|
||||
|
||||
mutate {
|
||||
replace => { "type" => "server" }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
output {
|
||||
stdout {
|
||||
codec => rubydebug
|
||||
}
|
||||
|
||||
if [type] == "client" {
|
||||
elasticsearch {
|
||||
hosts => ["elasticsearch:9200"]
|
||||
index => "cvat.client"
|
||||
}
|
||||
} else if [type] == "server" {
|
||||
elasticsearch {
|
||||
hosts => ["elasticsearch:9200"]
|
||||
index => "cvat.server"
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,5 +0,0 @@
|
||||
|
||||
# Copyright (C) 2018 Intel Corporation
|
||||
#
|
||||
# SPDX-License-Identifier: MIT
|
||||
|
||||
@ -1,9 +0,0 @@
|
||||
|
||||
# Copyright (C) 2018 Intel Corporation
|
||||
#
|
||||
# SPDX-License-Identifier: MIT
|
||||
|
||||
from django.contrib import admin
|
||||
|
||||
# Register your models here.
|
||||
|
||||
@ -1,11 +0,0 @@
|
||||
|
||||
# Copyright (C) 2018 Intel Corporation
|
||||
#
|
||||
# SPDX-License-Identifier: MIT
|
||||
|
||||
from django.apps import AppConfig
|
||||
|
||||
|
||||
class LogProxyConfig(AppConfig):
|
||||
name = 'log_proxy'
|
||||
|
||||
@ -1,5 +0,0 @@
|
||||
|
||||
# Copyright (C) 2018 Intel Corporation
|
||||
#
|
||||
# SPDX-License-Identifier: MIT
|
||||
|
||||
@ -1,9 +0,0 @@
|
||||
|
||||
# Copyright (C) 2018 Intel Corporation
|
||||
#
|
||||
# SPDX-License-Identifier: MIT
|
||||
|
||||
from django.db import models
|
||||
|
||||
# Create your models here.
|
||||
|
||||
@ -1,91 +0,0 @@
|
||||
|
||||
# Copyright (C) 2018 Intel Corporation
|
||||
#
|
||||
# SPDX-License-Identifier: MIT
|
||||
|
||||
from django.conf import settings
|
||||
import os
|
||||
import logging
|
||||
import requests
|
||||
import json
|
||||
from urllib.parse import urlparse
|
||||
from enum import Enum
|
||||
from cvat.apps.engine.models import Job, Task
|
||||
|
||||
from requests.adapters import HTTPAdapter
|
||||
from urllib3.util.retry import Retry
|
||||
|
||||
class ClientLoggerStorage:
|
||||
def __init__(self):
|
||||
self._storage = dict()
|
||||
self._formatter = logging.Formatter('%(message)s')
|
||||
|
||||
def __getitem__(self, tid):
|
||||
if tid not in self._storage:
|
||||
self._storage[tid] = self._create_client_logger(tid)
|
||||
return self._storage[tid]
|
||||
|
||||
def _create_client_logger(self, tid):
|
||||
task = self._get_task(tid)
|
||||
logger = logging.getLogger(name='client_annotation_logger_{}'.format(tid))
|
||||
logger.setLevel(logging.INFO)
|
||||
handler = logging.FileHandler(filename=task.get_client_log_path())
|
||||
handler.setFormatter(self._formatter)
|
||||
logger.addHandler(handler)
|
||||
return logger
|
||||
|
||||
def _get_task(self, tid):
|
||||
try:
|
||||
return Task.objects.get(pk=tid)
|
||||
except Exception:
|
||||
raise Exception('Key must be task indentificator')
|
||||
|
||||
class ClientLogProxy():
|
||||
class _HandlerType(Enum):
|
||||
FILE = 1
|
||||
HTTP = 2
|
||||
|
||||
def __init__(self):
|
||||
self._client_logger = ClientLoggerStorage()
|
||||
def file_log_handler(tid, messages):
|
||||
for event in messages:
|
||||
self._client_logger[tid].info(json.dumps(event))
|
||||
|
||||
self._handlers = {self._HandlerType.FILE: file_log_handler}
|
||||
|
||||
log_server_url = os.environ.get('DJANGO_LOG_SERVER_URL')
|
||||
|
||||
def create_retry_session(retries=3, session=None, backoff_factor=0.3):
|
||||
session = session or requests.Session()
|
||||
retry = Retry(total=retries, backoff_factor=backoff_factor)
|
||||
adapter = HTTPAdapter(max_retries=retry)
|
||||
session.mount('http://', adapter)
|
||||
session.mount('https://', adapter)
|
||||
return session
|
||||
|
||||
if log_server_url:
|
||||
parse_result = urlparse(log_server_url)
|
||||
|
||||
if parse_result.scheme and 'http' not in parse_result.scheme:
|
||||
raise Exception('unsuported annotation log destination')
|
||||
|
||||
def http_log_handler(taskID, messages):
|
||||
r = create_retry_session().post(url=log_server_url, json=messages, verify=False)
|
||||
r.raise_for_status()
|
||||
|
||||
self._handlers[self._HandlerType.HTTP] = http_log_handler
|
||||
|
||||
def push_logs(self, jid, logs):
|
||||
taskID = self._get_task_id(jid)
|
||||
|
||||
for handler in self._handlers.values():
|
||||
handler(taskID, logs)
|
||||
|
||||
def _get_task_id(self, jid):
|
||||
try:
|
||||
job = Job.objects.select_related("segment__task").get(id=jid)
|
||||
return job.segment.task.id
|
||||
except:
|
||||
raise Exception('Key must be job indentificator')
|
||||
|
||||
client_log_proxy = ClientLogProxy()
|
||||
@ -1,9 +0,0 @@
|
||||
|
||||
# Copyright (C) 2018 Intel Corporation
|
||||
#
|
||||
# SPDX-License-Identifier: MIT
|
||||
|
||||
from django.test import TestCase
|
||||
|
||||
# Create your tests here.
|
||||
|
||||
@ -1,12 +0,0 @@
|
||||
|
||||
# Copyright (C) 2018 Intel Corporation
|
||||
#
|
||||
# SPDX-License-Identifier: MIT
|
||||
|
||||
from django.urls import path
|
||||
from . import views
|
||||
|
||||
urlpatterns = [
|
||||
path('exception/<int:jid>', views.exception_receiver),
|
||||
]
|
||||
|
||||
@ -1,25 +0,0 @@
|
||||
|
||||
# Copyright (C) 2018 Intel Corporation
|
||||
#
|
||||
# SPDX-License-Identifier: MIT
|
||||
|
||||
from django.http import HttpResponse, HttpResponseBadRequest
|
||||
from django.contrib.auth.decorators import permission_required
|
||||
from .proxy_logger import client_log_proxy
|
||||
from cvat.apps.authentication.decorators import login_required
|
||||
|
||||
|
||||
import json
|
||||
|
||||
# Create your views here.
|
||||
@login_required()
|
||||
@permission_required('engine.view_task', raise_exception=True)
|
||||
def exception_receiver(request, jid):
|
||||
data = json.loads(request.body.decode('utf-8'))
|
||||
try:
|
||||
if 'exceptions' in data:
|
||||
client_log_proxy.push_logs(jid, data['exceptions'])
|
||||
except Exception as e:
|
||||
return HttpResponseBadRequest(str(e))
|
||||
|
||||
return HttpResponse()
|
||||
Loading…
Reference in New Issue