From 641178bdb7c1832369e164cefda2738eebd7b9c4 Mon Sep 17 00:00:00 2001 From: JThink Date: Tue, 8 Aug 2017 15:48:14 +0800 Subject: [PATCH] add file --- .../src/main/resources/shell/create-topic.sh | 6 ++ .../src/main/resources/shell/create-index.py | 59 ++++++++++++++++++ .../src/main/resources/shell/install.sh | 6 ++ .../src/main/resources/shell/start.sh | 12 ++++ .../src/main/resources/shell/create-index.py | 60 +++++++++++++++++++ .../src/main/resources/shell/install.sh | 6 ++ .../src/main/resources/shell/start.sh | 12 ++++ .../src/main/resources/shell/hbase | 3 + 8 files changed, 164 insertions(+) create mode 100644 skyeye-collector/skyeye-collector-core/src/main/resources/shell/create-topic.sh create mode 100644 skyeye-collector/skyeye-collector-indexer/src/main/resources/shell/create-index.py create mode 100644 skyeye-collector/skyeye-collector-indexer/src/main/resources/shell/install.sh create mode 100644 skyeye-collector/skyeye-collector-indexer/src/main/resources/shell/start.sh create mode 100644 skyeye-collector/skyeye-collector-metrics/src/main/resources/shell/create-index.py create mode 100644 skyeye-collector/skyeye-collector-metrics/src/main/resources/shell/install.sh create mode 100644 skyeye-collector/skyeye-collector-metrics/src/main/resources/shell/start.sh create mode 100644 skyeye-collector/skyeye-collector-trace/src/main/resources/shell/hbase diff --git a/skyeye-collector/skyeye-collector-core/src/main/resources/shell/create-topic.sh b/skyeye-collector/skyeye-collector-core/src/main/resources/shell/create-topic.sh new file mode 100644 index 0000000..a6ae5d3 --- /dev/null +++ b/skyeye-collector/skyeye-collector-core/src/main/resources/shell/create-topic.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +# zk servers, 192.168.88.70:2181,192.168.88.71:2181,192.168.88.72:2181/kafka/0.10.0.1 +urls=$1 + +bin/kafka-topics.sh --create --zookeeper ${urls} --replication-factor 3 --partitions 3 --topic app-log \ No newline at end of file diff --git a/skyeye-collector/skyeye-collector-indexer/src/main/resources/shell/create-index.py b/skyeye-collector/skyeye-collector-indexer/src/main/resources/shell/create-index.py new file mode 100644 index 0000000..5b05503 --- /dev/null +++ b/skyeye-collector/skyeye-collector-indexer/src/main/resources/shell/create-index.py @@ -0,0 +1,59 @@ +#!/usr/bin/python198 +# -*- coding: UTF-8 -*- + +import sys +import datetime +from pyelasticsearch import ElasticSearch +from pyelasticsearch import bulk_chunks + +def main(argv): + index = argv[1] + doc_type = 'log' + url = [] + urls = argv[2].strip().split(',') + for u in urls: + url.append(u) + + es = ElasticSearch(urls = url, timeout = 60, max_retries = 0) + create_mapping(es, index, doc_type) + +def create_mapping(es, index, doc_type): + mapping = { + 'settings': { + 'index': { + 'number_of_replicas': 1, + 'number_of_shards': 6, + 'refresh_interval': '5s' + } + }, + 'mappings': { + '_default_': { + '_all': { + 'enabled': False + } + }, + doc_type : { + 'properties' : { + 'day': { 'type': 'string', 'index': 'not_analyzed'}, + 'time': { 'type': 'string', 'index': 'not_analyzed'}, + 'nanoTime': { 'type': 'string', 'index': 'not_analyzed'}, + 'created': { 'type': 'date', 'index': 'not_analyzed'}, + 'app': { 'type': 'string', 'index': 'not_analyzed'}, + 'host': { 'type': 'string', 'index': 'not_analyzed'}, + 'thread': { 'type': 'string', 'index': 'not_analyzed'}, + 'level': { 'type': 'string', 'index': 'not_analyzed'}, + 'eventType': { 'type': 'string', 'index': 'not_analyzed'}, + 'pack': { 'type': 'string', 'index': 'not_analyzed'}, + 'clazz': { 'type': 'string', 'index': 'not_analyzed'}, + 'line': { 'type': 'string', 'index': 'not_analyzed'}, + 'messageSmart': { 'type': 'string', 'analyzer': 'ik_smart', 'search_analyzer': 'ik_smart', 'include_in_all': 'true', 'boost': 8}, + 'messageMax': { 'type': 'string', 'analyzer': 'ik_max_word', 'search_analyzer': 'ik_max_word', 'include_in_all': 'true', 'boost': 8} + } + } + } + } + es.create_index(index = index, settings = mapping) + + +if __name__ == '__main__': + main(sys.argv) \ No newline at end of file diff --git a/skyeye-collector/skyeye-collector-indexer/src/main/resources/shell/install.sh b/skyeye-collector/skyeye-collector-indexer/src/main/resources/shell/install.sh new file mode 100644 index 0000000..11a6e96 --- /dev/null +++ b/skyeye-collector/skyeye-collector-indexer/src/main/resources/shell/install.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +echo "--> install python-pip module" +sudo apt-get install python-pip +echo "--> install elasticsearch module" +sudo pip install pyelasticsearch \ No newline at end of file diff --git a/skyeye-collector/skyeye-collector-indexer/src/main/resources/shell/start.sh b/skyeye-collector/skyeye-collector-indexer/src/main/resources/shell/start.sh new file mode 100644 index 0000000..d38d58b --- /dev/null +++ b/skyeye-collector/skyeye-collector-indexer/src/main/resources/shell/start.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +# 索引 +# urls http://192.168.88.70:9200,http://192.168.88.71:9200,http://192.168.88.72:9200 +# call: bash start.sh app-log http://192.168.88.70:9200,http://192.168.88.71:9200,http://192.168.88.72:9200 + +index=$1 +urls=$2 + +echo '创建索引开始' +python create-index.py ${index} ${urls} +echo '创建索引结束' \ No newline at end of file diff --git a/skyeye-collector/skyeye-collector-metrics/src/main/resources/shell/create-index.py b/skyeye-collector/skyeye-collector-metrics/src/main/resources/shell/create-index.py new file mode 100644 index 0000000..bada3c6 --- /dev/null +++ b/skyeye-collector/skyeye-collector-metrics/src/main/resources/shell/create-index.py @@ -0,0 +1,60 @@ +#!/usr/bin/python198 +# -*- coding: UTF-8 -*- + +import sys +import datetime +from pyelasticsearch import ElasticSearch +from pyelasticsearch import bulk_chunks + +def main(argv): + index = argv[1] + doc_type = 'log' + url = [] + urls = argv[2].strip().split(',') + for u in urls: + url.append(u) + + es = ElasticSearch(urls = url, timeout = 60, max_retries = 0) + create_mapping(es, index, doc_type) + +def create_mapping(es, index, doc_type): + mapping = { + 'settings': { + 'index': { + 'number_of_replicas': 1, + 'number_of_shards': 6, + 'refresh_interval': '5s' + } + }, + 'mappings': { + '_default_': { + '_all': { + 'enabled': False + } + }, + doc_type : { + 'properties' : { + 'created': { 'type': 'date', 'index': 'not_analyzed'}, + 'time': { 'type': 'string', 'index': 'not_analyzed'}, + 'day': { 'type': 'string', 'index': 'not_analyzed'}, + 'week': { 'type': 'string', 'index': 'not_analyzed'}, + 'month': { 'type': 'string', 'index': 'not_analyzed'}, + 'year': { 'type': 'string', 'index': 'not_analyzed'}, + 'app': { 'type': 'string', 'index': 'not_analyzed'}, + 'host': { 'type': 'string', 'index': 'not_analyzed'}, + 'eventType': { 'type': 'string', 'index': 'not_analyzed'}, + 'account': { 'type': 'string', 'index': 'not_analyzed'}, + 'uniqueName': { 'type': 'string', 'index': 'not_analyzed'}, + 'cost': { 'type': 'long', 'index': 'not_analyzed'}, + 'status': { 'type': 'string', 'index': 'not_analyzed'}, + 'messageSmart': { 'type': 'string', 'analyzer': 'ik_smart', 'search_analyzer': 'ik_smart', 'include_in_all': 'true', 'boost': 8}, + 'messageMax': { 'type': 'string', 'analyzer': 'ik_max_word', 'search_analyzer': 'ik_max_word', 'include_in_all': 'true', 'boost': 8} + } + } + } + } + es.create_index(index = index, settings = mapping) + + +if __name__ == '__main__': + main(sys.argv) \ No newline at end of file diff --git a/skyeye-collector/skyeye-collector-metrics/src/main/resources/shell/install.sh b/skyeye-collector/skyeye-collector-metrics/src/main/resources/shell/install.sh new file mode 100644 index 0000000..11a6e96 --- /dev/null +++ b/skyeye-collector/skyeye-collector-metrics/src/main/resources/shell/install.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +echo "--> install python-pip module" +sudo apt-get install python-pip +echo "--> install elasticsearch module" +sudo pip install pyelasticsearch \ No newline at end of file diff --git a/skyeye-collector/skyeye-collector-metrics/src/main/resources/shell/start.sh b/skyeye-collector/skyeye-collector-metrics/src/main/resources/shell/start.sh new file mode 100644 index 0000000..4ef4c2e --- /dev/null +++ b/skyeye-collector/skyeye-collector-metrics/src/main/resources/shell/start.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +# 索引 +# urls http://192.168.88.70:9200,http://192.168.88.71:9200,http://192.168.88.72:9200 +# call: bash start.sh event-log http://192.168.88.70:9200,http://192.168.88.71:9200,http://192.168.88.72:9200 + +index=$1 +urls=$2 + +echo '创建索引开始' +python create-index.py ${index} ${urls} +echo '创建索引结束' \ No newline at end of file diff --git a/skyeye-collector/skyeye-collector-trace/src/main/resources/shell/hbase b/skyeye-collector/skyeye-collector-trace/src/main/resources/shell/hbase new file mode 100644 index 0000000..a75fc56 --- /dev/null +++ b/skyeye-collector/skyeye-collector-trace/src/main/resources/shell/hbase @@ -0,0 +1,3 @@ +create 'trace', {NAME => 'span', VERSIONS => 1, COMPRESSION => 'SNAPPY'}, {NUMREGIONS => 3, SPLITALGO => 'HexStringSplit'} +create 'time_consume', {NAME => 'trace', VERSIONS => 1, COMPRESSION => 'SNAPPY'}, {NUMREGIONS => 3, SPLITALGO => 'HexStringSplit'} +create 'annotation', {NAME => 'trace', VERSIONS => 1, COMPRESSION => 'SNAPPY'}, {NUMREGIONS => 3, SPLITALGO => 'HexStringSplit'}