搭建elk和head
1
2
3
4
5
|
docker pull libac/elasticsearch-rtf
docker run -d --name es -p 9200:9200 -p 9300:9300 -e ES_JAVA_OPTS="-Xms512m -Xmx512m" -e "discovery.type=single-node" -v /opt/es/es.yml:/usr/share/elasticsearch/config/elasticsearch.yml -v /opt/es/data/:/usr/share/elasticsearch/data libac/elasticsearch-rtf
docker pull mobz/elasticsearch-head:5
docker run -d -p 9100:9100 mobz/elasticsearch-head:5
|
/opt/es/es.yml参考
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
|
index:
analysis:
tokenizer:
my_pinyin:
type: pinyin
first_letter: prefix
padding_char: ''
pinyin_first_letter:
type: pinyin
first_letter: only
mmseg_maxword:
type: mmseg
seg_type: max_word
mmseg_complex:
type: mmseg
seg_type: complex
mmseg_simple:
type: mmseg
seg_type: simple
semicolon_spliter:
type: pattern
pattern: ";"
pct_spliter:
type: pattern
pattern: "[%]+"
ngram_1_to_2:
type: nGram
min_gram: 1
max_gram: 2
ngram_1_to_3:
type: nGram
min_gram: 1
max_gram: 3
filter:
ngram_min_3:
max_gram: 10
min_gram: 3
type: nGram
ngram_min_2:
max_gram: 10
min_gram: 2
type: nGram
ngram_min_1:
max_gram: 10
min_gram: 1
type: nGram
min2_length:
min: 2
max: 4
type: length
min3_length:
min: 3
max: 4
type: length
pinyin_first_letter:
type: pinyin
first_letter: only
analyzer:
lowercase_keyword:
type: custom
filter:
- lowercase
tokenizer: standard
lowercase_keyword_ngram_min_size1:
type: custom
filter:
- lowercase
- stop
- trim
- unique
tokenizer: nGram
lowercase_keyword_ngram_min_size2:
type: custom
filter:
- lowercase
- min2_length
- stop
- trim
- unique
tokenizer: nGram
lowercase_keyword_ngram_min_size3:
type: custom
filter:
- lowercase
- min3_length
- stop
- trim
- unique
tokenizer: ngram_1_to_3
lowercase_keyword_ngram:
type: custom
filter:
- lowercase
- stop
- trim
- unique
tokenizer: ngram_1_to_3
lowercase_keyword_without_standard:
type: custom
filter:
- lowercase
tokenizer: keyword
lowercase_whitespace:
type: custom
filter:
- lowercase
tokenizer: whitespace
ik:
alias:
- ik_analyzer
type: ik
ik_max_word:
type: ik
use_smart: false
ik_smart:
type: ik
use_smart: true
mmseg:
alias:
- mmseg_analyzer
type: mmseg
mmseg_maxword:
type: custom
filter:
- lowercase
tokenizer: mmseg_maxword
mmseg_complex:
type: custom
filter:
- lowercase
tokenizer: mmseg_complex
mmseg_simple:
type: custom
filter:
- lowercase
tokenizer: mmseg_simple
comma_spliter:
type: pattern
pattern: "[,|\\s]+"
pct_spliter:
type: pattern
pattern: "[%]+"
custom_snowball_analyzer:
type: snowball
language: English
simple_english_analyzer:
type: custom
tokenizer: whitespace
filter:
- standard
- lowercase
- snowball
edge_ngram:
type: custom
tokenizer: edgeNGram
filter:
- lowercase
pinyin_ngram_analyzer:
type: custom
tokenizer: my_pinyin
filter:
- lowercase
- nGram
- trim
- unique
pinyin_first_letter_analyzer:
type: custom
tokenizer: pinyin_first_letter
filter:
- standard
- lowercase
pinyin_first_letter_keyword_analyzer:
alias:
- pinyin_first_letter_analyzer_keyword
type: custom
tokenizer: keyword
filter:
- pinyin_first_letter
- lowercase
path_analyzer: #used for tokenize :/something/something/else
type: custom
tokenizer: path_hierarchy
index.analysis.analyzer.default.type: mmseg
#index.analysis.analyzer.default.type: keyword
# rtf.filter.redis.host: 127.0.0.1
# rtf.filter.redis.port: 6379
http.cors.enabled: true
http.cors.allow-origin: "*"
|
logstash搭建
1
|
nohup /usr/share/logstash/bin/logstash -f /usr/share/logstash/bin/conf-mysql/logstash-mysql-es.conf >/dev/null 2>&1 &
|
/usr/share/logstash/bin/conf-mysql/logstash-mysql-es.conf参考
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
|
input{
jdbc {
jdbc_driver_library => "/usr/share/logstash/mysql-connector-java-5.1.44-bin.jar"
jdbc_driver_class => "com.mysql.jdbc.Driver"
jdbc_connection_string => "jdbc:mysql://********:3306/pan"
jdbc_user => "root"
jdbc_password => "********"
jdbc_paging_enabled => "true"
jdbc_page_size => "1000"
jdbc_default_timezone =>"Asia/Shanghai"
schedule => "*/1 * * * *"
statement => "select * from aliyun where update_time > :sql_last_value"
use_column_value => true
tracking_column => "update_time"
last_run_metadata_path => "./logstash_jdbc_last_run"
}
}
output{
elasticsearch {
hosts => "127.0.0.1:9200"
index => "aliyun"
document_id => "%{id}"
}
stdout {
codec => json_lines
}
}
|
elk创建索引及搜索
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
|
curl -XPUT http://localhost:9200/_template/rtf -d '{
"template": "*",
"settings": { "number_of_shards": 1 },
"mappings": {
"_default_": {
"_all": {
"enabled": true
},
"dynamic_templates": [
{
"strings": {
"match_mapping_type": "string",
"mapping": {
"type": "text",
"analyzer":"ik_max_word",
"ignore_above": 256,
"fields": {
"keyword": {
"type": "keyword"
}
}
}
}
}
]
}
}
}'
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
|
curl -XPOST http://localhost:9200/aliyun/_default_/_mapping -H 'Content-Type:application/json' -d'
{
"properties": {
"title": {
"type": "string",
"analyzer": "ik_max_word"
},
"upload" : {"type" : "string","analyzer": "ik_max_word"},
"url" : {"type" : "string","analyzer": "ik_max_word"},
"pan_type" : {"type" : "string","analyzer": "ik_max_word"},
"type" : {"type" : "string","analyzer": "ik_max_word"},
"@version" : {"type" : "string","analyzer": "ik_max_word"}
}
}'
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
|
curl -XPOST http://localhost:9200/aliyun/_search -H 'Content-Type:application/json' -d'
{
"query": {
"filtered": {
"query": {
"match": {
"title": "周杰伦在东京"
}
},
"filter": {
"bool": {
"must": [
{
"term": {
"type": "music"
}
}
],
"must_not": [
{
"term": {
"disable": 1
}
}
]
}
}
}
},
"size": 10,
"from": 1,
"sort": [
{
"_timestamp": {
"order": "desc"
}
}
]
}'
|
过滤查询：
https://blog.csdn.net/lebron3v/article/details/84030836
组合查询：
https://www.elastic.co/guide/cn/elasticsearch/guide/current/combining-queries-together.html
微信打赏
支付宝打赏
