Monitoring services
The following services are capturing and exposing data to monitor Spider.
Below is their default configuration.
Alert
"@id": alert
"@type": ServerConfiguration
version: '0.1'
monitor:
searchLogs:
uri: http://monitor-read/v1/logs/_search
timeout: PT30S
label: GET /monitor-read/logs
searchStatus:
uri: http://whisps-status/v1/status/history/_search
timeout: PT30S
label: GET /whisps-status/status/history
searchMetricbeat:
uri: http://monitor-read/v1/metricbeat/_search
timeout: PT30S
label: GET /monitor-read/metricbeat
searchESNodes:
uri: http://monitor-read/v1/elasticsearchNodes/_search
timeout: PT30S
label: GET /monitor-read/elasticsearchNodes
searchParsers:
uri: http://monitor-read/v1/parsers/_search
timeout: PT30S
label: GET /monitor-read/parsers
searchPollers:
uri: http://monitor-read/v1/pollers/_search
timeout: PT30S
label: GET /monitor-read/pollers
checkLicense:
uri: http://stats-collector:3000/v1/stats/collect
timeout: PT30S
label: GET /stats/collect
mailSender:
uri: http://mail-sender/v1/emails
timeout: PT25S
emails:
admins:
- thibaut.raballand@floocus.com
probes:
tooManyLogs:
active: true
maxErrorLogsPerMin: 10
delayWhenInactive: PT5M
delayWhenActive: PT1M
noNewStatus:
active: true
minStatusPerMin: 5
delayWhenInactive: PT5M
delayWhenActive: PT1M
infraChange:
active: true
delayWhenInactive: PT5M
delayWhenActive: PT1M
lowESFreeSpace:
active: true
minESFreeSpaceInGB: 10
delayWhenInactive: PT5M
delayWhenActive: PT1M
healthcheckES:
active: true
delayWhenInactive: PT5M
delayWhenActive: PT1M
healthcheckRedis:
active: true
minRedisFreeMemoryInMB: 100
delayWhenInactive: PT5M
delayWhenActive: PT1M
parsingDelay:
active: true
threshold: PT25S
delayWhenInactive: PT5M
delayWhenActive: PT1M
licenseInvalid:
active: true
delayWhenInactive: P1D
delayWhenActive: PT1M
oldLicenseStats:
active: true
threshold: PT13H
delayWhenInactive: PT1H
delayWhenActive: PT1M
pollingQueued:
active: true
threshold: 10000
delayWhenInactive: PT5M
delayWhenActive: PT1M
stores:
elasticsearch:
targets:
- name: Cluster
cluster: http://elasticsearch:9200
redis:
targets:
- name: Redis cache
server: redis
port: 6379
downstreamCb:
circuitDuration: PT10S
circuitThreshold: 0.1
endpoint: http://spider.io/network
flips:
authenticatedHealthApi: false
CaptureStatusPoller
"@id": capture-status-poller
"@type": ServerConfiguration
version: '0.1'
itemStore:
node: http://elasticsearch:9200
useDataStoragePolicies: true
streamedDataStoragePolicies:
- name: default
indexSave: spider-active-capture-status-streaming-default
saveTimeout: PT5S
connectTimeout: PT2S
connectRetryDelay: PT15S
connectRetryTimes: 15
circuitDuration: PT30S
circuitThreshold: 1
itemCache:
server: redis
port: 6379
db: 4
timeOut: PT2S
circuitDuration: PT15S
circuitThreshold: 1
polling:
queue: statsSynchro
queueType: CAPTURE_STATUS
delayBeforePollingStatus: PT90S
size: 175
jobRunners: 1
delay: PT30S
automaticESIds: true
updateCache:
keepInCache: false
scoreAttribute: score
whisps:
config:
uri: http://whisp/v1/whisperers/{id}/config?view=full
label: "/whisp/whisperers/config"
timeout: PT10S
CiphersRawStatusPoller
"@id": ciphers-raw-status-poller
"@type": ServerConfiguration
version: '0.1'
logField: status
itemStore:
node: http://elasticsearch:9200
indexSave: spider-active-raw-ciphers-status
saveTimeout: PT5S
connectTimeout: PT2S
connectRetryDelay: PT15S
connectRetryTimes: 15
circuitDuration: PT30S
circuitThreshold: 1
itemCache:
server: redis
port: 6379
db: 15
timeOut: PT2S
ttl: PT45S
circuitDuration: PT15S
circuitThreshold: 1
polling:
queue: rawStatusToSynchronize
queueType: LIST
size: 100
jobRunners: 1
delay: PT20S
automaticESIds: true
updateCache:
keepInCache: false
removeFromCacheCondition: {}
noTTLCondition: {}
whisps:
config:
uri: http://whisp/v1/whisperers/{id}/config?view=full
label: "/whisp/whisperers/config"
timeout: PT10S
CiphersStatus
"@id": ciphers-status
"@type": ServerConfiguration
version: '0.1'
rawStatusStore:
node: http://elasticsearch:9200
indexGet: spider-search-raw-ciphers-status
getTimeout: PT10S
saveTimeout: PT2S
connectTimeout: PT2S
connectRetryDelay: PT15S
connectRetryTimes: 15
circuitDuration: PT15S
circuitThreshold: 1
rawStatusCache:
server: redis
port: 6379
db: 15
timeOut: PT2S
ttl: PT45S
circuitDuration: PT15S
circuitThreshold: 1
ciphersStatusStore:
node: http://elasticsearch:9200
indexGet: spider-ciphers-status
getTimeout: PT10S
saveTimeout: PT2S
connectTimeout: PT2S
connectRetryDelay: PT15S
connectRetryTimes: 15
circuitDuration: PT15S
circuitThreshold: 1
ciphersStatusCache:
server: redis
port: 6379
db: 15
timeOut: PT2S
ttl: PT45S
circuitDuration: PT15S
circuitThreshold: 1
searchRequest:
sizeLimitKB: 100
getStatusByListRequest:
sizeLimitKB: 200
ciphers:
statusLimitKB: 500
uri: http://ciphers/v1/gociphers
label: "/ciphers/gociphers"
timeout: PT10S
CiphersStatusAgg
"@id": ciphers-status-agg
"@type": ServerConfiguration
version: '0.1'
ciphersStatusStore:
node: http://elasticsearch:9200
indexGet: spider-ciphers-status
getTimeout: PT2S
connectTimeout: PT2S
connectRetryDelay: PT15S
connectRetryTimes: 15
circuitDuration: PT15S
circuitThreshold: 1
ciphersRawStatusCache:
server: redis
port: 6379
db: 15
timeOut: PT2S
circuitDuration: PT15S
circuitThreshold: 1
ciphersStatusCache:
server: redis
port: 6379
db: 15
timeOut: PT2S
circuitDuration: PT15S
circuitThreshold: 1
polling:
queue: statusToAggregate
size: 50
jobRunners: 1
delay: PT2S
resetSessionAfter: PT5M
includeInLatestStatsIfBefore: PT2M
removeStatusFromAggAfter: P7D
CiphersStatusPoller
"@id": ciphers-status-poller
"@type": ServerConfiguration
version: '0.1'
logField: ciphersStatus
itemStore:
node: http://elasticsearch:9200
indexSave: spider-ciphers-status
saveTimeout: PT5S
connectTimeout: PT2S
connectRetryDelay: PT15S
connectRetryTimes: 15
circuitDuration: PT30S
circuitThreshold: 1
itemCache:
server: redis
port: 6379
db: 15
timeOut: PT2S
ttl: PT24H
circuitDuration: PT15S
circuitThreshold: 1
polling:
queue: ciphersStatusToSynchronize
queueType: SORTED_SET
scoreAttribute: timeStamp
size: 100
jobRunners: 1
delay: PT20S
automaticESIds: false
updateCache:
keepInCache: true
removeFromCacheCondition: {}
noTTLCondition: {}
whisps:
config:
uri: http://whisp/v1/whisperers/{id}/config?view=full
label: "/whisp/whisperers/config"
timeout: PT10S
GuiLogs
"@id": gui-logs
"@type": ServerConfiguration
version: '0.1'
store:
node: http://elasticsearch:9200
indexSave: spider-active-guilogs
indexGet: spider-search-guilogs
getTimeout: PT2S
saveTimeout: PT2S
connectTimeout: PT2S
connectRetryDelay: PT15S
connectRetryTimes: 15
circuitDuration: PT15S
circuitThreshold: 1
jsonSizeLimitKB: 100
searchRequest:
sizeLimitKB: 100
MonitorRead
"@id": monitor-read
"@type": ServerConfiguration
version: '0.1'
circuitBreakers:
interval: PT20S
bucketSpan: PT5S
bucketCount: 5
duration: PT30S
threshold: 1
targets:
pollers:
endpoint: pollers
store:
node: http://elasticsearch:9200
indexGet: spider-search-collect-poller
getTimeout: PT30S
connectTimeout: PT2S
connectRetryDelay: PT15S
connectRetryTimes: 15
parsers:
endpoint: parsers
store:
node: http://elasticsearch:9200
indexGet: spider-search-collect-parsers
getTimeout: PT30S
connectTimeout: PT2S
connectRetryDelay: PT15S
connectRetryTimes: 15
parsingQueues:
endpoint: parsingqueues
store:
node: http://elasticsearch:9200
indexGet: spider-search-collect-parsingqueues
getTimeout: PT30S
connectTimeout: PT2S
connectRetryDelay: PT15S
connectRetryTimes: 15
circuitBreakers:
endpoint: circuitbreakers
store:
node: http://elasticsearch:9200
indexGet: spider-search-collect-circuitbreaker
getTimeout: PT30S
connectTimeout: PT2S
connectRetryDelay: PT15S
connectRetryTimes: 15
redis:
endpoint: redis
store:
node: http://elasticsearch:9200
indexGet: spider-search-collect-redis
getTimeout: PT30S
connectTimeout: PT2S
connectRetryDelay: PT15S
connectRetryTimes: 15
elasticSearch:
endpoint: elasticsearch
store:
node: http://elasticsearch:9200
indexGet: spider-search-collect-elasticsearch
getTimeout: PT30S
connectTimeout: PT2S
connectRetryDelay: PT15S
connectRetryTimes: 15
elasticSearchNodes:
endpoint: elasticsearchNodes
store:
node: http://elasticsearch:9200
indexGet: spider-search-collect-elasticsearchnodes
getTimeout: PT30S
connectTimeout: PT2S
connectRetryDelay: PT15S
connectRetryTimes: 15
process:
endpoint: processes
store:
node: http://elasticsearch:9200
indexGet: spider-search-collect-process
getTimeout: PT30S
connectTimeout: PT2S
connectRetryDelay: PT15S
connectRetryTimes: 15
api:
endpoint: api
store:
node: http://elasticsearch:9200
indexGet: spider-search-collect-api
getTimeout: PT30S
connectTimeout: PT2S
connectRetryDelay: PT15S
connectRetryTimes: 15
logs:
endpoint: logs
store:
node: http://elasticsearch:9200
indexGet: spider-search-logs
timeField: "@timestamp"
getTimeout: PT30S
connectTimeout: PT2S
connectRetryDelay: PT15S
connectRetryTimes: 15
metricbeat:
endpoint: metricbeat
store:
node: http://elasticsearch:9200
indexGet: spider-search-metricbeat
timeField: "@timestamp"
getTimeout: PT30S
connectTimeout: PT2S
connectRetryDelay: PT15S
connectRetryTimes: 15
searchRequest:
sizeLimitKB: 100
MonitorWrite
---
"@id": monitor-write
"@type": ServerConfiguration
version: '0.1'
targets:
pollers:
uri: http://{poller}/v1/queueStatus
timeout: PT10S
pollingDelay: PT20S
targets:
- name: PackPoller
poller: pack-poller
- name: TcpPoller
poller: tcp-poller
- name: HttpComPoller
poller: web-httpcom-poller
- name: HttpComContentPoller
poller: web-httpcom-content-poller
- name: HttpPersPoller
poller: web-httppers-poller
- name: StatusPoller
poller: status-poller
- name: HostsPoller
poller: hosts-poller
- name: HostsAgg
poller: hosts-agg
- name: WhispsStatusPoller
poller: whisp-status-poller
- name: WhispsStatusAgg
poller: whisps-status-agg
- name: ParsingStatusHttpPersPoller
poller: parsing-status-httppers-poller
- name: ParsingStatusTcpSessionPoller
poller: parsing-status-tcpsession-poller
- name: CaptureStatusPoller
poller: capture-status-poller
- name: CiphersStatusPoller
poller: ciphers-status-poller
- name: CiphersRawStatusPoller
poller: ciphers-raw-status-poller
parsers:
uri: http://{ip}:3000/v1/stats/parsing
timeout: PT10S
pollingDelay: PT20S
targets:
- name: WebWrite
virtualHost: web-write
- name: TlsKeysLinker
virtualHost: tls-keys-linker
parsingQueues:
uri: http://{host}/v1/waiting-stats/{protocol}
timeout: PT10S
pollingDelay: PT20S
targets:
- name: Http
host: tcp-update
protocol: http
cbGroup: TcpUpdate
- name: Tls
host: tls-keys
protocol: tls
cbGroup: TlsKeys
circuitBreakers:
uri: http://{ip}:3000/v1/stats/cb
timeout: PT10S
pollingDelay: PT20S
targets:
- name: Ciphers
virtualHost: ciphers
- name: CiphersStatus
virtualHost: ciphers-status
- name: CiphersStatusAgg
virtualHost: ciphers-status-agg
- name: CiphersStatusPoller
virtualHost: ciphers-status-poller
- name: CiphersRawStatusPoller
virtualHost: ciphers-raw-status-poller
- name: Config
virtualHost: config
- name: Controls
virtualHost: controls
- name: PackPoller
virtualHost: pack-poller
- name: TcpPoller
virtualHost: tcp-poller
- name: HttpComPoller
virtualHost: web-httpcom-poller
- name: HttpComContentPoller
virtualHost: web-httpcom-content-poller
- name: HttpPersPoller
virtualHost: web-httppers-poller
- name: PackWrite
virtualHost: pack-write
- name: PackUpdate
virtualHost: pack-update
- name: PackRead
virtualHost: pack-read
- name: TcpWrite
virtualHost: tcp-write
- name: TcpUpdate
virtualHost: tcp-update
- name: WebWrite
virtualHost: web-write
- name: WebUpload
virtualHost: web-upload
- name: MonitorWrite
virtualHost: monitor-write
- name: Whisp
virtualHost: whisp
- name: WhispsStatus
virtualHost: whisps-status
- name: Hosts
virtualHost: hosts
- name: TcpRead
virtualHost: tcp-read
- name: WebRead
virtualHost: web-read
- name: MonitorRead
virtualHost: monitor-read
- name: StatusPoller
virtualHost: status-poller
- name: HostsPoller
virtualHost: hosts-poller
- name: HostsAgg
virtualHost: hosts-agg
- name: WhispsStatusPoller
virtualHost: whisp-status-poller
- name: WhispsStatusAgg
virtualHost: whisps-status-agg
- name: Customer
virtualHost: customer
- name: MailSender
virtualHost: mail-sender
- name: GUISettings
virtualHost: gui-settings
- name: GUILogs
virtualHost: gui-logs
- name: Jobs
virtualHost: job
- name: Links
virtualHost: link
- name: Sessions
virtualHost: session
- name: ParsingStatusHttpPersPoller
virtualHost: parsing-status-httppers-poller
- name: ParsingStatusTcpSessionPoller
virtualHost: parsing-status-tcpsession-poller
- name: CaptureStatusPoller
virtualHost: capture-status-poller
- name: Login
virtualHost: login
- name: SelfMonitoring
virtualHost: self-monitoring
- name: NetworkView
virtualHost: networkview
- name: Alert
virtualHost: alert
- name: Teams
virtualHost: teams
- name: StatsCollector
virtualHost: stats-collector
- name: Maintenance
virtualHost: maintenance
- name: Plugins
virtualHost: plugins
- name: TlsKeys
virtualHost: tls-keys
- name: TlsKeysLinker
virtualHost: tls-keys-linker
process:
uri: http://{ip}:3000/v1/stats/process
timeout: PT10S
pollingDelay: PT20S
targets:
- name: Ciphers
virtualHost: ciphers
- name: CiphersStatus
virtualHost: ciphers-status
- name: CiphersStatusAgg
virtualHost: ciphers-status-agg
- name: CiphersStatusPoller
virtualHost: ciphers-status-poller
- name: CiphersRawStatusPoller
virtualHost: ciphers-raw-status-poller
- name: Config
virtualHost: config
- name: Controls
virtualHost: controls
- name: PackPoller
virtualHost: pack-poller
- name: TcpPoller
virtualHost: tcp-poller
- name: HttpComPoller
virtualHost: web-httpcom-poller
- name: HttpComContentPoller
virtualHost: web-httpcom-content-poller
- name: HttpPersPoller
virtualHost: web-httppers-poller
- name: PackWrite
virtualHost: pack-write
- name: PackUpdate
virtualHost: pack-update
- name: PackRead
virtualHost: pack-read
- name: TcpWrite
virtualHost: tcp-write
- name: TcpUpdate
virtualHost: tcp-update
- name: TcpRead
virtualHost: tcp-read
- name: WebWrite
virtualHost: web-write
- name: WebUpload
virtualHost: web-upload
- name: WebRead
virtualHost: web-read
- name: Whisp
virtualHost: whisp
- name: WhispsStatus
virtualHost: whisps-status
- name: Hosts
virtualHost: hosts
- name: Customer
virtualHost: customer
- name: MonitorWrite
virtualHost: monitor-write
- name: MonitorRead
virtualHost: monitor-read
- name: Jobs
virtualHost: job
- name: Links
virtualHost: link
- name: Sessions
virtualHost: session
- name: StatusPoller
virtualHost: status-poller
- name: HostsPoller
virtualHost: hosts-poller
- name: HostsAgg
virtualHost: hosts-agg
- name: WhispsStatusPoller
virtualHost: whisp-status-poller
- name: WhispsStatusAgg
virtualHost: whisps-status-agg
- name: GUISettings
virtualHost: gui-settings
- name: GUILogs
virtualHost: gui-logs
- name: MailSender
virtualHost: mail-sender
- name: ParsingStatusHttpPersPoller
virtualHost: parsing-status-httppers-poller
- name: ParsingStatusTcpSessionPoller
virtualHost: parsing-status-tcpsession-poller
- name: CaptureStatusPoller
virtualHost: capture-status-poller
- name: Login
virtualHost: login
- name: SelfMonitoring
virtualHost: self-monitoring
- name: NetworkView
virtualHost: networkview
- name: Alert
virtualHost: alert
- name: Teams
virtualHost: teams
- name: StatsCollector
virtualHost: stats-collector
- name: Maintenance
virtualHost: maintenance
- name: Plugins
virtualHost: plugins
- name: TlsKeys
virtualHost: tls-keys
- name: TlsKeysLinker
virtualHost: tls-keys-linker
apis:
uri: http://{ip}:3000/v1/stats/api
timeout: PT10S
pollingDelay: PT20S
targets:
- name: Ciphers
virtualHost: ciphers
- name: CiphersStatus
virtualHost: ciphers-status
- name: CiphersStatusAgg
virtualHost: ciphers-status-agg
- name: CiphersStatusPoller
virtualHost: ciphers-status-poller
- name: CiphersRawStatusPoller
virtualHost: ciphers-raw-status-poller
- name: Config
virtualHost: config
- name: Controls
virtualHost: controls
- name: PackPoller
virtualHost: pack-poller
- name: TcpPoller
virtualHost: tcp-poller
- name: HttpComPoller
virtualHost: web-httpcom-poller
- name: HttpComContentPoller
virtualHost: web-httpcom-content-poller
- name: HttpPersPoller
virtualHost: web-httppers-poller
- name: PackWrite
virtualHost: pack-write
- name: PackUpdate
virtualHost: pack-update
- name: PackRead
virtualHost: pack-read
- name: TcpWrite
virtualHost: tcp-write
- name: TcpUpdate
virtualHost: tcp-update
- name: TcpRead
virtualHost: tcp-read
- name: WebWrite
virtualHost: web-write
- name: WebUpload
virtualHost: web-upload
- name: WebRead
virtualHost: web-read
- name: Whisp
virtualHost: whisp
- name: WhispsStatus
virtualHost: whisps-status
- name: Hosts
virtualHost: hosts
- name: Customer
virtualHost: customer
- name: MonitorWrite
virtualHost: monitor-write
- name: MonitorRead
virtualHost: monitor-read
- name: Jobs
virtualHost: job
- name: Links
virtualHost: link
- name: Sessions
virtualHost: session
- name: StatusPoller
virtualHost: status-poller
- name: HostsPoller
virtualHost: hosts-poller
- name: HostsAgg
virtualHost: hosts-agg
- name: WhispsStatusPoller
virtualHost: whisp-status-poller
- name: WhispsStatusAgg
virtualHost: whisps-status-agg
- name: GUISettings
virtualHost: gui-settings
- name: GUILogs
virtualHost: gui-logs
- name: MailSender
virtualHost: mail-sender
- name: ParsingStatusHttpPersPoller
virtualHost: parsing-status-httppers-poller
- name: ParsingStatusTcpSessionPoller
virtualHost: parsing-status-tcpsession-poller
- name: CaptureStatusPoller
virtualHost: capture-status-poller
- name: Login
virtualHost: login
- name: SelfMonitoring
virtualHost: self-monitoring
- name: NetworkView
virtualHost: networkview
- name: Alert
virtualHost: alert
- name: Teams
virtualHost: teams
- name: StatsCollector
virtualHost: stats-collector
- name: Maintenance
virtualHost: maintenance
- name: Plugins
virtualHost: plugins
- name: TlsKeys
virtualHost: tls-keys
- name: TlsKeysLinker
virtualHost: tls-keys-linker
redis:
pollingDelay: PT5S
targets:
- name: main
server: redis
port: 6379
elasticsearch:
uri: "{cluster}/{index}/_stats/docs,store,indexing,get,search,merge,refresh,flush,warmer,recovery"
timeout: PT10S
pollingDelay: PT20S
targets:
- name: Pack
cluster: http://elasticsearch:9200
index: spider-search-pack
- name: TcpSession
cluster: http://elasticsearch:9200
index: spider-search-tcpsession
- name: HttpCom
cluster: http://elasticsearch:9200
index: spider-search-httpcom
- name: HttpComContent
cluster: http://elasticsearch:9200
index: spider-search-httpcomcontent
- name: HttpPers
cluster: http://elasticsearch:9200
index: spider-search-httppers
- name: Host
cluster: http://elasticsearch:9200
index: spider-search-hosts
- name: Status
cluster: http://elasticsearch:9200
index: spider-search-status
- name: WhispStatus
cluster: http://elasticsearch:9200
index: spider-whisp-status
- name: Sessions
cluster: http://elasticsearch:9200
index: spider-session
- name: Links
cluster: http://elasticsearch:9200
index: spider-link
- name: Jobs
cluster: http://elasticsearch:9200
index: spider-job
- name: Customers
cluster: http://elasticsearch:9200
index: spider-customers
- name: Whisps
cluster: http://elasticsearch:9200
index: spider-whisp
- name: Logs
cluster: http://elasticsearch:9200
index: spider-search-logs
- name: Metricbeat
cluster: http://elasticsearch:9200
index: spider-search-metricbeat
- name: MonitoringES
cluster: http://elasticsearch:9200
index: spider-search-collect-elasticsearch
- name: MonitoringESNodes
cluster: http://elasticsearch:9200
index: spider-search-collect-elasticsearchnodes
- name: MonitoringRedis
cluster: http://elasticsearch:9200
index: spider-search-collect-redis
- name: MonitoringPoller
cluster: http://elasticsearch:9200
index: spider-search-collect-poller
- name: MonitoringParsingQueues
cluster: http://elasticsearch:9200
index: spider-search-collect-parsingqueues
- name: MonitoringParsers
cluster: http://elasticsearch:9200
index: spider-search-collect-parsers
- name: MonitoringCB
cluster: http://elasticsearch:9200
index: spider-search-collect-circuitbreaker
- name: MonitoringProcess
cluster: http://elasticsearch:9200
index: spider-search-collect-process
- name: GUILogs
cluster: http://elasticsearch:9200
index: spider-search-guilogs
- name: GUISettings
cluster: http://elasticsearch:9200
index: spider-guisettings
- name: TcpSessionParsingStatus
cluster: http://elasticsearch:9200
index: spider-search-parsing-status-tcpsession
- name: HttpPersParsingStatus
cluster: http://elasticsearch:9200
index: spider-search-parsing-status-httppers
- name: MonitoringApi
cluster: http://elasticsearch:9200
index: spider-search-collect-api
- name: CaptureStatus
cluster: http://elasticsearch:9200
index: spider-search-capture-status
- name: Teams
cluster: http://elasticsearch:9200
index: spider-teams
- name: Plugins
cluster: http://elasticsearch:9200
index: spider-plugins
- name: Controllers
cluster: http://elasticsearch:9200
index: spider-controllers
- name: Attachments
cluster: http://elasticsearch:9200
index: spider-attachments
- name: Ciphers
cluster: http://elasticsearch:9200
index: spider-ciphers
- name: CiphersStatus
cluster: http://elasticsearch:9200
index: spider-ciphers-status
- name: CiphersRawStatus
cluster: http://elasticsearch:9200
index: spider-search-raw-ciphers-status
- name: DSP-Upload-default
cluster: http://elasticsearch:9200
index: spider-dsp-upload-default
- name: DSP-Streaming-default
cluster: http://elasticsearch:9200
index: spider-dsp-streaming-default
elasticsearchNodes:
uri: "{cluster}/_nodes/stats/os,process,fs,jvm"
timeout: PT10S
pollingDelay: PT20S
targets:
- name: ElasticSearch
cluster: http://elasticsearch:9200
replicas:
validationPattern: "^[^.]+_\\d+\\."
ttl: PT5M
store:
node: http://elasticsearch:9200
indexMap:
PollersStatus: spider-active-collect-poller
ParsingQueueStatus: spider-active-collect-parsingqueues
ParserStatus: spider-active-collect-parsers
CircuitbreakersStatus: spider-active-collect-circuitbreaker
ProcessStatus: spider-active-collect-process
ApiStatus: spider-active-collect-api
RedisStatus: spider-active-collect-redis
ElasticsearchIndexStatus: spider-active-collect-elasticsearch
ElasticsearchNodeStatus: spider-active-collect-elasticsearchnodes
saveTimeout: PT15S
connectTimeout: PT2S
connectRetryDelay: PT15S
connectRetryTimes: 15
circuitDuration: PT30S
circuitThreshold: 1
stats:
interval: PT20S
bucketSpan: PT5S
bucketCount: 5
ParsingStatusHttpPersPoller
"@id": parsing-status-httppers-poller
"@type": ServerConfiguration
version: '0.1'
itemStore:
node: http://elasticsearch:9200
useDataStoragePolicies: true
useDistinctPolicyForUpload: true
streamedDataStoragePolicies:
- name: default
indexSave: spider-active-parsing-status-httppers-streaming-default
uploadedDataStoragePolicies:
- name: default
indexSave: spider-active-parsing-status-httppers-upload-default
saveTimeout: PT5S
connectTimeout: PT2S
connectRetryDelay: PT15S
connectRetryTimes: 15
circuitDuration: PT30S
circuitThreshold: 1
itemCache:
server: redis
port: 6379
db: 3
timeOut: PT2S
circuitDuration: PT15S
circuitThreshold: 1
polling:
queue: statsSynchro
queueType: PARSING_STATUS
delayBeforePollingStatus: PT65S
size: 175
jobRunners: 1
delay: PT5S
automaticESIds: true
updateCache:
keepInCache: false
scoreAttribute: score
whisps:
config:
uri: http://whisp/v1/whisperers/{id}/config?view=full
label: "/whisp/whisperers/config"
timeout: PT10S
ParsingStatusTcpSessionPoller
"@id": parsing-status-tcpsession-poller
"@type": ServerConfiguration
version: '0.1'
itemStore:
node: http://elasticsearch:9200
useDataStoragePolicies: true
useDistinctPolicyForUpload: true
streamedDataStoragePolicies:
- name: default
indexSave: spider-active-parsing-status-tcpsession-streaming-default
uploadedDataStoragePolicies:
- name: default
indexSave: spider-active-parsing-status-tcpsession-upload-default
saveTimeout: PT5S
connectTimeout: PT2S
connectRetryDelay: PT15S
connectRetryTimes: 15
circuitDuration: PT30S
circuitThreshold: 1
itemCache:
server: redis
port: 6379
db: 0
timeOut: PT2S
ttl: PT600S
circuitDuration: PT15S
circuitThreshold: 1
polling:
queue: statsSynchro
queueType: PARSING_STATUS
delayBeforePollingStatus: PT65S
size: 175
jobRunners: 1
delay: PT5S
automaticESIds: false
updateCache:
keepInCache: true
scoreAttribute: score
whisps:
config:
uri: http://whisp/v1/whisperers/{id}/config?view=full
label: "/whisp/whisperers/config"
timeout: PT10S
Sessions
"@id": sessions
"@type": ServerConfiguration
version: '0.1'
store:
node: http://elasticsearch:9200
indexSave: spider-session
indexGet: spider-session
getTimeout: PT10S
saveTimeout: PT10S
connectTimeout: PT2S
connectRetryDelay: PT15S
connectRetryTimes: 15
searchRequest:
sizeLimitKB: 2500
jsonSizeLimitKB: 100
StatusPoller
"@id": status-poller
"@type": ServerConfiguration
version: '0.1'
logField: status
itemStore:
node: http://elasticsearch:9200
useDataStoragePolicies: true
streamedDataStoragePolicies:
- name: default
indexSave: spider-active-status-streaming-default
saveTimeout: PT5S
connectTimeout: PT2S
connectRetryDelay: PT15S
connectRetryTimes: 15
circuitDuration: PT30S
circuitThreshold: 1
itemCache:
server: redis
port: 6379
db: 4
timeOut: PT2S
ttl: PT45S
circuitDuration: PT15S
circuitThreshold: 1
polling:
queue: statusToSynchronize
queueType: LIST
size: 100
jobRunners: 1
delay: PT20S
automaticESIds: true
updateCache:
keepInCache: false
removeFromCacheCondition: {}
noTTLCondition: {}
whisps:
config:
uri: http://whisp/v1/whisperers/{id}/config?view=full
label: "/whisp/whisperers/config"
timeout: PT10S
WhispStatusPoller
"@type": ServerConfiguration
version: '0.1'
logField: whispStatus
itemStore:
node: http://elasticsearch:9200
indexSave: spider-whisp-status
saveTimeout: PT5S
connectTimeout: PT2S
connectRetryDelay: PT15S
connectRetryTimes: 15
circuitDuration: PT30S
circuitThreshold: 1
itemCache:
server: redis
port: 6379
db: 5
timeOut: PT2S
ttl: PT24H
circuitDuration: PT15S
circuitThreshold: 1
polling:
queue: whispStatusToSynchronize
queueType: SORTED_SET
scoreAttribute: timeStamp
size: 100
jobRunners: 1
delay: PT20S
automaticESIds: false
updateCache:
keepInCache: true
removeFromCacheCondition: {}
noTTLCondition: {}
whisps:
config:
uri: http://whisp/v1/whisperers/{id}/config?view=full
label: "/whisp/whisperers/config"
timeout: PT10S
WhispsStatus
"@id": whisps-status
"@type": ServerConfiguration
version: '0.1'
statusStore:
node: http://elasticsearch:9200
indexSave: spider-active-status
indexGet: spider-search-status
indexRollUp: spider-search-capture-status
getTimeout: PT10S
saveTimeout: PT2S
connectTimeout: PT2S
connectRetryDelay: PT15S
connectRetryTimes: 15
circuitDuration: PT15S
circuitThreshold: 1
statusCache:
server: redis
port: 6379
db: 4
timeOut: PT2S
ttl: PT45S
circuitDuration: PT15S
circuitThreshold: 1
whispStatusStore:
node: http://elasticsearch:9200
indexSave: spider-whisp-status
indexGet: spider-whisp-status
getTimeout: PT10S
saveTimeout: PT2S
connectTimeout: PT2S
connectRetryDelay: PT15S
connectRetryTimes: 15
circuitDuration: PT15S
circuitThreshold: 1
whispStatusCache:
server: redis
port: 6379
db: 5
timeOut: PT2S
ttl: PT45S
circuitDuration: PT15S
circuitThreshold: 1
whisperers:
statusLimitKB: 500
searchRequest:
sizeLimitKB: 100
getStatusByListRequest:
sizeLimitKB: 200
whisps:
uri: http://whisp/v1/whisperers
label: "/whisp/whisperers"
timeout: PT10S
WhispsStatusAgg
"@id": whisps-status-agg
"@type": ServerConfiguration
version: '0.1'
whispStatusStore:
node: http://elasticsearch:9200
indexGet: spider-whisp-status
getTimeout: PT2S
connectTimeout: PT2S
connectRetryDelay: PT15S
connectRetryTimes: 15
circuitDuration: PT15S
circuitThreshold: 1
statusCache:
server: redis
port: 6379
db: 4
timeOut: PT2S
circuitDuration: PT15S
circuitThreshold: 1
whispStatusCache:
server: redis
port: 6379
db: 5
timeOut: PT2S
circuitDuration: PT15S
circuitThreshold: 1
polling:
queue: statusToAggregate
size: 50
jobRunners: 1
delay: PT2S
resetSessionAfter: PT5M
includeInLatestStatsIfBefore: PT2M
removeStatusFromAggAfter: P7D