Skip to main content

Hosts management

The following services are managing Hosts.
Below is their default configuration.

Hosts​

"@id": hosts
"@type": ServerConfiguration
version: '0.1'
dnsStore:
  node: http://elasticsearch:9200
  indexGet: spider-search-hosts
  getTimeout: PT10S
  connectTimeout: PT2S
  connectRetryDelay: PT15S
  connectRetryTimes: 15
  circuitDuration: PT15S
  circuitThreshold: 1
dnsCache:
  server: redis
  port: 6379
  db: 6
  timeOut: PT2S
  ttl: PT45S
  circuitDuration: PT15S
  circuitThreshold: 1
hosts:
  updateLimitKB: 100
searchRequest:
  sizeLimitKB: 100
whisps:
  uri: http://whisp/v1/whisperers
  label: "/whisp/whisperers"
  timeout: PT10S

HostsAggs​

"@id": hosts-agg
"@type": ServerConfiguration
version: '0.1'
itemStore:
  node: http://elasticsearch:9200
  indexGet: spider-search-hosts
  getTimeout: PT10S
  connectTimeout: PT2S
  connectRetryDelay: PT15S
  connectRetryTimes: 15
  circuitDuration: PT15S
  circuitThreshold: 1
itemCache:
  server: redis
  port: 6379
  db: 6
  timeOut: PT2S
  circuitDuration: PT15S
  circuitThreshold: 1
polling:
  queue: rawHostsToAggregate
  size: 50
  jobRunners: 1
  delay: PT2S
whisps:
  uri: http://whisp/v1/whisperers
  label: "/whisp/whisperers"
  timeout: PT10S

HostsPoller​

"@id": hosts-poller
"@type": ServerConfiguration
version: '0.1'
logField: hosts
itemStore:
  node: http://elasticsearch:9200
  useDataStoragePolicies: true
  useDistinctPolicyForUpload: true
  streamedDataStoragePolicies:
    - name: default
      indexSave: spider-active-hosts-streaming-default
  uploadedDataStoragePolicies:
    - name: default
      indexSave: spider-active-hosts-upload-default
  saveTimeout: PT5S
  connectTimeout: PT2S
  connectRetryDelay: PT15S
  connectRetryTimes: 15
  circuitDuration: PT30S
  circuitThreshold: 1
itemCache:
  server: redis
  port: 6379
  db: 6
  timeOut: PT2S
  ttl: PT5M
  circuitDuration: PT15S
  circuitThreshold: 1
polling:
  queue: hostsToSynchronize
  queueType: SORTED_SET
  scoreAttribute: timeStamp
  size: 100
  jobRunners: 1
  delay: PT20S
  automaticESIds: false
  updateCache:
    keepInCache: true
    removeFromCacheCondition: {}
    noTTLCondition: {}
whisps:
  config:
    uri: http://whisp/v1/whisperers/{id}/config?view=full
    label: "/whisp/whisperers/config"
    timeout: PT10S