Http Parsing

The following services are managing HTTP communications parsing.
Below is their default configuration.

webWrite

"@id": web-streams-write
"@type": ServerConfiguration
version: '0.1'
httpComStore:
  node: http://elasticsearch:9200
  indexPurge: spider-search-httpcom-upload
  indexGet: spider-search-httpcom
  getTimeout: PT10S
  purgeTimeout: PT10S
  connectTimeout: PT2S
  connectRetryDelay: PT15S
  connectRetryTimes: 15
  circuitDuration: PT30S
  circuitThreshold: 1
httpContentStore:
  node: http://elasticsearch:9200
  indexGet: spider-search-httpcomcontent
  indexPurge: spider-active-httpcomcontent-upload
  getTimeout: PT15S
  purgeTimeout: PT15S
  connectTimeout: PT2S
  connectRetryDelay: PT15S
  connectRetryTimes: 15
  circuitDuration: PT15S
  circuitThreshold: 0.8
httpPersStore:
  node: http://elasticsearch:9200
  indexPurge: spider-search-httppers-upload
  indexGet: spider-search-httppers
  getTimeout: PT10S
  purgeTimeout: PT10S
  connectTimeout: PT2S
  connectRetryDelay: PT15S
  connectRetryTimes: 15
  circuitDuration: PT30S
  circuitThreshold: 1
httpComsCache:
  server: redis
  port: 6379
  db: 2
  timeOut: PT10S
  ttl: PT25S
  circuitDuration: PT30S
  circuitThreshold: 1
  compressed:  false
httpComsContentCache:
  server: redis
  port: 6379
  db: 12
  timeOut: PT10S
  ttl: PT25S
  circuitDuration: PT30S
  circuitThreshold: 1
  compressed:  false
httpPersCache:
  server: redis
  port: 6379
  db: 3
  timeOut: PT10S
  ttl: PT2M
  circuitDuration: PT30S
  circuitThreshold: 1
parsingJob:
  pollingDelay: PT5S
  jobRunners: 1
  uri: http://tcp-update/v1/parsing-jobs/http
  label: "/tcp-update/parsing-jobs/http"
  timeout: PT10S
  parsingDelay: PT10S
purgeJob:
  sizeLimitKB: 100
parsingStatusSynchro:
  delay: PT5S
  queue: statsSynchro
  maxBucketSize: 100
tcpSessions:
  uri: http://tcp-update/v1/tcp-sessions
  label: "/tcp-update/tcp-sessions"
  timeout: PT10S
  sizeLimitKB: 100
  compressed: true
packets:
  uri: http://pack-read/v1/packets/by/list/
  label: "/pack-read/packets/by/list"
  timeout: PT10S
  delayForRetry: PT10S
  delayWhenRetry: PT10S
packetsByIndex:
  uri: http://pack-read/v1/packets/of/tcpsession
  label: "/pack-read/packets/of/tcpsession"
  timeout: PT10S
whisps:
  get:
    uri: http://whisp/v1/whisperers/{id}
    label: "/whisp/whisperers"
    timeout: PT10S
  config:
    uri: http://whisp/v1/whisperers/{id}/config?view=full
    label: "/whisp/whisperers/config"
    timeout: PT10S

webRead

"@id": web-streams-read
"@type": ServerConfiguration
version: '0.1'
httpComStore:
  node: http://elasticsearch:9200
  indexGet: spider-search-httpcom
  getTimeout: PT15S
  connectTimeout: PT2S
  connectRetryDelay: PT15S
  connectRetryTimes: 15
  circuitDuration: PT15S
  circuitThreshold: 0.8
httpComContentStore:
  node: http://elasticsearch:9200
  indexGet: spider-search-httpcomcontent
  getTimeout: PT15S
  connectTimeout: PT2S
  connectRetryDelay: PT15S
  connectRetryTimes: 15
  circuitDuration: PT15S
  circuitThreshold: 0.8
httpPersStore:
  node: http://elasticsearch:9200
  indexGet: spider-search-httppers
  indexParsingStatus: spider-search-parsing-status-httppers
  getTimeout: PT10S
  connectTimeout: PT2S
  connectRetryDelay: PT15S
  connectRetryTimes: 15
  circuitDuration: PT30S
  circuitThreshold: 1
packets:
  uri: http://pack-read/v1/packets/payload/tcp/?force=true
  label: "/packets/payload/tcp"
  timeout: PT10S
searchRequest:
  sizeLimitKB: 100
whisps:
  uri: http://whisp/v1/whisperers/{id}/config?view=server
  label: "/whisp/whisperers/config"
  timeout: PT10S

webUpload

"@id": web-streams-upload
"@type": ServerConfiguration
version: '0.1'
httpComsCache:
  server: redis
  port: 6379
  db: 2
  timeOut: PT2S
  ttl: PT25S
  circuitDuration: PT30S
  circuitThreshold: 1
  compressed:  false
httpComsContentCache:
  server: redis
  port: 6379
  db: 12
  timeOut: PT2S
  ttl: PT25S
  circuitDuration: PT30S
  circuitThreshold: 1
  compressed:  false
upload:
  sizeLimitKB: 2000
whisps:
  uri: http://whisp/v1/whisperers
  label: "/whisp/whisperers"
  timeout: PT10S

webHttpComPoller

"@id": web-httpcom-poller
"@type": ServerConfiguration
version: '0.1'
logField: httpCom
itemStore:
  node: http://elasticsearch:9200
  useDataStoragePolicies: true
  useDistinctPolicyForUpload: true
  streamedDataStoragePolicies:
    - name: default
      indexSave: spider-active-httpcom-streaming-default
  uploadedDataStoragePolicies:
    - name: default
      indexSave: spider-active-httpcom-upload-default
  saveTimeout: PT5S
  connectTimeout: PT2S
  connectRetryDelay: PT15S
  connectRetryTimes: 15
  circuitDuration: PT30S
  circuitThreshold: 1
itemCache:
  server: redis
  port: 6379
  db: 2
  timeOut: PT2S
  ttl: PT45S
  circuitDuration: PT15S
  circuitThreshold: 1
  compressed:  false
polling:
  queue: httpComToSynchronize
  queueType: SORTED_SET
  scoreAttribute: _update
  size: 175
  jobRunners: 1
  delay: PT2S
  automaticESIds: false
  updateCache:
    keepInCache: false
    removeFromCacheCondition: {}
    noTTLCondition: {}
whisps:
  config:
    uri: http://whisp/v1/whisperers/{id}/config?view=full
    label: "/whisp/whisperers/config"
    timeout: PT10S

webHttpComContentPoller

"@id": web-httpcom-content-poller
"@type": ServerConfiguration
version: '0.1'
logField: httpCom
itemStore:
  node: http://elasticsearch:9200
  useDataStoragePolicies: true
  useDistinctPolicyForUpload: true
  streamedDataStoragePolicies:
    - name: default
      indexSave: spider-active-httpcomcontent-streaming-default
  uploadedDataStoragePolicies:
    - name: default
      indexSave: spider-active-httpcomcontent-upload-default
  saveTimeout: PT5S
  connectTimeout: PT2S
  connectRetryDelay: PT15S
  connectRetryTimes: 15
  circuitDuration: PT30S
  circuitThreshold: 1
itemCache:
  server: redis
  port: 6379
  db: 12
  timeOut: PT2S
  ttl: PT45S
  circuitDuration: PT15S
  circuitThreshold: 1
  compressed:  false
polling:
  queue: httpComContentToSynchronize
  queueType: SORTED_SET
  scoreAttribute: _update
  size: 175
  jobRunners: 1
  delay: PT2S
  automaticESIds: false
  updateCache:
    keepInCache: false
    removeFromCacheCondition: {}
    noTTLCondition: {}
whisps:
  config:
    uri: http://whisp/v1/whisperers/{id}/config?view=full
    label: "/whisp/whisperers/config"
    timeout: PT10S

webHttpPersPoller

"@id": web-httppers-poller
"@type": ServerConfiguration
version: '0.1'
logField: httpPers
itemStore:
  node: http://elasticsearch:9200
  useDataStoragePolicies: true
  useDistinctPolicyForUpload: true
  streamedDataStoragePolicies:
    - name: default
      indexSave: spider-active-httppers-streaming-default
  uploadedDataStoragePolicies:
    - name: default
      indexSave: spider-active-httppers-upload-default
  saveTimeout: PT5S
  connectTimeout: PT2S
  connectRetryDelay: PT15S
  connectRetryTimes: 15
  circuitDuration: PT30S
  circuitThreshold: 1
itemCache:
  server: redis
  port: 6379
  db: 3
  timeOut: PT2S
  ttl: PT45S
  longTtl: PT3M30S
  circuitDuration: PT15S
  circuitThreshold: 1
  compressed:  false
polling:
  queue: httpPersToSynchronize
  queueType: SORTED_SET_HMAP
  scoreAttribute: first
  size: 175
  jobRunners: 1
  delay: PT5S
  automaticESIds: false
  updateCache:
    keepInCache: true
    useLongTtlAsSafety: true
    removeFromCacheCondition:
      property: status
      values:
      - COMPLETED
      - ERROR
    noTTLCondition: {}
  saveInESConditions:
  - property: state
    values:
    - CLOSED
    - CLOSE_WAIT
    - ESTABLISHED
    - LAST_ACK
whisps:
  config:
    uri: http://whisp/v1/whisperers/{id}/config?view=full
    label: "/whisp/whisperers/config"
    timeout: PT10S