Http Parsing
The following services are managing HTTP communications parsing.
Below is their default configuration.
webWrite
"@id": web-streams-write
"@type": ServerConfiguration
version: '0.1'
httpComStore:
node: http://elasticsearch:9200
indexPurge: spider-search-httpcom-upload
indexGet: spider-search-httpcom
getTimeout: PT10S
purgeTimeout: PT10S
connectTimeout: PT2S
connectRetryDelay: PT15S
connectRetryTimes: 15
circuitDuration: PT30S
circuitThreshold: 1
httpContentStore:
node: http://elasticsearch:9200
indexGet: spider-search-httpcomcontent
indexPurge: spider-active-httpcomcontent-upload
getTimeout: PT15S
purgeTimeout: PT15S
connectTimeout: PT2S
connectRetryDelay: PT15S
connectRetryTimes: 15
circuitDuration: PT15S
circuitThreshold: 0.8
httpPersStore:
node: http://elasticsearch:9200
indexPurge: spider-search-httppers-upload
indexGet: spider-search-httppers
getTimeout: PT10S
purgeTimeout: PT10S
connectTimeout: PT2S
connectRetryDelay: PT15S
connectRetryTimes: 15
circuitDuration: PT30S
circuitThreshold: 1
httpComsCache:
server: redis
port: 6379
db: 2
timeOut: PT10S
ttl: PT25S
circuitDuration: PT30S
circuitThreshold: 1
compressed: false
httpComsContentCache:
server: redis
port: 6379
db: 12
timeOut: PT10S
ttl: PT25S
circuitDuration: PT30S
circuitThreshold: 1
compressed: false
httpPersCache:
server: redis
port: 6379
db: 3
timeOut: PT10S
ttl: PT2M
circuitDuration: PT30S
circuitThreshold: 1
parsingJob:
pollingDelay: PT5S
jobRunners: 1
uri: http://tcp-update/v1/parsing-jobs/http
label: "/tcp-update/parsing-jobs/http"
timeout: PT10S
parsingDelay: PT10S
parsingWarningsJob:
pollingDelay: PT15S
jobRunners: 0
uri: http://tcp-update/v1/parsing-jobs/http/warnings
label: "/tcp-update/parsing-jobs/http/warnings"
timeout: PT10S
parsingDelay: PT3M
purgeJob:
sizeLimitKB: 100
parsingStatusSynchro:
delay: PT5S
queue: statsSynchro
maxBucketSize: 100
tcpSessions:
uri: http://tcp-update/v1/tcp-sessions
label: "/tcp-update/tcp-sessions"
timeout: PT10S
sizeLimitKB: 100
compressed: true
packets:
uri: http://pack-read/v1/packets/by/list/
label: "/pack-read/packets/by/list"
timeout: PT10S
delayForRetry: PT10S
delayWhenRetry: PT10S
packetsByIndex:
uri: http://pack-read/v1/packets/of/tcpsession
label: "/pack-read/packets/of/tcpsession"
timeout: PT10S
whisps:
get:
uri: http://whisp/v1/whisperers/{id}
label: "/whisp/whisperers"
timeout: PT10S
config:
uri: http://whisp/v1/whisperers/{id}/config?view=full
label: "/whisp/whisperers/config"
timeout: PT10S
webWriteWarnings
"@id": web-streams-write-warnings
"@type": ServerConfiguration
version: '0.1'
httpComStore:
node: http://elasticsearch:9200
indexPurge: spider-search-httpcom-upload
indexGet: spider-search-httpcom
getTimeout: PT10S
purgeTimeout: PT10S
connectTimeout: PT2S
connectRetryDelay: PT15S
connectRetryTimes: 15
circuitDuration: PT30S
circuitThreshold: 1
httpContentStore:
node: http://elasticsearch:9200
indexGet: spider-search-httpcomcontent
indexPurge: spider-active-httpcomcontent-upload
getTimeout: PT15S
purgeTimeout: PT15S
connectTimeout: PT2S
connectRetryDelay: PT15S
connectRetryTimes: 15
circuitDuration: PT15S
circuitThreshold: 0.8
httpPersStore:
node: http://elasticsearch:9200
indexPurge: spider-search-httppers-upload
indexGet: spider-search-httppers
getTimeout: PT10S
purgeTimeout: PT10S
connectTimeout: PT2S
connectRetryDelay: PT15S
connectRetryTimes: 15
circuitDuration: PT30S
circuitThreshold: 1
httpComsCache:
server: redis
port: 6379
db: 2
timeOut: PT2S
ttl: PT5S
circuitDuration: PT30S
circuitThreshold: 1
compressed: false
httpComsContentCache:
server: redis
port: 6379
db: 12
timeOut: PT2S
ttl: PT5S
circuitDuration: PT30S
circuitThreshold: 1
compressed: false
httpPersCache:
server: redis
port: 6379
db: 3
timeOut: PT2S
ttl: PT2M
circuitDuration: PT30S
circuitThreshold: 1
parsingJob:
pollingDelay: PT5S
jobRunners: 0
uri: http://tcp-update/v1/parsing-jobs/http
label: "/tcp-update/parsing-jobs/http"
timeout: PT10S
parsingDelay: PT10S
parsingWarningsJob:
pollingDelay: PT15S
jobRunners: 1
uri: http://tcp-update/v1/parsing-jobs/http/warnings
label: "/tcp-update/parsing-jobs/http/warnings"
timeout: PT10S
parsingDelay: PT3M
purgeJob:
sizeLimitKB: 100
parsingStatusSynchro:
delay: PT5S
queue: statsSynchro
maxBucketSize: 100
tcpSessions:
uri: http://tcp-update/v1/tcp-sessions
label: "/tcp-update/tcp-sessions"
timeout: PT10S
sizeLimitKB: 100
compressed: true
packets:
uri: http://pack-read/v1/packets/by/list/
label: "/pack-read/packets/by/list"
timeout: PT10S
delayForRetry: PT10S
delayWhenRetry: PT10S
packetsByIndex:
uri: http://pack-read/v1/packets/of/tcpsession
label: "/pack-read/packets/of/tcpsession"
timeout: PT10S
whisps:
get:
uri: http://whisp/v1/whisperers/{id}
label: "/whisp/whisperers"
timeout: PT10S
config:
uri: http://whisp/v1/whisperers/{id}/config?view=full
label: "/whisp/whisperers/config"
timeout: PT10S
webRead
"@id": web-streams-read
"@type": ServerConfiguration
version: '0.1'
httpComStore:
node: http://elasticsearch:9200
indexGet: spider-search-httpcom
getTimeout: PT15S
connectTimeout: PT2S
connectRetryDelay: PT15S
connectRetryTimes: 15
circuitDuration: PT15S
circuitThreshold: 0.8
httpComContentStore:
node: http://elasticsearch:9200
indexGet: spider-search-httpcomcontent
getTimeout: PT15S
connectTimeout: PT2S
connectRetryDelay: PT15S
connectRetryTimes: 15
circuitDuration: PT15S
circuitThreshold: 0.8
httpPersStore:
node: http://elasticsearch:9200
indexGet: spider-search-httppers
indexParsingStatus: spider-search-parsing-status-httppers
getTimeout: PT10S
connectTimeout: PT2S
connectRetryDelay: PT15S
connectRetryTimes: 15
circuitDuration: PT30S
circuitThreshold: 1
packets:
uri: http://pack-read/v1/packets/payload/tcp/?force=true
label: "/packets/payload/tcp"
timeout: PT10S
searchRequest:
sizeLimitKB: 100
whisps:
uri: http://whisp/v1/whisperers/{id}/config?view=server
label: "/whisp/whisperers/config"
timeout: PT10S
webUpload
"@id": web-streams-upload
"@type": ServerConfiguration
version: '0.1'
httpComsCache:
server: redis
port: 6379
db: 2
timeOut: PT2S
ttl: PT25S
circuitDuration: PT30S
circuitThreshold: 1
compressed: false
httpComsContentCache:
server: redis
port: 6379
db: 12
timeOut: PT2S
ttl: PT25S
circuitDuration: PT30S
circuitThreshold: 1
compressed: false
upload:
sizeLimitKB: 2000
whisps:
uri: http://whisp/v1/whisperers
label: "/whisp/whisperers"
timeout: PT10S
webHttpComPoller
"@id": web-httpcom-poller
"@type": ServerConfiguration
version: '0.1'
logField: httpCom
itemStore:
node: http://elasticsearch:9200
useDataStoragePolicies: true
useDistinctPolicyForUpload: true
streamedDataStoragePolicies:
- name: default
indexSave: spider-active-httpcom-streaming-default
uploadedDataStoragePolicies:
- name: default
indexSave: spider-active-httpcom-upload-default
saveTimeout: PT5S
connectTimeout: PT2S
connectRetryDelay: PT15S
connectRetryTimes: 15
circuitDuration: PT30S
circuitThreshold: 1
itemCache:
server: redis
port: 6379
db: 2
timeOut: PT2S
ttl: PT45S
circuitDuration: PT15S
circuitThreshold: 1
compressed: false
polling:
queue: httpComToSynchronize
queueType: SORTED_SET
scoreAttribute: _update
size: 175
jobRunners: 1
delay: PT2S
automaticESIds: false
updateCache:
keepInCache: false
removeFromCacheCondition: {}
noTTLCondition: {}
whisps:
config:
uri: http://whisp/v1/whisperers/{id}/config?view=full
label: "/whisp/whisperers/config"
timeout: PT10S
webHttpComContentPoller
"@id": web-httpcom-content-poller
"@type": ServerConfiguration
version: '0.1'
logField: httpCom
itemStore:
node: http://elasticsearch:9200
useDataStoragePolicies: true
useDistinctPolicyForUpload: true
streamedDataStoragePolicies:
- name: default
indexSave: spider-active-httpcomcontent-streaming-default
uploadedDataStoragePolicies:
- name: default
indexSave: spider-active-httpcomcontent-upload-default
saveTimeout: PT5S
connectTimeout: PT2S
connectRetryDelay: PT15S
connectRetryTimes: 15
circuitDuration: PT30S
circuitThreshold: 1
itemCache:
server: redis
port: 6379
db: 12
timeOut: PT2S
ttl: PT45S
circuitDuration: PT15S
circuitThreshold: 1
compressed: false
polling:
queue: httpComContentToSynchronize
queueType: SORTED_SET
scoreAttribute: _update
size: 175
jobRunners: 1
delay: PT2S
automaticESIds: false
updateCache:
keepInCache: false
removeFromCacheCondition: {}
noTTLCondition: {}
whisps:
config:
uri: http://whisp/v1/whisperers/{id}/config?view=full
label: "/whisp/whisperers/config"
timeout: PT10S
webHttpPersPoller
"@id": web-httppers-poller
"@type": ServerConfiguration
version: '0.1'
logField: httpPers
itemStore:
node: http://elasticsearch:9200
useDataStoragePolicies: true
useDistinctPolicyForUpload: true
streamedDataStoragePolicies:
- name: default
indexSave: spider-active-httppers-streaming-default
uploadedDataStoragePolicies:
- name: default
indexSave: spider-active-httppers-upload-default
saveTimeout: PT5S
connectTimeout: PT2S
connectRetryDelay: PT15S
connectRetryTimes: 15
circuitDuration: PT30S
circuitThreshold: 1
itemCache:
server: redis
port: 6379
db: 3
timeOut: PT2S
ttl: PT45S
longTtl: PT3M30S
circuitDuration: PT15S
circuitThreshold: 1
compressed: false
polling:
queue: httpPersToSynchronize
queueType: SORTED_SET_HMAP
scoreAttribute: first
size: 175
jobRunners: 1
delay: PT5S
automaticESIds: false
updateCache:
keepInCache: true
useLongTtlAsSafety: true
removeFromCacheCondition:
property: status
values:
- COMPLETED
- ERROR
noTTLCondition: {}
saveInESConditions:
- property: state
values:
- CLOSED
- CLOSE_WAIT
- ESTABLISHED
- LAST_ACK
whisps:
config:
uri: http://whisp/v1/whisperers/{id}/config?view=full
label: "/whisp/whisperers/config"
timeout: PT10S