TCP sessions management
The following services are managing TCP sessions.
Below is their default configuration.
TcpWrite
"@id": tcp-streams-write
"@type": ServerConfiguration
version: '0.1'
tcpSessionStore:
node: http://elasticsearch:9200
indexGet: spider-search-tcpsession
getTimeout: PT10S
connectTimeout: PT2S
connectRetryDelay: PT15S
connectRetryTimes: 15
circuitDuration: PT30S
circuitThreshold: 1
tcpSessionCache:
server: redis
port: 6379
db: 0
timeOut: PT2S
ttl: PT2M
circuitDuration: PT15S
circuitThreshold: 1
compressed: false
tcpSessionsSizeLimitKB: 2000
pollingSize: 20
purgeJob:
sizeLimitKB: 100
registerParsedPackets:
uri: http://pack-update/v1/packets/parsed/
label: "/pack-update/packets/parsed"
timeout: PT10S
registerParsedPacketsV2:
uri: http://pack-update/v2/packets/parsed/
label: "/pack-update/v2/packets/parsed"
timeout: PT10S
parsingStatusSynchro:
delay: PT5S
queue: statsSynchro
maxBucketSize: 1000
whisps:
uri: http://whisp/v1/whisperers/{id}/config?view=server
label: "/whisp/whisperers/config"
timeout: PT10S
searchRequest:
sizeLimitKB: 2500
maxAllowedPacketsCount: 2000
delayBetweenParsingsOfSameSession: PT5S
TcpRead
"@id": tcp-streams-read
"@type": ServerConfiguration
version: '0.1'
tcpSessionStore:
node: http://elasticsearch:9200
indexGet: spider-search-tcpsession
indexParsingStatus: spider-search-parsing-status-tcpsession
getTimeout: PT30S
connectTimeout: PT2S
connectRetryDelay: PT15S
connectRetryTimes: 15
circuitDuration: PT30S
circuitThreshold: 1
compressed: false
tcpSessionsSizeLimitKB: 1000
patchSessionSizeLimitKB: 100
searchRequest:
sizeLimitKB: 100
whisps:
uri: http://whisp/v1/whisperers/{id}/config?view=server
label: "/whisp/whisperers/config"
timeout: PT10S
TcpUpdate
"@id": tcp-streams-update
"@type": ServerConfiguration
version: '0.1'
tcpSessionStore:
node: http://elasticsearch:9200
indexGet: spider-search-tcpsession
indexPurge: spider-search-tcpsession-upload
getTimeout: PT10S
purgeTimeout: PT10S
connectTimeout: PT2S
connectRetryDelay: PT15S
connectRetryTimes: 15
circuitDuration: PT30S
circuitThreshold: 1
tcpSessionCache:
server: redis
port: 6379
db: 0
timeOut: PT2S
ttl: PT2M
longTtl: PT3M30S
circuitDuration: PT15S
circuitThreshold: 1
tcpSessionsSizeLimitKB: 1000
patchSessionSizeLimitKB: 100
pollingSize: 20
purgeJob:
sizeLimitKB: 100
whisps:
get:
uri: http://whisp/v1/whisperers/{id}
label: "/whisp/whisperers"
timeout: PT10S
config:
uri: http://whisp/v1/whisperers/{id}/config?view=server
label: "/whisp/whisperers/config"
timeout: PT10S
registerParsedPackets:
uri: http://pack-update/v1/packets/parsed/
label: "/pack-update/packets/parsed"
timeout: PT10S
registerParsedPacketsV2:
uri: http://pack-update/v2/packets/parsed/
label: "/pack-update/v2/packets/parsed"
timeout: PT10S
parsingStatusSynchro:
delay: PT5S
queue: statsSynchro
maxBucketSize: 1000
searchRequest:
sizeLimitKB: 2500
maxAllowedWaitingAndParsingToResumeAtStart: 1000
apis:
postParsingJob:
compressed: true
TcpPoller
"@id": tcp-poller
"@type": ServerConfiguration
version: '0.1'
logField: tcpSession
itemStore:
node: http://elasticsearch:9200
useDataStoragePolicies: true
useDistinctPolicyForUpload: true
streamedDataStoragePolicies:
- name: default
indexSave: spider-active-tcpsession-streaming-default
uploadedDataStoragePolicies:
- name: default
indexSave: spider-active-tcpsession-upload-default
saveTimeout: PT5S
connectTimeout: PT2S
connectRetryDelay: PT15S
connectRetryTimes: 15
circuitDuration: PT30S
circuitThreshold: 1
itemCache:
server: redis
port: 6379
db: 0
timeOut: PT2S
ttl: PT2M
longTtl: PT3M30S
circuitDuration: PT15S
circuitThreshold: 1
compressed: false
polling:
queue: sessionsToSynchronize
queueType: SESSIONS
scoreAttribute: first
size: 175
jobRunners: 1
delay: PT5S
automaticESIds: false
updateCache:
keepInCache: true
saveInESConditions:
- property: state
values:
- CLOSED
- CLOSE_WAIT
- ESTABLISHED
- LAST_ACK
- property: parsers.http.status
values:
- WAITING
- N/A
- COMPLETED
- NO_DATA
- ERROR
- PARSED
- WARNING
- INCOMPLETE
whisps:
config:
uri: http://whisp/v1/whisperers/{id}/config?view=full
label: "/whisp/whisperers/config"
timeout: PT10S