Skip to main content

PostgreSQL Parsing

The following services are managing PostgreSQL communications parsing.
Below is their default configuration.

pgParser​

---
"@id": pg-parser
"@type": ServerConfiguration
version: '0.1'
pgComStore:
  node: http://elasticsearch:9200
  indexPurge: spider-search-pgcom-upload
  indexGet: spider-search-pgcom
  getTimeout: PT10S
  purgeTimeout: PT10S
  connectTimeout: PT2S
  connectRetryDelay: PT15S
  connectRetryTimes: 15
  circuitDuration: PT30S
  circuitThreshold: 1
pgComContentStore:
  node: http://elasticsearch:9200
  indexGet: spider-search-pgcomcontent
  indexPurge: spider-active-pgcomcontent-upload-default
  getTimeout: PT15S
  purgeTimeout: PT15S
  connectTimeout: PT2S
  connectRetryDelay: PT15S
  connectRetryTimes: 15
  circuitDuration: PT15S
  circuitThreshold: 0.8
pgParsingStore:
  node: http://elasticsearch:9200
  indexPurge: spider-search-pgparsing-upload
  indexGet: spider-search-pgparsing
  getTimeout: PT10S
  purgeTimeout: PT10S
  connectTimeout: PT2S
  connectRetryDelay: PT15S
  connectRetryTimes: 15
  circuitDuration: PT30S
  circuitThreshold: 1
pgComsCache:
  server: redis
  port: 6379
  db: 2
  timeOut: PT10S
  ttl: PT25S
  circuitDuration: PT30S
  circuitThreshold: 1
  compressed: false
pgComsContentCache:
  server: redis
  port: 6379
  db: 12
  timeOut: PT10S
  ttl: PT25S
  circuitDuration: PT30S
  circuitThreshold: 1
  compressed: false
pgParsingCache:
  server: redis
  port: 6379
  db: 3
  timeOut: PT10S
  ttl: PT2M
  circuitDuration: PT30S
  circuitThreshold: 1
  compressed: false
parsingJob:
  pollingDelay: PT5S
  jobRunners: 1
  uri: http://tcp-update/v1/parsing-jobs/postgresql
  label: "/tcp-update/parsing-jobs/postgresql"
  timeout: PT10S
  parsingDelay: PT10S
purgeJob:
  sizeLimitKB: 100
parsingStatusSynchro:
  delay: PT5S
  queue: statsSynchro
  maxBucketSize: 100
tcpSessions:
  uri: http://tcp-update/v1/tcp-sessions
  label: "/tcp-update/tcp-sessions"
  timeout: PT10S
  sizeLimitKB: 100
  compressed: true
packets:
  uri: http://pack-read/v1/packets/by/list/
  label: "/pack-read/packets/by/list"
  timeout: PT10S
  delayForRetry: PT10S
  delayWhenRetry: PT10S
packetsByIndex:
  uri: http://pack-read/v1/packets/of/tcpsession
  label: "/pack-read/packets/of/tcpsession"
  timeout: PT10S
whisps:
  get:
    uri: http://whisp/v1/whisperers/{id}
    label: "/whisp/whisperers"
    timeout: PT10S
  config:
    uri: http://whisp/v1/whisperers/{id}/config?view=full
    label: "/whisp/whisperers/config"
    timeout: PT10S

pgRead​

---
"@id": pg-read
"@type": ServerConfiguration
version: '0.1'
pgComStore:
  node: http://elasticsearch:9200
  indexGet: spider-search-pgcom
  getTimeout: PT15S
  connectTimeout: PT2S
  connectRetryDelay: PT15S
  connectRetryTimes: 15
  circuitDuration: PT15S
  circuitThreshold: 0.8
pgComContentStore:
  node: http://elasticsearch:9200
  indexGet: spider-search-pgcomcontent
  getTimeout: PT15S
  connectTimeout: PT2S
  connectRetryDelay: PT15S
  connectRetryTimes: 15
  circuitDuration: PT15S
  circuitThreshold: 0.8
pgParsingStore:
  node: http://elasticsearch:9200
  indexGet: spider-search-pgparsing
  indexParsingStatus: spider-search-parsing-status-pgparsing
  getTimeout: PT10S
  connectTimeout: PT2S
  connectRetryDelay: PT15S
  connectRetryTimes: 15
  circuitDuration: PT30S
  circuitThreshold: 1
packets:
  uri: http://pack-read/v1/packets/payload/tcp/?force=true
  label: "/packets/payload/tcp"
  timeout: PT10S
searchRequest:
  sizeLimitKB: 100
whisps:
  uri: http://whisp/v1/whisperers/{id}/config?view=server
  label: "/whisp/whisperers/config"
  timeout: PT10S

pgComPoller​

---
"@id": pg-com-poller
"@type": ServerConfiguration
version: '0.1'
logField: pgCom
itemStore:
  node: http://elasticsearch:9200
  useDataStoragePolicies: true
  useDistinctPolicyForUpload: true
  streamedDataStoragePolicies:
  - name: default
    indexSave: spider-active-pgcom-streaming-default
  uploadedDataStoragePolicies:
  - name: default
    indexSave: spider-active-pgcom-upload-default
  saveTimeout: PT5S
  connectTimeout: PT2S
  connectRetryDelay: PT15S
  connectRetryTimes: 15
  circuitDuration: PT30S
  circuitThreshold: 1
itemCache:
  server: redis
  port: 6379
  db: 2
  timeOut: PT2S
  ttl: PT45S
  circuitDuration: PT15S
  circuitThreshold: 1
  compressed: false
polling:
  queue: pgComToSynchronizeList
  queueType: LIST
  scoreAttribute: _update
  size: 175
  jobRunners: 1
  delay: PT2S
  automaticESIds: false
  updateCache:
    suffixIdWithScore: true
    keepInCache: false
    removeFromCacheCondition: {}
    noTTLCondition: {}
whisps:
  config:
    uri: http://whisp/v1/whisperers/{id}/config?view=full
    label: "/whisp/whisperers/config"
    timeout: PT10S

pgComContentPoller​

---
"@id": pg-com-content-poller
"@type": ServerConfiguration
version: '0.1'
logField: pgCom
itemStore:
  node: http://elasticsearch:9200
  useDataStoragePolicies: true
  useDistinctPolicyForUpload: true
  streamedDataStoragePolicies:
  - name: default
    indexSave: spider-active-pgcomcontent-streaming-default
  uploadedDataStoragePolicies:
  - name: default
    indexSave: spider-active-pgcomcontent-upload-default
  saveTimeout: PT5S
  connectTimeout: PT2S
  connectRetryDelay: PT15S
  connectRetryTimes: 15
  circuitDuration: PT30S
  circuitThreshold: 1
itemCache:
  server: redis
  port: 6379
  db: 12
  timeOut: PT2S
  ttl: PT45S
  circuitDuration: PT15S
  circuitThreshold: 1
  compressed: false
polling:
  queue: pgComContentToSynchronizeList
  queueType: LIST
  scoreAttribute: _update
  size: 175
  jobRunners: 1
  delay: PT2S
  automaticESIds: false
  updateCache:
    suffixIdWithScore: true
    keepInCache: false
    removeFromCacheCondition: {}
    noTTLCondition: {}
whisps:
  config:
    uri: http://whisp/v1/whisperers/{id}/config?view=full
    label: "/whisp/whisperers/config"
    timeout: PT10S

pgParsingPoller​

{
  "@id": "pg-parsing-poller",
  "@type": "ServerConfiguration",
  "version": "0.1",
  "logField": "pgParsing",
  "itemStore": {
    "node": "http://elasticsearch:9200",
    "useDataStoragePolicies": true,
    "useDistinctPolicyForUpload": true,
    "streamedDataStoragePolicies": [{
      "name": "default",
      "indexSave": "spider-active-pgparsing-streaming-default"
    }],
    "uploadedDataStoragePolicies": [{
      "name": "default",
      "indexSave": "spider-active-pgparsing-upload-default"
    }],
    "saveTimeout": "PT5S",
    "connectTimeout" : "PT2S",
    "connectRetryDelay" : "PT15S",
    "connectRetryTimes" : 15,
    "circuitDuration": "PT30S",
    "circuitThreshold": 1
  },
  "itemCache": {
    "server": "redis",
    "port": 6379,
    "db": 3,
    "timeOut": "PT2S",
    "ttl": "PT45S",
    "longTtl": "PT3M30S",
    "circuitDuration": "PT15S",
    "circuitThreshold": 1,
    "compressed":  false
  },
  "polling":{
    "queue": "pgParsingToSynchronize",
    "queueType": "SORTED_SET_HMAP",
    "scoreAttribute": "first",
    "size": 175,
    "jobRunners": 1,
    "delay": "PT5S",
    "automaticESIds": false,
    "updateCache": {
      "keepInCache": true,
      "useLongTtlAsSafety": true,
      "removeFromCacheCondition": {
        "property": "status",
        "values": ["COMPLETED", "ERROR"]
      },
      "noTTLCondition":{}
    },
    "saveInESConditions": [{
      "property": "state",
      "values": ["CLOSED","CLOSE_WAIT","ESTABLISHED","LAST_ACK"]
    }]
  },
  "whisps": {
    "config": {
      "uri": "http://whisp/v1/whisperers/{id}/config?view=full",
      "label": "/whisp/whisperers/config",
      "timeout": "PT10S"
    }
  }
}