IntelMQ¶

neither store nor cache

# To start the docker daemon
systemctl start docker.service
# To enable the docker daemon for the future
systemctl enable docker.service

git clone https://github.com/certat/intelmq-docker.git --recursive
cd intelmq-docker
sudo docker-compose pull
sudo docker-compose up

apt install python3-pip python3-dnspython python3-psutil python3-redis python3-requests python3-termstyle python3-tz python3-dateutil redis-server bash-completion jq
# optional dependencies
apt install python3-pymongo python3-psycopg2

yum install epel-release
yum install python36 python36-dns python36-requests python3-setuptools redis bash-completion jq
yum install gcc gcc-c++ python36-devel
# optional dependencies
yum install python3-psycopg2

dnf install epel-release
dnf install python3-dateutil python3-dns python3-pip python3-psutil python3-redis python3-requests redis bash-completion jq
# optional dependencies
dnf install python3-psycopg2 python3-pymongo

zypper install python3-dateutil python3-dnspython python3-psutil python3-redis python3-requests python3-python-termstyle redis bash-completion jq
# optional dependencies
zypper in python3-psycopg2 python3-pymongo

sudo -i
pip3 install intelmq
useradd -d /opt/intelmq -U -s /bin/bash intelmq
sudo intelmqsetup

sudo docker pull redis:latest

sudo docker pull certat/intelmq-full:latest

sudo docker pull certat/intelmq-nginx:latest

sudo docker network create intelmq-internal

sudo docker run -v ~/intelmq/example_config/redis/redis.conf:/redis.conf \
                --network intelmq-internal \
                --name redis \
                redis:latest

sudo docker run --network intelmq-internal \
                --name nginx \
                certat/intelmq-nginx:latest

sudo docker run -e INTELMQ_IS_DOCKER="true" \
                -e INTELMQ_SOURCE_PIPELINE_BROKER: "redis" \
                -e INTELMQ_PIPELINE_BROKER: "redis" \
                -e INTELMQ_DESTIONATION_PIPELINE_BROKER: "redis" \
                -e INTELMQ_PIPELINE_HOST: redis \
                -e INTELMQ_SOURCE_PIPELINE_HOST: redis \
                -e INTELMQ_DESTINATION_PIPELINE_HOST: redis \
                -e INTELMQ_REDIS_CACHE_HOST: redis \
                -v $(pwd)/example_config/intelmq/etc/:/etc/intelmq/etc/ \
                -v $(pwd)/example_config/intelmq-api/config.json:/etc/intelmq/api-config.json \
                -v $(pwd)/intelmq_logs:/etc/intelmq/var/log \
                -v $(pwd)/intelmq_output:/etc/intelmq/var/lib/bots \
                -v ~/intelmq/lib:/etc/intelmq/var/lib \
                --network intelmq-internal \
                --name intelmq \
                certat/intelmq-full:latest

-e INTELMQ_API_USER: "your username"
-e INTELMQ_API_PASS: "your password"

sudo cp -R /opt/intelmq /opt/intelmq-backup

intelmqctl check
intelmqctl list queues -q

docker pull certat/intelmq-full:latest

docker pull certat/intelmq-nginx:latest

docker-compose pull

docker inspect --format '{{ index .Config.Labels "org.opencontainers.image.version" }}' intelmq-full:latest

docker-compose down

docker ps | grep certat

docker stop CONTAINER_ID

pip install -U --no-deps intelmq
sudo intelmqsetup

pip install .
sudo intelmqsetup

intelmqctl upgrade-config
intelmqctl check

intelmqctl start

systemctl enable redis.service
systemctl start redis.service

apt install supervisor python-pip
pip install supervisor_twiddler

[rpcinterface:twiddler]
supervisor.rpcinterface_factory=supervisor_twiddler.rpcinterface:make_twiddler_rpcinterface

[group:intelmq]

process_manager: supervisor

<bot ID>:
  group: <bot type (Collector, Parser, Expert, Output)>
  name: <human-readable bot name>
  module: <bot code (python module)>
  description: <generic description of the bot>
  parameters:
    <parameter 1>: <value 1>
    <parameter 2>: <value 2>
    <parameter 3>: <value 3>

blocklistde-apache-collector:
  group: Collector
  name: Blocklist.de Apache List
  module: intelmq.bots.collectors.http.collector_http
  description: Blocklist.de Apache Collector fetches all IP addresses which have been reported within the last 48 hours as having run attacks on the service Apache, Apache-DDOS, RFI-Attacks.
  parameters:
    http_url: https://lists.blocklist.de/lists/apache.txt
    name: Blocklist.de Apache
    rate_limit: 3600

blocklistde-apache-collector:
  group: Collector
  name: Blocklist.de Apache List
  module: intelmq.bots.collectors.http.collector_http
  description: Blocklist.de Apache Collector fetches all IP addresses which have been reported within the last 48 hours as having run attacks on the service Apache, Apache-DDOS, RFI-Attacks.
  enabled: false
  parameters:
    http_url: https://lists.blocklist.de/lists/apache.txt
    name: Blocklist.de Apache
    rate_limit: 3600

source-queue: example-bot-queue

destination-queues:
  _default:
    - <first destination pipeline name>
    - <second destination pipeline name>
  _on_error:
    - <optional first destination pipeline name in case of errors>
    - <optional second destination pipeline name in case of errors>
  other-path:
    - <second destination pipeline name>
    - <third destination pipeline name>

{
    "<message type>": {
        "<field 1>": {
            "description": "<field 1 description>",
            "type": "<field value type>"
        },
        "<field 2>": {
            "description": "<field 2 description>",
            "type": "<field value type>"
        }
    },
}

{
    "event": {
        "destination.asn": {
            "description": "The autonomous system number from which originated the connection.",
            "type": "Integer"
        },
        "destination.geolocation.cc": {
            "description": "Country-Code according to ISO3166-1 alpha-2 for the destination IP.",
            "regex": "^[a-zA-Z0-9]{2}$",
            "type": "String"
        },
    },
}

blocklistde-apache-collector:
  name: Generic URL Fetcher
  group: Collector
  module: intelmq.bots.collectors.http.collector_http
  description: All IP addresses which have been reported within the last 48 hours as having run attacks on the service Apache, Apache-DDOS, RFI-Attacks.
  enabled: false
  run_mode: scheduled
  parameters:
    feed: Blocklist.de Apache
    provider: Blocklist.de
    http_url: https://lists.blocklist.de/lists/apache.txt
    ssl_client_certificate: null

0 0 * * * intelmqctl start blocklistde-apache-collector

blocklistde-apache-parser:
  name: Blocklist.de Parser
  group: Parser
  module: intelmq.bots.parsers.blocklistde.parser
  description: Blocklist.DE Parser is the bot responsible to parse the report and sanitize the information.
  enabled: false
  run_mode: continuous
  parameters: ...

intelmqctl start blocklistde-apache-parser

redis-cli FLUSHDB
redis-cli FLUSHALL

intelmqdump -h
usage:
    intelmqdump [botid]
    intelmqdump [-h|--help]

intelmqdump can inspect dumped messages, show, delete or reinject them into
the pipeline. It's an interactive tool, directly start it to get a list of
available dumps or call it with a known bot id as parameter.

positional arguments:
  botid       botid to inspect dumps of

optional arguments:
  -h, --help  show this help message and exit
  --truncate TRUNCATE, -t TRUNCATE
                        Truncate raw-data with more characters than given. 0 for no truncating. Default: 1000.

Interactive actions after a file has been selected:
- r, Recover by IDs
  > r id{,id} [queue name]
  > r 3,4,6
  > r 3,7,90 modify-expert-queue
  The messages identified by a consecutive numbering will be stored in the
  original queue or the given one and removed from the file.
- a, Recover all
  > a [queue name]
  > a
  > a modify-expert-queue
  All messages in the opened file will be recovered to the stored or given
  queue and removed from the file.
- d, Delete entries by IDs
  > d id{,id}
  > d 3,5
  The entries will be deleted from the dump file.
- d, Delete file
  > d
  Delete the opened file as a whole.
- s, Show by IDs
  > s id{,id}
  > s 0,4,5
  Show the selected IP in a readable format. It's still a raw format from
  repr, but with newlines for message and traceback.
- e, Edit by ID
  > e id
  > e 0
  > e 1,2
  Opens an editor (by calling `sensible-editor`) on the message. The modified message is then saved in the dump.
- q, Quit
  > q

$ intelmqdump
 id: name (bot id)                    content
  0: alienvault-otx-parser            1 dumps
  1: cymru-whois-expert               8 dumps
  2: deduplicator-expert              2 dumps
  3: dragon-research-group-ssh-parser 2 dumps
  4: file-output2                     1 dumps
  5: fraunhofer-dga-parser            1 dumps
  6: spamhaus-cert-parser             4 dumps
  7: test-bot                         2 dumps
Which dump file to process (id or name)? 3
Processing dragon-research-group-ssh-parser: 2 dumps
  0: 2015-09-03T13:13:22.159014 InvalidValue: invalid value u'NA' (<type 'unicode'>) for key u'source.asn'
  1: 2015-09-01T14:40:20.973743 InvalidValue: invalid value u'NA' (<type 'unicode'>) for key u'source.asn'
(r)ecover by ids, recover (a)ll, delete (e)ntries, (d)elete file, (s)how by ids, (q)uit, edit id (v)? d
Deleted file /opt/intelmq/var/log/dragon-research-group-ssh-parser.dump

tail -f /opt/intelmq/var/log/*.log
tail -f /var/log/intelmq/*.log

pip3 uninstall intelmq
rm -r /opt/intelmq

bot-id:
  parameters:
      runtime parameters...
  initialization parameters...

abusech-feodo-domains-collector:
  parameters:
    provider: Abuse.ch
    name: Abuse.ch Feodo Domains
    http_url: http://example.org/feodo-domains.txt
  name: Generic URL Fetcher
  group: Collector
  module: intelmq.bots.collectors.http.collector_http
  description: collect report messages from remote hosts using http protocol
  enabled: true
  run_mode: scheduled

pip3 install -r intelmq/bots/collectors/alienvault_otx/REQUIREMENTS.txt

pip3 install -r intelmq/bots/collectors/blueliv/REQUIREMENTS.txt

pip3 install -r intelmq/bots/collectors/eset/REQUIREMENTS.txt

pip3 install -r intelmq/bots/collectors/stomp/REQUIREMENTS.txt

curl -X POST http://localhost:5000/intelmq/push -H 'Content-Type: application/json' --data '{"source.ip": "127.0.0.101", "classification.type": "system-compromise"}'

defaults_fields:
  classification.type: c2-server
  protocol.transport: tcp

"columns": [
     "",
     "source.fqdn",
     "extra.http_host_header",
     "__IGNORE__"
],

"columns": ",source.fqdn,extra.http_host_header,"

"columns": "source.url|source.fqdn|source.ip"

# Host,Path
example.com,/foo/
example.net,/bar/

{"source.url": "http://{0}{1}"}

http://example.com/foo/
http://example.net/bar/

{
    "columns": [ "source.ip", "source.url", "extra.tags"],
    "data_type": "{\"extra.tags\":\"json\"}"
}

{
     "filter_text": "ipset add ",
     "filter_type": "whitelist",
     "columns": [ "__IGNORE__", "__IGNORE__", "__IGNORE__", "source.ip"]
}

{"malware_download": "malware-distribution"}

"columns": [
     "",
     "source.fqdn",
     "extra.http_host_header",
     "__IGNORE__"
],

"columns": ",source.fqdn,extra.http_host_header,"

"columns": "source.url|source.fqdn|source.ip"

"ignore_values": [
     "",
     "unknown",
     "Not listed",
 ],

"ignore_values": ",unknown,Not listed,"

"columns": [
     "source.url",
     "malware.name",
     "extra.SBL",
],
"ignore_values": [
     "",
     "unknown",
     "Not listed",
],

"attribute_name": "class",
"attribute_value": "details"

"split_column": "source.fqdn",
"split_separator": " ",
"split_index": 1,

"keys": {
    "srcip": "source.ip",
    "dstip": "destination.ip"
}

key="long value" key2="other value"

srcip=192.0.2.1 srcmac="00:00:5e:00:17:17"

pip3 install querycontacts

crontab -e

0,30 * * * *   intelmqctl reload my-aggregate-bot

pip3 install pyasn

intelmq.bots.experts.asn_lookup.expert --update-database

com
at
gv.at

*.example.com

!www.example.com

intelmq.bots.experts.domain_suffix.expert --update-database

parameters:
  redis_cache_db: 6
  redis_cache_host: "127.0.0.1"
  redis_cache_password: null
  redis_cache_port: 6379
  redis_cache_ttl: 86400
  filter_type: "whitelist"
  filter_keys: "source.ip,destination.ip"

parameters:
  redis_cache_db: 6
  redis_cache_host: "127.0.0.1"
  redis_cache_password: null
  redis_cache_port: 6379
  redis_cache_ttl: 86400
  filter_type: "blacklist"
  filter_keys: "source.ip,destination.ip"

redis-cli -n 6
flushdb

"columns": [
     "malware.name",
     "extra.tags"
],

"columns": "malware.name,extra.tags"

fields:
  output: The provider is {{ msg['feed.provider'] }}!
  feed.url: "{{ msg['feed.url'] | upper }}"
  extra.somejinjaoutput: file:///etc/intelmq/somejinjatemplate.j2

intelmq.bots.experts.maxmind_geoip.expert --update-database

[
    {
        "rulename": "Standard Protocols http",
        "if": {
            "source.port": "^(80|443)$"
        },
        "then": {
            "protocol.application": "http"
        }
    },
    {
        "rulename": "Spamhaus Cert conficker",
        "if": {
            "malware.name": "^conficker(ab)?$"
        },
        "then": {
            "classification.identifier": "conficker"
        }
    },
    {
        "rulename": "bitdefender",
        "if": {
            "malware.name": "bitdefender-(.*)$"
        },
        "then": {
            "malware.name": "{matches[malware.name][1]}"
        }
    },
    {
        "rulename": "urlzone",
        "if": {
            "malware.name": "^urlzone2?$"
        },
        "then": {
            "classification.identifier": "urlzone"
        }
    },
    {
        "rulename": "default",
        "if": {
            "feed.name": "^Spamhaus Cert$"
        },
        "then": {
            "classification.identifier": "{msg[malware.name]}"
        }
    }
]

{
   "at": {
      "url": "rdap.server.at/v1/,
      "auth": {
         "type": "jwt",
         "token": "ey..."
      }
   },
   "de": "rdap.service:1337/v1/"
}

intelmq.bots.experts.recordedfuture_iprisk.expert --update-database

pip3 install -r intelmq/bots/experts/sieve/REQUIREMENTS.txt

if :exists source.fqdn {
  keep  // aborts processing of subsequent rules and forwards the event.
}

if :notexists source.abuse_contact || source.abuse_contact =~ '.*@example.com' {
  drop  // aborts processing of subsequent rules and drops the event.
}

if source.ip << '192.0.0.0/24' {
    add! comment = 'bogon' // sets the field comment to this value and overwrites existing values
    path 'other-path' // the message is sent to the given path
}

if classification.type :in ['phishing', 'malware-distribution'] && source.fqdn =~ '.*\.(ch|li)$' {
  add! comment = 'domainabuse'
  keep
} elif classification.type == 'scanner' {
  add! comment = 'ignore'
  drop
} else {
  remove comment
}

if EXPRESSION {
    ACTIONS
} elif EXPRESSION {
    ACTIONS
} else {
    ACTIONS
}

$ intelmq.bots.experts.sieve.validator
usage: intelmq.bots.experts.sieve.validator [-h] sievefile

Validates the syntax of sievebot files.

positional arguments:
  sievefile   Sieve file

optional arguments:
  -h, --help  show this help message and exit

"search_parameters": {
    "source.ip": "ip"
}

"result_fields": {
    "username": "source.account"
}

index="dhcp" ipv4address="$ip$" | ... | fields _time username ether

"add_keys": {
    "classification.type": "spam",
    "comment": "Started more than 10 SMTP connections"
}

intelmq.bots.experts.tor_nodes.expert --update-database

'never' --> intelmq
'daily' --> intelmq-2018-02-02
'weekly' --> intelmq-2018-42
'monthly' --> intelmq-2018-02
'yearly' --> intelmq-2018

> curl -XGET 'http://localhost:9200/intelmq/events/_search?pretty=True'

pip3 install pymongo>=2.7.1

createuser --no-superuser --no-createrole --no-createdb --encrypted --pwprompt intelmq

createdb --encoding='utf-8' --owner=intelmq intelmq-events

psql -h localhost intelmq-events intelmq </tmp/initdb.sql

sqlite3 your-db.db
sqlite> .read /tmp/initdb.sql

pip3 install -r intelmq/bots/collectors/templated_smtp/REQUIREMENTS.txt

- content-type: string, templated, content-type to use.
  text: string, templated, attachment text.
  name: string, templated, filename of attachment.

mail_to: "{{ event['source.abuse_contact'] }}"

{%- set output = from_json(event['output']) %}

attachments:
  - content-type: application/json
    text: |
      {
        "malware": "{{ event['malware.name'] }}",
        {%- set comma = joiner(", ") %}
        {%- for key in event %}
           {%- if key.startswith('source.') %}
        {{ comma() }}"{{ key }}": "{{ event[key] }}"
           {%- endif %}
        {%- endfor %}
      }
    name: report.json

attachments:
  - content-type: text/csv
    text: |
      {%- set fields = ["classification.taxonomy", "classification.type", "classification.identifier", "source.ip", "source.asn", "source.port"] %}
      {%- set sep = joiner(";") %}
      {%- for field in fields %}{{ sep() }}{{ field }}{%- endfor %}
      {% set sep = joiner(";") %}
      {%- for field in fields %}{{ sep() }}{{ event[field] }}{%- endfor %}
    name: event.csv

{"raw": "MjAxNi8wNC8yNV8xMTozOSxzY2hpenppbm8ub21hcmF0aG9uLmNvbS9na0NDSnVUSE0vRFBlQ1pFay9XdFZOSERLbC1tWFllRk5Iai8sODUuMjUuMTYwLjExNCxzdGF0aWMtaXAtODUtMjUtMTYwLTExNC5pbmFkZHIuaXAtcG9vbC5jb20uLEFuZ2xlciBFSywtLDg5NzI=", "source": {"asn": 8972, "ip": "85.25.160.114", "url": "http://schizzino.omarathon.com/gkCCJuTHM/DPeCZEk/WtVNHDKl-mXYeFNHj/", "reverse_dns": "static-ip-85-25-160-114.inaddr.ip-pool.com"}, "classification": {"type": "malware-distribution"}, "event_description": {"text": "Angler EK"}, "feed": {"url": "http://www.malwaredomainlist.com/updatescsv.php", "name": "Malware Domain List", "accuracy": 100.0}, "time": {"observation": "2016-04-29T10:59:34+00:00", "source": "2016-04-25T11:39:00+00:00"}}

Apr 29 11:01:29 header example {"raw": "MjAxNi8wNC8yNV8xMTozOSxzY2hpenppbm8ub21hcmF0aG9uLmNvbS9na0NDSnVUSE0vRFBlQ1pFay9XdFZOSERLbC1tWFllRk5Iai8sODUuMjUuMTYwLjExNCxzdGF0aWMtaXAtODUtMjUtMTYwLTExNC5pbmFkZHIuaXAtcG9vbC5jb20uLEFuZ2xlciBFSywtLDg5NzI=", "source": {"asn": 8972, "ip": "85.25.160.114", "url": "http://schizzino.omarathon.com/gkCCJuTHM/DPeCZEk/WtVNHDKl-mXYeFNHj/", "reverse_dns": "static-ip-85-25-160-114.inaddr.ip-pool.com"}, "classification": {"type": "malware-distribution"}, "event_description": {"text": "Angler EK"}, "feed": {"url": "http://www.malwaredomainlist.com/updatescsv.php", "name": "Malware Domain List", "accuracy": 100.0}, "time": {"observation": "2016-04-29T10:59:34+00:00", "source": "2016-04-25T11:39:00+00:00"}}

Apr 29 11:17:47 localhost IntelMQ-event|source.ip: 85.25.160.114|time.source:2016-04-25T11:39:00+00:00|feed.url:http://www.malwaredomainlist.com/updatescsv.php|time.observation:2016-04-29T11:17:44+00:00|source.reverse_dns:static-ip-85-25-160-114.inaddr.ip-pool.com|feed.name:Malware Domain List|event_description.text:Angler EK|source.url:http://schizzino.omarathon.com/gkCCJuTHM/DPeCZEk/WtVNHDKl-mXYeFNHj/|source.asn:8972|classification.type:malware-distribution|feed.accuracy:100.0

> intelmqctl start file-output
Starting file-output...
file-output is running.

> intelmqctl start file-output
file-output is running.

> intelmqctl stop file-output
Stopping file-output...
file-output is stopped.

> intelmqctl stop file-output
file-output was NOT RUNNING.

> intelmqctl stop file-output
file-output is still running

> intelmqctl status file-output
file-output is stopped.
> intelmqctl start file-output
Starting file-output...
file-output is running.
> intelmqctl status file-output
file-output is running.

> intelmqctl restart file-output
Stopping file-output...
file-output is stopped.
Starting file-output...
file-output is running.

> intelmqctl reload file-output
Reloading file-output ...
file-output is running.

> intelmqctl reload file-output
file-output was NOT RUNNING.

> intelmqctl run file-output
file-output: RestAPIOutputBot initialized with id file-output and version 3.5.2 as process 12345.
file-output: Bot is starting.
file-output: Loading source pipeline and queue 'file-output-queue'.
file-output: Connected to source queue.
file-output: No destination queues to load.
file-output: Bot initialization completed.
file-output: Waiting for incoming message.

> intelmqctl run file-output --help

> intelmqctl run file-output
Main instance of the bot is running in the background. You may want to launch: intelmqctl stop file-output

> intelmqctl run file-output console
*** Using console ipdb. Please use 'self' to access to the bot instance properties. ***
ipdb> self. ...

> intelmqctl run file-output console pudb

> intelmqctl run file-output message get
file-output: Waiting for a message to get...
{
    "classification.type": "c&c",
    "feed.url": "https://example.com",
    "raw": "1233",
    "source.ip": "1.2.3.4",
    "time.observation": "2017-05-17T22:00:33+00:00",
    "time.source": "2017-05-17T22:00:32+00:00"
}

> intelmqctl run file-output message send '{"time.observation": "2017-05-17T22:00:33+00:00", "time.source": "2017-05-17T22:00:32+00:00"}'
file-output: Bot has no destination queues.

> intelmqctl run file-output process
file-output: Bot is starting.
file-output: Bot initialization completed.
file-output: Processing...
file-output: Waiting for incoming message.
file-output: Received message {'raw': '1234'}.

> intelmqctl run file-output process -d
file-output:  * Dryrun only, no message will be really sent through.
...
file-output: DRYRUN: Message would be acknowledged now!

> intelmqctl run file-output process -m '{"source.ip":"1.2.3.4"}'
file-output:  * Message from cli will be used when processing.
...

> intelmqctl status file-output
file-output is stopped.
> intelmqctl disable file-output
> intelmqctl status file-output
file-output is disabled.

> intelmqctl status file-output
file-output is disabled.
> intelmqctl enable file-output
> intelmqctl status file-output
file-output is stopped.

> intelmqctl start
Starting abusech-domain-parser...
abusech-domain-parser is running.
Starting abusech-feodo-domains-collector...
abusech-feodo-domains-collector is running.
Starting deduplicator-expert...
deduplicator-expert is running.
file-output is disabled.
Botnet is running.

> intelmqctl stop
Stopping Botnet...
Stopping abusech-domain-parser...
abusech-domain-parser is stopped.
Stopping abusech-feodo-domains-collector...
abusech-feodo-domains-collector is stopped.
Stopping deduplicator-expert...
deduplicator-expert is stopped.
Stopping file-output...
file-output is stopped.
Botnet is stopped.

> intelmqctl status
abusech-domain-parser is running.
abusech-feodo-domains-collector is running.
deduplicator-expert is running.
file-output is disabled.

> intelmqctl status
abusech-domain-parser is running.
abusech-feodo-domains-collector is running.
deduplicator-expert is running.
file-output is running.

> intelmqctl status
abusech-domain-parser is stopped.
abusech-feodo-domains-collector is stopped.
deduplicator-expert is stopped.
file-output is disabled.

> intelmqctl status
file-output is stopped.
malware-domain-list-collector is stopped.
malware-domain-list-parser is stopped.
> intelmqctl disable file-output
> intelmqctl status
file-output is disabled.
malware-domain-list-collector is stopped.
malware-domain-list-parser is stopped.
> intelmqctl enable file-output
> intelmqctl status
file-output is stopped.
malware-domain-list-collector is stopped.
malware-domain-list-parser is stopped.

> intelmqctl list queues
abusech-domain-parser-queue - 0
abusech-domain-parser-queue-internal - 0
deduplicator-expert-queue - 0
deduplicator-expert-queue-internal - 0
file-output-queue - 234
file-output-queue-internal - 0

> intelmqctl list queues -q
file-output-queue - 234

> intelmqctl list queues --sum
42

redis-cli -n 2
keys * # lists all existing non-empty queues
llen [queue-name] # shows the length of the queue [queue-name]
lindex [queue-name] [index] # show the [index]'s message of the queue [queue-name]
del [queue-name] # remove the queue [queue-name]

"intelmqctl_check_orphaned_queues_ignore": ["Taichung-Parser"],

server {
    listen 443 ssl http2;
    server_name [your host name];
    client_max_body_size 50M;

    ssl_certificate [path to your key];
    ssl_certificate_key [path to your certificate];

    location /[your private url] {
         if ($http_authorization != '[your private password]') {
             return 403;
         }
         proxy_pass http://localhost:5001/intelmq/push;
         proxy_read_timeout 30;
         proxy_connect_timeout 30;
     }
}

-----BEGIN PGP PUBLIC KEY BLOCK-----
Version: SKS 1.1.6
Comment: Hostname: pgp.mit.edu

mQINBFRl7D8BEADaRFoDa/+r27Gtqrdn8sZL4aSYTU4Q3gDr3TfigK8H26Un/Y79a/DUL1o0
o8SRae3uwVcjJDHZ6KDnxThbqF7URfpuCcCYxOs8p/eu3dSueqEGTODHWF4ChIh2japJDc4t
3FQHbIh2e3GHotVqJGhvxMmWqBFoZ/mlWvhjs99FFBZ87qbUNk7l1UAGEXeWeECgz9nGox40
3YpCgEsnJJsKC53y5LD/wBf4z+z0GsLg2GMRejmPRgrkSE/d9VjF/+niifAj2ZVFoINSVjjI
8wQFc8qLiExdzwLdgc+ggdzk5scY3ugI5IBt1zflxMIOG4BxKj/5IWsnhKMG2NLVGUYOODoG
pKhcY0gCHypw1bmkp2m+BDVyg4KM2fFPgQ554DAX3xdukMCzzZyBxR3UdT4dN7xRVhpph3Y2
Amh1E/dpde9uwKFk1oRHkRZ3UT1XtpbXtFNY0wCiGXPt6KznJAJcomYFkeLHjJo3nMK0hISV
GSNetVLfNWlTkeo93E1innbSaDEN70H4jPivjdVjSrLtIGfr2IudUJI84dGmvMxssWuM2qdg
FSzoTHw9UE9KT3SltKPS+F7u9x3h1J492YaVDncATRjPZUBDhbvo6Pcezhup7XTnI3gbRQc2
oEUDb933nwuobHm3VsUcf9686v6j8TYehsbjk+zdA4BoS/IdCwARAQABtC5UdXJyaXMgR3Jl
eWxpc3QgR2VuZXJhdG9yIDxncmV5bGlzdEB0dXJyaXMuY3o+iQI4BBMBAgAiBQJUZew/AhsD
BgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAKCRDAQrU3EIdmZoH4D/9Jo6j9RZxCAPTaQ9WZ
WOdb1Eqd/206bObEX+xJAago+8vuy+waatHYBM9/+yxh0SIg2g5whd6J7A++7ePpt5XzX6hq
bzdG8qGtsCRu+CpDJ40UwHep79Ck6O/A9KbZcZW1z/DhbYT3z/ZVWALy4RtgmyC67Vr+j/C7
KNQ529bs3kP9AzvEIeBC4wdKl8dUSuZIPFbgf565zRNKLtHVgVhiuDPcxKmBEl4/PLYF30a9
5Tgp8/PNa2qp1DV/EZjcsxvSRIZB3InGBvdKdSzvs4N/wLnKWedj1GGm7tJhSkJa4MLBSOIx
yamhTS/3A5Cd1qoDhLkp7DGVXSdgEtpoZDC0jR7nTS6pXojcgQaF7SfJ3cjZaLI5rjsx0YLk
G4PzonQKCAAQG1G9haCDniD8NrrkZ3eFiafoKEECRFETIG0BJHjPdSWcK9jtNCupBYb7JCiz
Q0hwLh2wrw/wCutQezD8XfsBFFIQC18TsJAVgdHLZnGYkd5dIbV/1scOcm52w6EGIeMBBYlB
J2+JNukH5sJDA6zAXNl2I1H1eZsP4+FSNIfB6LdovHVPAjn7qXCw3+IonnQK8+g8YJkbbhKJ
sPejfg+ndpe5u0zX+GvQCFBFu03muANA0Y/OOeGIQwU93d/akN0P1SRfq+bDXnkRIJQOD6XV
0ZPKVXlNOjy/z2iN2A==
=wjkM
-----END PGP PUBLIC KEY BLOCK-----

./scripts/run_dev.sh

pip install intelmq-api

apt install gunicorn apache2

a2enmod proxy_http

intelmq-api-setup-systemd

INTELMQ_API_CONFIG=/etc/intelmq/api-config.json ./scripts/run_dev.sh

{
    "intelmq_ctl_cmd": ["sudo", "-u", "intelmq", "intelmqctl"],
    "allowed_path": "/opt/intelmq/var/lib/bots/",
    "session_store": "/etc/intelmq/api-session.sqlite",
    "session_duration": 86400,
    "allow_origins": ["*"]
}

intelmq-api-adduser --user <username> --password <password>

setenforce 0

{
    "username": "$your_username",
    "password": "$your_password"
}

Authorization: $login_token

> curl --location --request POST "http://localhost/intelmq/v1/api/login/"\
    --header "Content-Type: application/x-www-form-urlencoded"\
    --data-urlencode "username=$username"\
    --data-urlencode "password=$password"
{"login_token":"68b329da9893e34099c7d8ad5cb9c940","username":"$username"}
> curl --location "http://localhost/intelmq/v1/api/version"\
    --header "Authorization: 68b329da9893e34099c7d8ad5cb9c940"
{"intelmq":"3.0.0rc1","intelmq-manager":"2.3.1"}

intelmq_manager.runctl.IntelMQCtlError: <ERROR_MESSAGE>

journalctl -u intelmq-api

pip3 install intelmq-manager

Alias /intelmq-manager /usr/share/intelmq_manager/html/

<Directory /usr/share/intelmq_manager/html>
    <IfModule mod_headers.c>
    Header set Content-Security-Policy "script-src 'self'"
    Header set X-Content-Security-Policy "script-src 'self'"
    </IfModule>
</Directory>

Content-Security-Policy: script-src 'self'
X-Content-Security-Policy: script-src 'self'

intelmq.lib.exceptions.PipelineError: pipeline failed - ConnectionError('Error 13 connecting to unix socket: /var/run/redis/redis.sock. Permission denied.',)

unixsocketperm 770

usermod -aG redis intelmq

intelmq.lib.exceptions.InvalidValue: invalid value '2017-03-06T07:36:29' () for key 'time.source'

UPDATE events SET raw = NULL WHERE "time.source" < '2018-07-01';

server_tokens off; # turn off server_token, instead of nginx/13.2 now it will only show nginx
add_header X-Frame-Options SAMEORIGIN; # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Frame-Options
add_header X-Content-Type-Options nosniff; # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Content-Type-Options
add_header X-XSS-Protection "1; mode=block"; # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-XSS-Protection
add_header Content-Security-Policy "script-src 'self' 'unsafe-inline' http(s)://<your-domain>; frame-src 'self' http(s)://<your-domain>; object-src 'self' http(s)://<your-domain>"; # https://developer.mozilla.org/en-US/docs/Web/HTTP/CSP

volumes:
- ./example_config/nginx/security.conf:/etc/nginx/conf.d/security.conf

bot-id: logstash-output
redis_server_ip: 10.10.10.10
redis_server_port: 6379
redis_db: 4
redis_queue: logstash-queue

input {
  redis {
    host => "10.10.10.10"
    port => 6379
    db => 4
    data_type => "list"
    key => "logstash-queue"
  }
}

filter {
  mutate {
    lowercase => ["source.geolocation.city", "classification.identifier"]
    remove_field => ["__type", "@version"]
  }
  date {
    match => ["time.observation", "ISO8601"]
  }
}

output {
  elasticsearch {
    hosts => ["http://10.10.10.11:9200", "http://10.10.10.12:9200"]
    index => "intelmq-%{+YYYY.MM}"
  }
}

apply_mapping_eventdb.py -h
apply_domain_suffix.py -h

# Adjust paths if you want to use non-standard directories
export INTELMQ_REPO=/opt/dev_intelmq
export INTELMQ_ROOT_DIR=/opt/intelmq

sudo -s

git clone https://github.com/<your username>/intelmq.git $INTELMQ_REPO
cd $INTELMQ_REPO

pip3 install -e .

useradd -d $INTELMQ_ROOT_DIR -U -s /bin/bash intelmq

intelmqsetup

git clone https://github.com/<your username>/intelmq.git $INTELMQ_REPO
cd $INTELMQ_REPO

python -m venv .venv
source .venv/bin/activate

pip install -e .

# If you use a non-local directory as INTELMQ_ROOT_DIR, use following
# command to create it and change the ownership.
sudo install -g `whoami` -o `whoami` -d $INTELMQ_ROOT_DIR
# For local directory, just create it with mkdir:
mkdir $INTELMQ_ROOT_DIR

intelmqsetup --skip-ownership

# For older Docker versions, you may need to use `docker-compose` command
docker compose -f contrib/development-tools/docker-compose-common-services.yaml up -d

su - intelmq # Use for global installation
source .venv/bin/activate # Use for virtual environment installation

intelmqctl start spamhaus-drop-collector

tail -f $INTELMQ_ROOT_DIR/var/log/spamhaus-drop-collector.log

sudo -s # Use for global installation
source .venv/bin/activate # Use for virtual environment installation

cd /opt/dev_intelmq
pip3 install -e .

find $INTELMQ_ROOT_DIR/ -type d -exec chmod 0770 {} \+
find $INTELMQ_ROOT_DIR/ -type f -exec chmod 0660 {} \+
chown -R intelmq.intelmq $INTELMQ_ROOT_DIR
## if you use the intelmq manager (adapt the webservers' group if needed):
chown intelmq.www-data $INTELMQ_ROOT_DIR/etc/*.conf

su - intelmq # Use for global installation
source .venv/bin/activate # Use for virtual environment installation

intelmqctl start <bot_id>

pip3 install -e .[development]

cd $INTELMQ_REPO
sudo -u intelmq python3 -m unittest {discover|filename}  # or
sudo -u intelmq pytest [filename]
sudo -u intelmq python3 setup.py test  # uses a build environment (no external dependencies)

INTELMQ_TEST_DATABASES=1 INTELMQ_TEST_EXOTIC=1 INTELMQ_TEST_INSTALLATION=1 pytest intelmq/tests/

intelmq/
  lib/
    bot.py
    cache.py
    message.py
    pipeline.py
    utils.py
  bots/
    collector/
      <bot name>/
            collector.py
    parser/
      <bot name>/
            parser.py
    expert/
      <bot name>/
            expert.py
    output/
      <bot name>/
            output.py
  /conf
    runtime.yaml

/intelmq/bots/parser/abusech/parser_domain.py
/intelmq/bots/parser/abusech/parser_ip.py
/intelmq/bots/parser/abusech/parser_ransomware.py

/intelmq/bots/parser/abusech/parser_malicious_url.py

intelmq/bots/parser/taichung/parser.py
intelmq/bots/parser/cymru/parser_full_bogons.py
intelmq/bots/parser/abusech/parser_ransomware.py

> git remote add upstream https://github.com/certtools/intelmq.git

> git checkout develop
> git pull upstream develop
> git push origin develop

> git checkout develop
> git checkout -b bugfix
# your work
> git commit

> git checkout maintenance
> git checkout -b new-feature
# your work
> git commit

> git checkout develop
> git pull upstream develop
> git push origin develop

> git checkout bugfix
> git rebase develop

> git checkout bugfix
> git merge develop

$ sudo -i intelmq
$ intelmqctl run file-output  # if configured
$ intelmq.bots.outputs.file.output file-output

"""
SPDX-FileCopyrightText: 2021 Your Name
SPDX-License-Identifier: AGPL-3.0-or-later

Parse data from example.com, be a nice ExampleParserBot.

Document possible necessary configurations.
"""
import sys

# imports for additional libraries and intelmq
from intelmq.lib.bot import ParserBot

class ExampleParserBot(ParserBot):

    option1: str = "defaultvalue"
    option2: bool = False

    def process(self):
        report = self.receive_message()

        event = self.new_event(report)  # copies feed.name, time.observation
        ... # implement the logic here
        event.add('source.ip', '127.0.0.1')
        event.add('extra', {"os.name": "Linux"})
        if self.option2:
             event.add('extra', {"customvalue": self.option1})

        self.send_message(event)
        self.acknowledge_message()

BOT = ExampleParserBot

class HTTPCollectorBot(CollectorBot, HttpMixin):

<timestamp> - <bot id> - <log level> - <log message>

pip3 install -r docs/requirements.txt

cd docs
make html

pip3 install --pre intelmq

from intelmq.lib.bot import BotLibSettings
from intelmq.bots.experts.domain_suffix.expert import DomainSuffixExpertBot

domain_suffix = DomainSuffixExpertBot('domain-suffix',  # bot id
                                      settings=BotLibSettings | {
                                               'field': 'fqdn',
                                               'suffix_file': '/usr/share/publicsuffix/public_suffix_list.dat'}

queues = domain_suffix.process_message({'source.fqdn': 'www.example.com'})

domain_suffix.process_message({'source.fqdn': 'www.example.com'}, {'source.fqdn': 'www.example.net'})
domain_suffix.process_message(*[{'source.fqdn': 'www.example.com'}, {'source.fqdn': 'www.example.net'}])

>>> output['output'][0]['source.domain_suffix']
'com'

rm -r build/
python3 setup.py sdist bdist_wheel

### Configuration

### Core

### Development

### Data Format

### Bots
#### Collectors

#### Parsers

#### Experts

#### Outputs

### Documentation

### Packaging

### Tests

### Tools

### Contrib

### Known issues

### Requirements

### Tools

### Data Format

### Configuration

### Libraries

### Postgres databases

### Pages

#### Landing page

#### Configuration

#### Management

#### Monitor

#### Check

### Documentation

### Third-party libraries

### Packaging

### Known issues

version_history = [..., [2, 0, 0], [2, 0, 1]]
upgrades = {
    "v112_feodo_tracker_domains": true,
    "v112_feodo_tracker_ips": false,
    "v200beta1_ripe_expert": false
    }
results = [
    {"function": "v112_feodo_tracker_domains",
     "success": true,
     "retval": null,
     "time": "..."},
    {"function": "v112_feodo_tracker_domains",
     "success": false,
     "retval": "fix it manually",
     "message": "fix it manually",
     "time": "..."},
    {"function": "v200beta1_ripe_expert",
     "success": false,
     "traceback": "...",
     "time": "..."}
    ]

# define EAI_BADFLAGS     -1    /* Invalid value for `ai_flags' field.  */
# define EAI_NONAME       -2    /* NAME or SERVICE is unknown.  */
# define EAI_AGAIN        -3    /* Temporary failure in name resolution.  */
# define EAI_FAIL         -4    /* Non-recoverable failure in name res.  */
# define EAI_NODATA       -5    /* No address associated with NAME.  */
# define EAI_FAMILY       -6    /* `ai_family' not supported.  */
# define EAI_SOCKTYPE     -7    /* `ai_socktype' not supported.  */
# define EAI_SERVICE      -8    /* SERVICE not supported for `ai_socktype'.  */
# define EAI_ADDRFAMILY   -9    /* Address family for NAME not supported.  */
# define EAI_MEMORY       -10   /* Memory allocation failure.  */
# define EAI_SYSTEM       -11   /* System error returned in `errno'.  */

"add_feed_provider_as_tag": true,
"add_feed_name_as_tag": true,
"misp_additional_correlation_fields": ["source.asn"],
"misp_additional_tags": ["OSINT", "osint:certainty=="90""],
"misp_publish": false,
"misp_to_ids_fields": ["source.fqdn", "source.reverse_dns"],
"significant_fields": ["source.fqdn", "source.reverse_dns"],

parse = ParserBot.parse_csv

parse = ParserBot.parse_csv

parse = ParserBot.parse_csv

parse = ParserBot.parse_csv

parse = ParserBot.parse_csv

parse = ParserBot.parse_csv

parse = ParserBot.parse_csv

parse = ParserBot.parse_csv

parse = ParserBot.parse_csv

{
  "DataFeed": "CTIP-Infected",
  "SourcedFrom": "SinkHoleMessage|SensorMessage"",
  "DateTimeReceivedUtc": nt time
  "DateTimeReceivedUtcTxt": human readable
  "Malware":
  "ThreatCode": "B67-SS-TINBA",
  "ThreatConfidence": "High|Medium|Low|Informational", -> 100/50/20/10
  "TotalEncounters": 3,
  "TLP": "Amber",
  "SourceIp":
  "SourcePort":
  "DestinationIp":
  "DestinationPort":
  "TargetIp": Deprecated, so we gonne ignore it
  "TargetPort": Deprecated, so we gonne ignore it
  "SourceIpInfo": {
    "SourceIpAsnNumber":
    "SourceIpAsnOrgName":
    "SourceIpCountryCode":
    "SourceIpRegion":
    "SourceIpCity"
    "SourceIpPostalCode"
    "SourceIpLatitude"
    "SourceIpLongitude"
    "SourceIpMetroCode"
    "SourceIpAreaCode"
    "SourceIpConnectionType"
  },
  "HttpInfo": {
    "HttpHost": "",
    "HttpRequest": "",
    "HttpMethod": "",
    "HttpReferrer": "",
    "HttpUserAgent": "",
    "HttpVersion": ""
  },
  "CustomInfo": {
    "CustomField1": "",
    "CustomField2": "",
    "CustomField3": "",
    "CustomField4": "",
    "CustomField5": ""
  },
  "Payload": base64 encoded json with meaningful dictionary keys or JSON-string with numbered dictionary keys
}

parse = ParserBot.parse_csv

parse = ParserBot.parse_csv

parse = ParserBot.parse_csv

parse = ParserBot.parse_csv

parse = ParserBot.parse_csv

>>> exc = IndexError('This is a test')
>>> error_message_from_exc(exc)
'This is a test'

>>> parse_relative('4 hours')
240

feed name	file name
Accessible-ADB	scan_adb
Accessible-AFP	scan_afp
Accessible-AMQP	scan_amqp
Accessible-ARD	scan_ard
Accessible-Cisco-Smart-Install	cisco_smart_install
Accessible-CoAP	scan_coap
Accessible-CWMP	scan_cwmp
Accessible-MS-RDPEUDP	scan_msrdpeudp
Accessible-FTP	scan_ftp
Accessible-Hadoop	scan_hadoop
Accessible-HTTP	scan_http
Accessible-Radmin	scan_radmin
Accessible-RDP	scan_rdp
Accessible-Rsync	scan_rsync
Accessible-SMB	scan_smb
Accessible-Telnet	scan_telnet
Accessible-Ubiquiti-Discovery-Service	scan_ubiquiti
Accessible-VNC	scan_vnc
Blacklisted-IP (deprecated)	blacklist
Blocklist	blocklist
Compromised-Website	compromised_website
Device-Identification IPv4 / IPv6	device_id/device_id6
DNS-Open-Resolvers	scan_dns
Honeypot-Amplification-DDoS-Events	event4_honeypot_ddos_amp
Honeypot-Brute-Force-Events	event4_honeypot_brute_force
Honeypot-Darknet	event4_honeypot_darknet
Honeypot-HTTP-Scan	event4_honeypot_http_scan
HTTP-Scanners	hp_http_scan
ICS-Scanners	hp_ics_scan
IP-Spoofer-Events	event4_ip_spoofer
Microsoft-Sinkhole-Events IPv4	event4_microsoft_sinkhole
Microsoft-Sinkhole-Events-HTTP IPv4	event4_microsoft_sinkhole_http
NTP-Monitor	scan_ntpmonitor
NTP-Version	scan_ntp
Open-Chargen	scan_chargen
Open-DB2-Discovery-Service	scan_db2
Open-Elasticsearch	scan_elasticsearch
Open-IPMI	scan_ipmi
Open-IPP	scan_ipp
Open-LDAP	scan_ldap
Open-LDAP-TCP	scan_ldap_tcp
Open-mDNS	scan_mdns
Open-Memcached	scan_memcached
Open-MongoDB	scan_mongodb
Open-MQTT	scan_mqtt
Open-MSSQL	scan_mssql
Open-NATPMP	scan_nat_pmp
Open-NetBIOS-Nameservice	scan_netbios
Open-Netis	netis_router
Open-Portmapper	scan_portmapper
Open-QOTD	scan_qotd
Open-Redis	scan_redis
Open-SNMP	scan_snmp
Open-SSDP	scan_ssdp
Open-TFTP	scan_tftp
Open-XDMCP	scan_xdmcp
Outdated-DNSSEC-Key	outdated_dnssec_key
Outdated-DNSSEC-Key-IPv6	outdated_dnssec_key_v6
Sandbox-URL	cwsandbox_url
Sinkhole-DNS	sinkhole_dns
Sinkhole-Events	event4_sinkhole/event6_sinkhole
Sinkhole-Events IPv4	event4_sinkhole
Sinkhole-Events IPv6	event6_sinkhole
Sinkhole-HTTP-Events	event4_sinkhole_http/event6_sinkhole_http
Sinkhole-HTTP-Events IPv4	event4_sinkhole_http
Sinkhole-HTTP-Events IPv6	event6_sinkhole_http
Sinkhole-Events-HTTP-Referer	event4_sinkhole_http_referer/event6_sinkhole_http_referer
Sinkhole-Events-HTTP-Referer IPv4	event4_sinkhole_http_referer
Sinkhole-Events-HTTP-Referer IPv6	event6_sinkhole_http_referer
Spam-URL	spam_url
SSL-FREAK-Vulnerable-Servers	scan_ssl_freak
SSL-POODLE-Vulnerable-Servers	scan_ssl_poodle/scan6_ssl_poodle
Vulnerable-Exchange-Server *	scan_exchange
Vulnerable-ISAKMP	scan_isakmp
Vulnerable-HTTP	scan_http
Vulnerable-SMTP	scan_smtp_vulnerable

feed name	successor feed name	file name
Amplification-DDoS-Victim	Honeypot-Amplification-DDoS-Events	`ddos_amplification`
CAIDA-IP-Spoofer	IP-Spoofer-Events	`caida_ip_spoofer`
Darknet	Honeypot-Darknet	`darknet`
Drone	Sinkhole-Events	`botnet_drone`
Drone-Brute-Force	Honeypot-Brute-Force-Events, Sinkhole-HTTP-Events	`drone_brute_force`
Microsoft-Sinkhole	Sinkhole-HTTP-Events	`microsoft_sinkhole`
Sinkhole-HTTP-Drone	Sinkhole-HTTP-Events	`sinkhole_http_drone`
IPv6-Sinkhole-HTTP-Drone	Sinkhole-HTTP-Events	`sinkhole6_http`

action	match	_default	action_other	filter_match	filter_no_match
keep	✓	✓	✗	✓	✗
keep	✗	✗	✓	✗	✓
drop	✓	✗	✓	✓	✗
drop	✗	✓	✗	✗	✓

Taxonomy	Type	Description
abusive-content	harmful-speech	Discreditation or discrimination of somebody, e.g. cyber stalking, racism or threats against one or more individuals.
abusive content	spam	Or ‘Unsolicited Bulk Email’, this means that the recipient has not granted verifiable permission for the message to be sent and that the message is sent as part of a larger collection of messages, all having a functionally comparable content.
abusive-content	violence	Child pornography, glorification of violence, etc.
availability	ddos	Distributed Denial of Service attack, e.g. SYN-Flood or UDP-based reflection/amplification attacks.
availability	dos	Denial of Service attack, e.g. sending specially crafted requests to a web application which causes the application to crash or slow down.
availability	misconfiguration	Software misconfiguration resulting in service availability issues, e.g. DNS server with outdated DNSSEC Root Zone KSK.
availability	outage	Outage caused e.g. by air condition failure or natural disaster.
availability	sabotage	Physical sabotage, e.g cutting wires or malicious arson.
fraud	copyright	Offering or Installing copies of unlicensed commercial software or other copyright protected materials (Warez).
fraud	masquerade	Type of attack in which one entity illegitimately impersonates the identity of another in order to benefit from it.
fraud	phishing	Masquerading as another entity in order to persuade the user to reveal private credentials.
fraud	unauthorized-use-of-resources	Using resources for unauthorized purposes including profit-making ventures, e.g. the use of e-mail to participate in illegal profit chain letters or pyramid schemes.
information-content-security	data-leak	Leaked confidential information like credentials or personal data.
information-content-security	data-loss	Loss of data, e.g. caused by harddisk failure or physical theft.
information-content-security	unauthorised-information-access	Unauthorized access to information, e.g. by abusing stolen login credentials for a system or application, intercepting traffic or gaining access to physical documents.
information-content-security	unauthorised-information-modification	Unauthorised modification of information, e.g. by an attacker abusing stolen login credentials for a system or application or a ransomware encrypting data.
information-gathering	scanner	Attacks that send requests to a system to discover weaknesses. This also includes testing processes to gather information on hosts, services and accounts. Examples: fingerd, DNS querying, ICMP, SMTP (EXPN, RCPT, …), port scanning.
information-gathering	sniffing	Observing and recording of network traffic (wiretapping).
information-gathering	social-engineering	Gathering information from a human being in a non-technical way (e.g. lies, tricks, bribes, or threats). This IOC refers to a resource, which has been observed to perform brute-force attacks over a given application protocol.
intrusion-attempts	brute-force	Multiple login attempts (Guessing / cracking of passwords, brute force).
intrusion-attempts	exploit	An attack using an unknown exploit.
intrusion-attempts	ids-alert	IOCs based on a sensor network. This is a generic IOC denomination, should it be difficult to reliably denote the exact type of activity involved for example due to an anecdotal nature of the rule that triggered the alert.
intrusions	application-compromise	Compromise of an application by exploiting (un)known software vulnerabilities, e.g. SQL injection.
intrusions	burglary	Physical intrusion, e.g. into corporate building or data center.
intrusions	privileged-account-compromise	Compromise of a system where the attacker gained administrative privileges.
intrusions	system-compromise	Compromise of a system, e.g. unauthorised logins or commands. This includes compromising attempts on honeypot systems.
intrusions	unprivileged-account-compromise	Compromise of a system using an unprivileged (user/service) account.
malicious-code	c2-server	This is a command and control server in charge of a given number of botnet drones.
malicious-code	infected-system	This is a compromised machine, which has been observed to make a connection to a command and control server.
malicious-code	malware-configuration	This is a resource which updates botnet drones with a new configuration.
malicious-code	malware-distribution	URI used for malware distribution, e.g. a download URL included in fake invoice malware spam.
other	blacklist	Some sources provide blacklists, which clearly refer to abusive behavior, such as spamming, but fail to denote the exact reason why a given identity has been blacklisted. The reason may be that the justification is anecdotal or missing entirely. This type should only be used if the typing fits the definition of a blacklist, but an event specific denomination is not possible for one reason or another. Not in RSIT.
other	dga-domain	DGA Domains are seen various families of malware that are used to periodically generate a large number of domain names that can be used as rendezvous points with their command and control servers. Not in RSIT.
other	other	All incidents which don’t fit in one of the given categories should be put into this class.
other	malware	An IoC referring to a malware (sample) itself. Not in RSIT.
other	proxy	This refers to the use of proxies from inside your network. Not in RSIT.
test	test	Meant for testing. Not in RSIT.
other	tor	This IOC refers to incidents related to TOR network infrastructure. Not in RSIT.
other	undetermined	The categorisation of the incident is unknown/undetermined.
vulnerable	ddos-amplifier	Publicly accessible services that can be abused for conducting DDoS reflection/amplification attacks, e.g. DNS open-resolvers or NTP servers with monlist enabled.
vulnerable	information-disclosure	Publicly accessible services potentially disclosing sensitive information, e.g. SNMP or Redis.
vulnerable	potentially-unwanted-accessible	Potentially unwanted publicly accessible services, e.g. Telnet, RDP or VNC.
vulnerable	vulnerable-system	A system which is vulnerable to certain attacks. Example: misconfigured client proxy settings (example: WPAD), outdated operating system version, etc.
vulnerable	weak-crypto	Publicly accessible services offering weak crypto, e.g. web servers susceptible to POODLE/FREAK attacks.

Type	Source	Destination	Possible identifiers
blacklist	blacklisted device
brute-force	attacker	target
c2-server	(sinkholed) c&c server		zeus, palevo, feodo
ddos	attacker	target
dga-domain	infected device
dropzone	server hosting stolen data
exploit	hosting server
ids-alert	triggering device
infected-system	infected device	contacted c2c server
malware	infected device		zeus, palevo, feodo
malware configuration	infected device
malware-distribution	server hosting malware
phishing	phishing website
proxy	server allowing policy and security bypass
scanner	scanning device	scanned device	http,modbus,wordpress
spam	infected device	targeted server
system-compromise	server
vulnerable-system	vulnerable device		heartbleed, openresolver, snmp, wpad

Category	Key	Terminology
Feed	feed.name	Should
Classification	classification.type	Should
Classification	classification.taxonomy	Should
Time	time.source	Should
Time	time.observation	Should
Identity	source.ip	Should*
Identity	source.fqdn	Should*
Identity	source.url	Should*
Identity	source.account	Should*

Section	Name	Type	Description
Classification	classification.identifier	String	The lowercase identifier defines the actual software or service (e.g. `heartbleed` or `ntp_version`) or standardized malware name (e.g. `zeus`). Note that you MAY overwrite this field during processing for your individual setup. This field is not standardized across IntelMQ setups/users.
Classification	classification.taxonomy	ClassificationTaxonomy	We recognize the need for the CSIRT teams to apply a static (incident) taxonomy to abuse data. With this goal in mind the type IOC will serve as a basis for this activity. Each value of the dynamic type mapping translates to a an element in the static taxonomy. The European CSIRT teams for example have decided to apply the eCSIRT.net incident classification. The value of the taxonomy key is thus a derivative of the dynamic type above. For more information about check ENISA taxonomies.
Classification	classification.type	ClassificationType	The abuse type IOC is one of the most crucial pieces of information for any given abuse event. The main idea of dynamic typing is to keep our ontology flexible, since we need to evolve with the evolving threatscape of abuse data. In contrast with the static taxonomy below, the dynamic typing is used to perform business decisions in the abuse handling pipeline. Furthermore, the value data set should be kept as minimal as possible to avoid type explosion, which in turn dilutes the business value of the dynamic typing. In general, we normally have two types of abuse type IOC: ones referring to a compromised resource or ones referring to pieces of the criminal infrastructure, such as a command and control servers for example.
	comment	String	Free text commentary about the abuse event inserted by an analyst.
Destination	destination.abuse_contact	LowercaseString	Abuse contact for destination address. A comma separated list.
Destination	destination.account	String	An account name or email address, which has been identified to relate to the destination of an abuse event.
Destination	destination.allocated	DateTime	Allocation date corresponding to BGP prefix.
Destination	destination.as_name	String	The autonomous system name to which the connection headed.
Destination	destination.asn	ASN	The autonomous system number to which the connection headed.
Destination	destination.domain_suffix	FQDN	The suffix of the domain from the public suffix list.
Destination	destination.fqdn	FQDN	A DNS name related to the host from which the connection originated. DNS allows even binary data in DNS, so we have to allow everything. A final point is stripped, string is converted to lower case characters.
Destination Geolocation	destination.geolocation.cc	UppercaseString	Country-Code according to ISO3166-1 alpha-2 for the destination IP.
Destination Geolocation	destination.geolocation.city	String	Some geolocation services refer to city-level geolocation.
Destination Geolocation	destination.geolocation.country	String	The country name derived from the ISO3166 country code (assigned to cc field).
Destination Geolocation	destination.geolocation.latitude	Float	Latitude coordinates derived from a geolocation service, such as MaxMind geoip db.
Destination Geolocation	destination.geolocation.longitude	Float	Longitude coordinates derived from a geolocation service, such as MaxMind geoip db.
Destination Geolocation	destination.geolocation.region	String	Some geolocation services refer to region-level geolocation.
Destination Geolocation	destination.geolocation.state	String	Some geolocation services refer to state-level geolocation.
Destination	destination.ip	IPAddress	The IP which is the target of the observed connections.
Destination	destination.local_hostname	String	Some sources report an internal hostname within a NAT related to the name configured for a compromised system
Destination	destination.local_ip	IPAddress	Some sources report an internal (NATed) IP address related a compromised system. N.B. RFC1918 IPs are OK here.
Destination	destination.network	IPNetwork	CIDR for an autonomous system. Also known as BGP prefix. If multiple values are possible, select the most specific.
Destination	destination.port	Integer	The port to which the connection headed.
Destination	destination.registry	Registry	The IP registry a given ip address is allocated by.
Destination	destination.reverse_dns	FQDN	Reverse DNS name acquired through a reverse DNS query on an IP address. N.B. Record types other than PTR records may also appear in the reverse DNS tree. Furthermore, unfortunately, there is no rule prohibiting people from writing anything in a PTR record. Even JavaScript will work. A final point is stripped, string is converted to lower case characters.
Destination	destination.tor_node	Boolean	If the destination IP was a known tor node.
Destination	destination.url	URL	A URL denotes on IOC, which refers to a malicious resource, whose interpretation is defined by the abuse type. A URL with the abuse type phishing refers to a phishing resource.
Destination	destination.urlpath	String	The path portion of an HTTP or related network request.
Event_Description	event_description.target	String	Some sources denominate the target (organization) of a an attack.
Event_Description	event_description.text	String	A free-form textual description of an abuse event.
Event_Description	event_description.url	URL	A description URL is a link to a further description of the the abuse event in question.
	event_hash	UppercaseString	Computed event hash with specific keys and values that identify a unique event. At present, the hash should default to using the SHA1 function. Please note that for an event hash to be able to match more than one event (deduplication) the receiver of an event should calculate it based on a minimal set of keys and values present in the event. Using for example the observation time in the calculation will most likely render the checksum useless for deduplication purposes.
	extra	JSONDict	All anecdotal information, which cannot be parsed into the data harmonization elements. E.g. os.name, os.version, etc. Note: this is only intended for mapping any fields which can not map naturally into the data harmonization. It is not intended for extending the data harmonization with your own fields.
Feed	feed.accuracy	Accuracy	A float between 0 and 100 that represents how accurate the data in the feed is
Feed	feed.code	String	Code name for the feed, e.g. DFGS, HSDAG etc.
Feed	feed.documentation	String	A URL or hint where to find the documentation of this feed.
Feed	feed.name	String	Name for the feed, usually found in collector bot configuration.
Feed	feed.provider	String	Name for the provider of the feed, usually found in collector bot configuration.
Feed	feed.url	URL	The URL of a given abuse feed, where applicable
Malware Hash	malware.hash.md5	String	A string depicting an MD5 checksum for a file, be it a malware sample for example.
Malware Hash	malware.hash.sha1	String	A string depicting a SHA1 checksum for a file, be it a malware sample for example.
Malware Hash	malware.hash.sha256	String	A string depicting a SHA256 checksum for a file, be it a malware sample for example.
Malware	malware.name	LowercaseString	The malware name in lower case.
Malware	malware.version	String	A version string for an identified artifact generation, e.g. a crime-ware kit.
Misp	misp.attribute_uuid	LowercaseString	MISP - Malware Information Sharing Platform & Threat Sharing UUID of an attribute.
Misp	misp.event_uuid	LowercaseString	MISP - Malware Information Sharing Platform & Threat Sharing UUID.
	output	JSON	Event data converted into foreign format, intended to be exported by output plugin.
Protocol	protocol.application	LowercaseString	e.g. vnc, ssh, sip, irc, http or smtp.
Protocol	protocol.transport	LowercaseString	e.g. tcp, udp, icmp.
	raw	Base64	The original line of the event from encoded in base64.
	rtir_id	Integer	Request Tracker Incident Response ticket id.
	screenshot_url	URL	Some source may report URLs related to a an image generated of a resource without any metadata. Or an URL pointing to resource, which has been rendered into a webshot, e.g. a PNG image and the relevant metadata related to its retrieval/generation.
Source	source.abuse_contact	LowercaseString	Abuse contact for source address. A comma separated list.
Source	source.account	String	An account name or email address, which has been identified to relate to the source of an abuse event.
Source	source.allocated	DateTime	Allocation date corresponding to BGP prefix.
Source	source.as_name	String	The autonomous system name from which the connection originated.
Source	source.asn	ASN	The autonomous system number from which originated the connection.
Source	source.domain_suffix	FQDN	The suffix of the domain from the public suffix list.
Source	source.fqdn	FQDN	A DNS name related to the host from which the connection originated. DNS allows even binary data in DNS, so we have to allow everything. A final point is stripped, string is converted to lower case characters.
Source Geolocation	source.geolocation.cc	UppercaseString	Country-Code according to ISO3166-1 alpha-2 for the source IP.
Source Geolocation	source.geolocation.city	String	Some geolocation services refer to city-level geolocation.
Source Geolocation	source.geolocation.country	String	The country name derived from the ISO3166 country code (assigned to cc field).
Source Geolocation	source.geolocation.cymru_cc	UppercaseString	The country code denoted for the ip by the Team Cymru asn to ip mapping service.
Source Geolocation	source.geolocation.geoip_cc	UppercaseString	MaxMind Country Code (ISO3166-1 alpha-2).
Source Geolocation	source.geolocation.latitude	Float	Latitude coordinates derived from a geolocation service, such as MaxMind geoip db.
Source Geolocation	source.geolocation.longitude	Float	Longitude coordinates derived from a geolocation service, such as MaxMind geoip db.
Source Geolocation	source.geolocation.region	String	Some geolocation services refer to region-level geolocation.
Source Geolocation	source.geolocation.state	String	Some geolocation services refer to state-level geolocation.
Source	source.ip	IPAddress	The ip observed to initiate the connection
Source	source.local_hostname	String	Some sources report a internal hostname within a NAT related to the name configured for a compromised system
Source	source.local_ip	IPAddress	Some sources report a internal (NATed) IP address related a compromised system. N.B. RFC1918 IPs are OK here.
Source	source.network	IPNetwork	CIDR for an autonomous system. Also known as BGP prefix. If multiple values are possible, select the most specific.
Source	source.port	Integer	The port from which the connection originated.
Source	source.registry	Registry	The IP registry a given ip address is allocated by.
Source	source.reverse_dns	FQDN	Reverse DNS name acquired through a reverse DNS query on an IP address. N.B. Record types other than PTR records may also appear in the reverse DNS tree. Furthermore, unfortunately, there is no rule prohibiting people from writing anything in a PTR record. Even JavaScript will work. A final point is stripped, string is converted to lower case characters.
Source	source.tor_node	Boolean	If the source IP was a known tor node.
Source	source.url	URL	A URL denotes an IOC, which refers to a malicious resource, whose interpretation is defined by the abuse type. A URL with the abuse type phishing refers to a phishing resource.
Source	source.urlpath	String	The path portion of an HTTP or related network request.
	status	String	Status of the malicious resource (phishing, dropzone, etc), e.g. online, offline.
Time	time.observation	DateTime	The time the collector of the local instance processed (observed) the event.
Time	time.source	DateTime	The time of occurrence of the event as reported the feed (source).
	tlp	TLP	Traffic Light Protocol level of the event.

IntelMQ¶

General information¶

Introduction¶

About¶

Usage¶

IntelMQ Manager¶

Contribute¶

User guide¶