2017 12 lutter es and kibana dockerized

docker/ESKibanaManager.sh

@@ -0,0 +1,99 @@
+#!/bin/bash
+
+# working dir is the directory the script is located in
+WD="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+cd "$WD"
+WEB_ENV="kibana elasticsearch"
+
+CONF="-p pcapscan -f docker-compose.yml "
+
+function exitIfErr() {
+	if [ $1 -ne 0 ]; then
+		echo
+		echo "$2"
+		echo "Abort execution."
+		echo
+		exit 1
+	fi
+}
+which docker &> /dev/null
+exitIfErr $? "Docker is noch installed on this system. Please install docker.io package or dockerCE."
+function installComposerIfNotAvailable() {
+        # is docker installed?
+        which docker &> /dev/null
+        exitIfErr $? "Can not find docker executable."
+        # check if docker composer is installed
+        which docker-composer &> /dev/null
+        if [ $? -eq 0 ]; then
+                echo "Docker composer is available from PATH: $( which docker-composer )."
+                return
+        elif [ -f "/usr/local/bin/docker-compose" ]; then
+                echo "Docker composer is available in /usr/local/bin/docker-compose"
+                return
+        fi
+        # download it and put it into /usr/local/
+        echo sudo curl -L "https://github.com/docker/compose/releases/download/1.18.0/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
+        sudo curl -L "https://github.com/docker/compose/releases/download/1.18.0/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
+        exitIfErr $? "Failed to download and install docker-compose."
+        # make executeable
+        sudo chmod +x /usr/local/bin/docker-compose
+        echo "Successfully downloaded and installed docker-composer from source."
+}
+function getAllComposeServiceNames() {
+        docker-compose $CONF config --services | head -c -1
+}
+function getAllComposeServiceNamesOneliner() {
+        getAllComposeServiceNames | tr '\n' '|'
+}
+# make sure docker-composer is available
+installComposerIfNotAvailable
+
+function usage() {
+        SERVICES="[$(getAllComposeServiceNamesOneliner)]"
+	echo "Usage:"
+	echo "$0  [start|stop|restart|list|shell|log|clear] "
+	echo
+	echo "$0 shell $SERVICES"
+	echo "$0 log $SERVICES"
+	exit 1
+}
+
+
+CONF="-f docker-compose-dev.yml -p pcapscan"
+
+if [ "$1" == "start" ]; then
+	docker-compose $CONF up -d
+	echo
+	echo "Endpoints:"
+	echo "elasticsearch:	http://localhost:9200"
+	echo "kibana:       	http://localhost:5601"
+	echo
+elif [ "$1" == "stop" ]; then
+	docker-compose $CONF down
+elif [ "$1" == "list" ]; then
+	docker-compose $CONF ps
+elif [ "$1" == "restart" ]; then
+	./$0 stop
+	./$0 start
+elif [ "$1" == "log" ]; then
+	docker-compose $CONF logs -f "$2"
+elif [ "$1" == "shell" ]; then
+	docker compose $CONF exec "$2" bash
+elif [ "$1" == "clear" ]; then
+	echo
+	echo "Clear elasticsearch index?"
+	echo
+	read -r -p "Are you sure? [y/N] " response
+	echo
+	if [[ ! $response =~ ^(yes|y)$ ]]; then
+    	echo "Got $response from user. Abort processing."
+  	exit 1
+	else
+    	echo "Ok, I'll do it."
+	fi
+	docker volume rm pcapscan_stats-elasticsearch
+else
+	echo "Unknown parameter $1. "
+	echo
+	usage
+fi

docker/docker-compose-dev.yml

@@ -0,0 +1,38 @@
+version: "2"
+services:
+
+# kibana + elastic search for global statistics
+  kibana:
+#    image: docker.elastic.co/kibana/kibana:5.3.0
+    image: kibana:5.3.0
+    links:
+      - elasticsearch
+    ports:
+      - "5601:5601"
+
+# kibana database backend elastic search
+  elasticsearch:
+    image: elasticsearch:5.3.0
+    ports:
+      - "9200:9200"
+#    image: docker.elastic.co/elasticsearch/elasticsearch:5.3.0
+#    environment:
+#      - bootstrap.memory_lock=true
+#      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+#      - "node.max_local_storage_nodes=4"
+#    ulimits:
+#      memlock:
+#        soft: -1
+#        hard: -1
+#      nofile:
+#        soft: 65536
+#        hard: 65536
+#    mem_limit: 1g
+#    cap_add:
+#      - IPC_LOCK
+    volumes:
+      - stats-elasticsearch:/usr/share/elasticsearch/data
+
+# definition of volumens (globally)
+volumes:
+  stats-elasticsearch:

pcapscanner/main.py

@@ -13,16 +13,10 @@
 import time
 from multiprocessing import Pool
 
-from analyzers import hosts, conversations
 import pcap
 
 NUM_THREADS = 4
 
-ANALYZERS = [
-    hosts,
-    conversations
-]
-
 ASCII_LOGO = """
 
 @@@@@@@    @@@@@@@   @@@@@@   @@@@@@@    @@@@@@    @@@@@@@   @@@@@@   @@@  @@@
@@ -38,9 +32,10 @@
 
 """
 
+
 class Main:
 
-    def __init__(self, outputdir, inputdir, parser):
+    def __init__(self, outputdir, inputdir):
 
         # log files
         self.outputdir = outputdir
@@ -59,12 +54,6 @@ def __init__(self, outputdir, inputdir, parser):
             )
         self.inputdir = inputdir
 
-        # initialize all analyzers
-        for a in ANALYZERS:
-            a.init()
-
-        self.parser = parser
-
     def _log_errors(self):
         if not self.ignoredFiles:
             return
@@ -75,9 +64,6 @@ def _log_errors(self):
 
         print("ignored {} files".format(len(self.ignoredFiles)))
 
-    def _log_results(self):
-        for a in ANALYZERS:
-            a.log(self.outputdir)
 
     def start(self):
         pcapfiles = pcap.walk(self.inputdir)
@@ -98,7 +84,7 @@ def start(self):
                 # asynchronously
                 pool.apply_async(
                     pcap.process_pcap,
-                    (fn, [a.analyze for a in ANALYZERS], progressbar_position, self.parser)
+                    (fn, progressbar_position)
                 )
 
             # close pool
@@ -108,7 +94,6 @@ def start(self):
             pool.join()
 
         self._log_errors()
-        self._log_results()
 
         # return number of pcap files
         return len(pcapfiles)
@@ -128,20 +113,13 @@ def start(self):
         default='.',
         help='path to the output directory'
     )
-    parser.add_argument(
-        '-p', '--parser',
-        nargs='?',
-        default=pcap.Parser.DPKT.name,
-        choices=[p.name for p in pcap.Parser]
-    )
 
     args = parser.parse_args()
     print(ASCII_LOGO)
 
     scanner = Main(
         outputdir=args.outputdir,
-        inputdir=args.inputdir,
-        parser=args.parser
+        inputdir=args.inputdir
     )
     # measure time
     startTime = time.time()