fileIngestModule.py

import jarray
import inspect
from java.lang import System
from java.util.logging import Level
from org.sleuthkit.datamodel import Score   
from org.sleuthkit.datamodel import SleuthkitCase
from org.sleuthkit.datamodel import AbstractFile
from org.sleuthkit.datamodel import ReadContentInputStream
from org.sleuthkit.datamodel import BlackboardArtifact
from org.sleuthkit.datamodel import BlackboardAttribute
from org.sleuthkit.datamodel import TskData
from org.sleuthkit.autopsy.ingest import IngestModule
from org.sleuthkit.autopsy.ingest.IngestModule import IngestModuleException
from org.sleuthkit.autopsy.ingest import DataSourceIngestModule
from org.sleuthkit.autopsy.ingest import FileIngestModule
from org.sleuthkit.autopsy.ingest import IngestModuleFactoryAdapter
from org.sleuthkit.autopsy.ingest import IngestMessage
from org.sleuthkit.autopsy.ingest import IngestServices
from org.sleuthkit.autopsy.ingest import ModuleDataEvent
from org.sleuthkit.autopsy.coreutils import Logger
from org.sleuthkit.autopsy.casemodule import Case
from org.sleuthkit.autopsy.casemodule.services import Services
from org.sleuthkit.autopsy.casemodule.services import FileManager
from org.sleuthkit.autopsy.casemodule.services import Blackboard
from java.util import ArrayList

from csv import DictWriter, reader
import os
import datetime
from Message import Message
from unpack import unpack
'''from struct import unpack as up

def unpack(type, bytes):
    return up(type,bytes)[0]'''

# Factory that defines the name and details of the module and allows Autopsy
# to create instances of the modules that will do the analysis.
class JythonDATFileIngestModuleFactory(IngestModuleFactoryAdapter):

    moduleName = "DAT file ingest Module"

    def getModuleDisplayName(self):
        return self.moduleName

    def getModuleDescription(self):
        return "DAT file module that does parses binary files for IMU and GPS records to analyse"

    def getModuleVersionNumber(self):
        return "1.0"

    # Return true if module wants to get called for each file
    def isFileIngestModuleFactory(self):
        return True

    # can return null if isFileIngestModuleFactory returns false
    def createFileIngestModule(self, ingestOptions):
        return JythonDATFileIngestModule()


# File-level ingest module.  One gets created per thread.
# Looks at the attributes of the passed in file.
class JythonDATFileIngestModule(FileIngestModule):

    _logger = Logger.getLogger(JythonDATFileIngestModuleFactory.moduleName)

    def log(self, level, msg):
        self._logger.logp(level, self.__class__.__name__, inspect.stack()[1][3], msg)

    def __init__(self):
        self.context = None

    # Where any setup and configuration is done
    # 'context' is an instance of org.sleuthkit.autopsy.ingest.IngestJobContext.
    # See: http://sleuthkit.org/autopsy/docs/api-docs/latest/classorg_1_1sleuthkit_1_1autopsy_1_1ingest_1_1_ingest_job_context.html
    def startUp(self, context):
        self.context = context
        self.filesFound = 0
        self.skCase = Case.getCurrentCase().getSleuthkitCase()
        self.artificatTypeID = self.skCase.addBlackboardArtifactType("TSK_DRONE_DATA", "Drone Data")

        # Throw an IngestModule.IngestModuleException exception if there was a problem setting up
        # raise IngestModuleException(" ")
        pass

    # Where the analysis is done.  Each file will be passed into here.
    # The 'file' object being passed in is of type org.sleuthkit.datamodel.AbstractFile.
    # See: http://www.sleuthkit.org/sleuthkit/docs/jni-docs/latest/classorg_1_1sleuthkit_1_1datamodel_1_1_abstract_file.html
    def process(self, file):
        # Skip non-files
        if ((file.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS) or
            (file.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.UNUSED_BLOCKS) or
            (file.isFile() == False)):
            return IngestModule.ProcessResult.OK

        if file.getName().lower().endswith(".dat"):
            '''inputStream = ReadContentInputStream(file)
            try:
                file_header = jarray.zeros(128, "b")
                inputStream.read(file_header, 0, 128)
                if file_header[16:21] != b'BUILD':
                    #nonDatFiles += 1
                    raise IngestModuleException('Not DAT file '+file.getName())
            except IngestModuleException as e:
                self.log(Level.INFO, str(e))
            '''
            in_fn = file.getLocalAbsPath()
            in_file = open(in_fn, 'rb')
            self.log(Level.INFO, "Input file path: " + str(in_fn))
            file_header = in_file.read(128)
            ##### str(map(ord,file_header[16:21])) == str(bytes(map(ord,file_header[16:21])))  := True
            try:
                if file_header[16:21] != b"BUILD":
                    nonDatFiles += 1
                    raise IngestModuleException('Not DAT file '+in_fn)
            except IngestModuleException as e:
                self.log(Level.INFO, str(e))
                
            self.filesFound += 1

            #Sha256Hash = 
            #Md5Hash =      ### calculate hashes
            #self.log(Level.INFO, "File SHA256 hash: " + str(Sha256Hash))
            #self.log(Level.INFO, "File MD5 hash: " + str(Md5Hash))

            ######################################################
            dir = "C:/Users/jfbaf/AppData/Roaming/autopsy/python_modules/DROP-analysis" # os.path.abspath(os.path.join(os.getcwd(), os.pardir))
            out_fn = os.path.join(dir,file.getName()[:-4]+"-output.csv")        ## ??permission denied
            
            self.log(Level.INFO, "Found a DAT file: " + file.getName()+ "\n\tFile size: " + str(file.getSize()) + "\n\tOutput file: " + out_fn)
            out_file = open(out_fn, 'w')
            writer = DictWriter(out_file, lineterminator='\n', fieldnames=Message.fieldnames)
            writer.writeheader()

            byte = in_file.read(1)
            message = Message(lambda msg : self.log(Level.INFO, msg))

            corruptPackets = 0   # keeps track of the number of corrupt packets - data blocks that do not meet the minimum formatting requirements to be a DJI flight data packet
            unparsedPackets = 0
            incorrectStart = 0

            count = 0
            start_issue = True
            #while readLen > 0:
            while len(byte) != 0: # and count < 100:
                #count += 1
                try:
                    if byte != b'\x55':
                        incorrectStart += 1
                        byte = in_file.read(1)
                        if start_issue:
                            start_issue = False
                            raise IngestModuleException("No new packet error")
                        continue
                        
                    start_issue = True
                    pktlen = 0xFF & ord(in_file.read(1))    # length of the packet
                    padding = in_file.read(1)
                    if padding == b'\x00':
                        header = in_file.read(7)
                        header = list(map(chr,map(ord,header)))
                        current = in_file.tell()
                        in_file.seek(current + pktlen - 10)
                        next_start = in_file.read(1)
                        if len(next_start) <= 0: break
                        if next_start != b'\x55':          # something is wrong with the packet length
                            in_file.seek(current)       # reset file pointer to just after header
                            byte = in_file.read(1)
                            raise IngestModuleException("Packet length error at byte " + str(current-9))
                        in_file.seek(current)       # go back to where we were if packet length is ok
                        
                        if pktlen <= 10:
                            byte = padding
                            raise IngestModuleException("pktlen equals 0")
                        thisPacketTickNo = unpack('I', header[3:7])
                        if thisPacketTickNo < 0:
                            byte = padding
                            corruptPackets += 1
                            raise IngestModuleException("Corrupted tick number. Tick No: " + str(thisPacketTickNo))
                        if message.tickNo == None:
                            message.setTickNo(thisPacketTickNo)
                        
                        payload = in_file.read(pktlen - 10)
                        payload = list(map(chr,map(ord,payload)))

                        message.writeRow(writer, thisPacketTickNo)
                        if message.addPacket(pktlen, header, payload) == False:
                            unparsedPackets += 1

                        byte = in_file.read(1)
                    else:
                        byte = padding
                except IngestModuleException as e:
                    continue
                
            writer.writerow(message.getRow())           # write the last row

            self.log(Level.INFO, "Corrupt packets: "+str(corruptPackets) + "\n\tUnparsed packets: "+str(unparsedPackets) + "\n\tIncorrect starts: "+str(incorrectStart))

            ##### check file hash value

            in_file.close()
            out_file.close()
            
            skCase = self.skCase
            artID = self.artificatTypeID
            moduleName = JythonDATFileIngestModuleFactory.moduleName
            
            with open(out_fn) as csv_file:
                csvReader = reader(csv_file, delimiter=',')
                firstLine = True
                for row in csvReader:
                    if firstLine: 
                        firstLine = False
                    else:
                        attributes = ArrayList()

                        attributes.add(BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_MSG_ID, moduleName, row[0]))
                        attributes.add(BlackboardAttribute(self.checkAttribute("TSK_DAT_OFFSETTIME", "OffsetTime", skCase), moduleName, row[1]))
                        
                        attributes.add(BlackboardAttribute(self.checkAttribute("TSK_DAT_LONG_DEG", "longDeg", skCase), moduleName, row[4]))
                        attributes.add(BlackboardAttribute(self.checkAttribute("TSK_DAT_LAT_DEG", "latDeg", skCase), moduleName, row[5]))

                        attributes.add(BlackboardAttribute(self.checkAttribute("TSK_DAT_BARO_ALTI", "baroAlti", skCase), moduleName, row[13]))

                        attributes.add(BlackboardAttribute(self.checkAttribute("TSK_DAT_VELN", "velN", skCase), moduleName, row[21]))
                        attributes.add(BlackboardAttribute(self.checkAttribute("TSK_DAT_VELE", "velE", skCase), moduleName, row[22]))
                        attributes.add(BlackboardAttribute(self.checkAttribute("TSK_DAT_VELD", "velD", skCase), moduleName, row[23]))
                        
                        attributes.add(BlackboardAttribute(self.checkAttribute("TSK_DAT_GPS_DATETIME", "GPSDateTime", skCase), moduleName, row[36]))
                        #attributes.add(BlackboardAttribute(self.checkAttribute("TSK_DAT_GPS_TIME", "GPSTime", skCase), moduleName, row[37]))

                        attributes.add(BlackboardAttribute(self.checkAttribute("TSK_DAT_PWM1", "pwm1", skCase), moduleName, row[47]))
                        attributes.add(BlackboardAttribute(self.checkAttribute("TSK_DAT_PWM2", "pwm2", skCase), moduleName, row[48]))
                        attributes.add(BlackboardAttribute(self.checkAttribute("TSK_DAT_PWM3", "pwm3", skCase), moduleName, row[49]))
                        attributes.add(BlackboardAttribute(self.checkAttribute("TSK_DAT_PWM4", "pwm4", skCase), moduleName, row[50]))

                        attributes.add(BlackboardAttribute(self.checkAttribute("TSK_DAT_POSN", "posN", skCase), moduleName, row[68]))
                        attributes.add(BlackboardAttribute(self.checkAttribute("TSK_DAT_POSE", "posE", skCase), moduleName, row[69]))
                        attributes.add(BlackboardAttribute(self.checkAttribute("TSK_DAT_POSD", "posD", skCase), moduleName, row[70]))
                        attributes.add(BlackboardAttribute(self.checkAttribute("TSK_DAT_DIFFVELN", "diffVelN", skCase), moduleName, row[71]))
                        attributes.add(BlackboardAttribute(self.checkAttribute("TSK_DAT_DIFFVELE", "diffVelE", skCase), moduleName, row[72]))
                        attributes.add(BlackboardAttribute(self.checkAttribute("TSK_DAT_DIFFVELD", "diffVelD", skCase), moduleName, row[73]))
                        
                        
                        #attributes.add(BlackboardAttribute(self.checkAttribute("TSK_DAT_", "", skCase), moduleName, row[]))

                        artifact = file.newDataArtifact(artID, attributes)

            try:
                blackboard = Case.getCurrentCase().getSleuthkitCase().getBlackboard()
                blackboard.postArtifact(artifact, moduleName, self.context.getJobId())
            except Blackboard.BlackboardException as ex:
                self.log(Level.SEVERE, "Unable to index blackboard artifact " + str(artifact.getArtifactTypeName()), ex)
                self.log(Level.SEVERE, traceback.format_exc())

            #Case.getCurrentCase().addReport(out_fn, moduleName, "CSV file") # localPath	- The path of the report file, must be in the database directory (case directory in Autopsy) or one of its subdirectories.
        
        return IngestModule.ProcessResult.OK

    # Where any shutdown code is run and resources are freed.
    def shutDown(self):
        numFiles = str(self.filesFound)
        message = IngestMessage.createMessage( IngestMessage.MessageType.DATA, JythonDATFileIngestModuleFactory.moduleName, numFiles + " files found")
        IngestServices.getInstance().postMessage(message)

    def checkAttribute(self, attributeName, attributeDescription, skCase):
        try:
            attID = skCase.addArtifactAttributeType(attributeName, BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING, attributeDescription)
        except:		
            attID = skCase.getAttributeType(attributeName)
        return attID