Skip to content
Snippets Groups Projects
Select Git revision
  • 3adf6c6fe312d6b3e09de7a6647aa72f2a7e5a47
  • main default protected
  • idacs_experiments
  • postgres_idacs
  • memgraph_fabian
  • postgres_julian
  • dev_ma
  • test_results
  • FZ_changed_code
  • FZ_Memgraph
10 results

streaming

Name Last commit Last update
..
clients
obsolet
mosquitto.conf
readme.md

Streaming

Ablauf:

  1. Wähle Random CDM Daten aus einem firstK Datensatz.
  2. Publishe Daten an MQTT Broker
  3. Subscriber parsed CDM Daten in Query für Einfüge Operationen.
  4. Subscriber führen Query aus bei Erhalt neuer Nachricht.

Bauen eines Dockercontainers zu einem Publisher

Befehl: docker build -t image_name directory dockerfile

Hinzufügen existierender Nodes in Neo4j

In Neo4j werden Nodes bei CREATE Query nicht aktualisiert, sondern hinzugefügt. Das ist beim Streaming sehr nützlich, da beliebig neue Daten erzeugt werden können.

Publisher Mapping vom CDM zu DB Format

Wie werden die Daten vom CDM in das jeweilige Datenformat gemappt?

Neo4j

Im existierenden Ansatz aus dem MA Projekt wurde aus dem Typ des Eintrags (Event, FileObject, Host ... ) die Node-Label.

Mögliche Typen (Node Label) sind die "Klassen" des CDM: Event Subject Object (File, Memory, Netflow) Value Host Tag Principal FetFlowObject

Die Relationships zwischen Nodes ergeben sich aus den Beziehungen im CDM Modell

Event -isGeneratedBy-> Subject Event -affects-> Object Event -affects-> Value Event -affects-> Subject

Value -affects-> Event

Object -affects-> Event Object -residesOn-> Host Object -isPartOf-> Object Object -hasOwningPrincipal-> Principal Object -hasTag-> Tag

Subject -hasParent-> Subject Subject -affects-> Event Subject -runsOn-> Host Subject -hasLocalPrincipal-> Principal

Principal -hasAccountOn-> Host

Aufbau der Event als JSON

Netflow data = { "datum":{ "com.bbn.tc.schema.avro.cdm18.NetFlowObject":{ # Label "uuid":"6D6E3E08-37DD-11E8-BF66-D9AA8AFF4A69", # Property "baseObject":{ "hostId":"83C8ED1F-5045-DBCD-B39F-918F0DF4F851", "permission":null, "epoch":null, "properties":{"map":{}} }, # Property "localAddress":"localhost", # Property "localPort":-1, # Property "remoteAddress":"128.55.12.10", # Property "remotePort":53, # Property "ipProtocol":null, # Property "fileDescriptor":null # Property } }, "CDMVersion":"18", # Property "source":"SOURCE_FREEBSD_DTRACE_CADETS" # Property }

# SrcSinkObject
data = {
    "datum":{
        "com.bbn.tc.schema.avro.cdm18.SrcSinkObject":{ # Label
            "uuid":"797A0981-37DD-11E8-BF66-D9AA8AFF4A69", # Property
            "baseObject":{
                "hostId":"83C8ED1F-5045-DBCD-B39F-918F0DF4F851", 
                "permission":null,
                "epoch":null,
                "properties":{"map":{}}
            }, # Property
            "type":"SRCSINK_IPC", # Property
            "fileDescriptor":null # Property
        }
    },
    "CDMVersion":"18", # Property
    "source":"SOURCE_FREEBSD_DTRACE_CADETS" # Property
}

# UnnamedPipeObject
data = {
    "datum":{
        "com.bbn.tc.schema.avro.cdm18.UnnamedPipeObject":{ # Label
            "uuid":"856390BF-CEB5-5426-A7FD-99CFC5AFB505", # Property
            "baseObject":{
                "hostId":"83C8ED1F-5045-DBCD-B39F-918F0DF4F851", 
                "permission":null,
                "epoch":null,
                "properties":{"map":{}}
            }, # Property
            "sourceFileDescriptor":null, # Property
            "sinkFileDescriptor":null, # Property
            "sourceUUID":{
                "com.bbn.tc.schema.avro.cdm18.UUID":"6CAC3E51-37DD-11E8-BF66-D9AA8AFF4A69"
            }, # affects
            "sinkUUID":{
                "com.bbn.tc.schema.avro.cdm18.UUID":"6CAC3E7B-37DD-11E8-BF66-D9AA8AFF4A69"
            } # affects2
        }
    },
    "CDMVersion":"18",  # Property
    "source":"SOURCE_FREEBSD_DTRACE_CADETS" # Property
}

# FileObject
data = {
    "datum":{
        "com.bbn.tc.schema.avro.cdm18.FileObject":{ # Label
            "uuid":"42DD2C9E-36C2-11E8-BF66-D9AA8AFF4A69", # Property
            "baseObject":{
                "hostId":"83C8ED1F-5045-DBCD-B39F-918F0DF4F851",
                "permission":null,
                "epoch":null,
                "properties":{"map":{}}
            }, # Property
            "type":"FILE_OBJECT_FILE", # Label??
            "fileDescriptor":null, # Property
            "localPrincipal":null, # Relationship: hasOwningPrincipal
            "size":null, # Property
            "peInfo":null, # Property
            "hashes":null # Property
        }
    },
    "CDMVersion":"18",  # Property
    "source":"SOURCE_FREEBSD_DTRACE_CADETS" # Property
}

# Principal
data = {
    "datum":{
        "com.bbn.tc.schema.avro.cdm18.Principal":{ # Label
            "uuid":"7DCA248E-1BBA-59F5-9227-B25D5F253594",  # Property
            "type":"PRINCIPAL_LOCAL", # Property
            "hostId":"83C8ED1F-5045-DBCD-B39F-918F0DF4F851", # Relationship: hasAccountOn
            "userId":"0", # Property
            "username":{"string":"root"}, # Property
            "groupIds":[], # Property
            "properties":{"map":{}} # Property
        }
    },
    "CDMVersion":"18", # Property
    "source":"SOURCE_FREEBSD_DTRACE_CADETS" # Property
    }

# Host
data = {
    "datum":{
        "com.bbn.tc.schema.avro.cdm18.Host":{ # Label
            "uuid":"83C8ED1F-5045-DBCD-B39F-918F0DF4F851", # Property
            "hostName":"ta1-cadets", # Property
            "hostIdentifiers":[], # Property
            "osDetails":"FreeBSD 12.0-CURRENT FreeBSD 12.0-CURRENT #1 1863588dca9(HEAD)-dirty: Wed Feb 28 17:23:37 UTC 2018     root@ta1-cadets:/usr/obj/data/update/build-meta/freebsd/amd64.amd64/sys/CADETS  amd64",  # Property
            "hostType":"HOST_DESKTOP", # Property
            "interfaces":[
                {"name":"vtnet0","macAddress":"52:54:00:f0:0d:23",
                "ipAddresses":["fe80::5054:ff:fef0:d23%vtnet0","10.0.6.23"]},
                {"name":"vtnet1","macAddress":"52:54:00:f0:08:23",
                 "ipAddresses":["fe80::5054:ff:fef0:823%vtnet1","128.55.12.73"]}
            ]  # Property
        }
    },
    "CDMVersion":"18", # Property
    "source":"SOURCE_FREEBSD_DTRACE_CADETS" # Property
    }

# Subject
data = {
    "datum":{
        "com.bbn.tc.schema.avro.cdm18.Subject":{ # Label
            "uuid":"4633B892-36C2-11E8-BF66-D9AA8AFF4A69",  # Property
            "type":"SUBJECT_PROCESS", # Property
            "cid":2807, # Property
            "parentSubject":{"com.bbn.tc.schema.avro.cdm18.UUID":"4B42F341-36C0-11E8-BF66-D9AA8AFF4A69"}, # Relationship: hasParent
            "hostId":"83C8ED1F-5045-DBCD-B39F-918F0DF4F851", # Relationship: runsOn
            "localPrincipal":"26B62C58-E245-578A-AE04-495EBA139703", # Relationship: hasLocalPrincipal
            "startTimestampNanos":1522706866413351857, # Property
            "unitId":null, # Property
            "iteration":null, # Property
            "count":null, # Property
            "cmdLine":null, # Property
            "privilegeLevel":null # Property
            ,"importedLibraries":null # Property
            ,"exportedLibraries":null # Property
            ,"properties":{"map":{"host":"83c8ed1f-5045-dbcd-b39f-918f0df4f851"}} # Property
        }
    },
    "CDMVersion":"18",
    "source":"SOURCE_FREEBSD_DTRACE_CADETS"
}

# Event
data = {
    "datum":{
        "com.bbn.tc.schema.avro.cdm18.Event":{ # Label
            "uuid":"08DB617B-6100-51F3-9742-902710EDCA7D", # Property
            "sequence":{"long":4600591}, # Property
            "type":"EVENT_FCNTL", # Property
            "threadId":{"int":100117}, # Property
            "hostId":"83C8ED1F-5045-DBCD-B39F-918F0DF4F851", # Relationship: runsOn
            "subject":{"com.bbn.tc.schema.avro.cdm18.UUID":"72FB0406-3678-11E8-BF66-D9AA8AFF4A69"}, # Relationship: isGeneratedBy
            "predicateObject":null, # Relationship: affects1
            "predicateObjectPath":null, # Property
            "predicateObject2":null, # Relationship: affects2
            "predicateObject2Path":null, # Property
            "timestampNanos":1522828473820631110, # Property
            "name":{"string":"aue_fcntl"}, # Property
            "parameters":{
                "array":[{
                    "size":-1,
                    "type":"VALUE_TYPE_CONTROL",
                    "valueDataType":"VALUE_DATA_TYPE_INT",
                    "isNull":false,
                    "name":{"string":"cmd"},
                    "runtimeDataType":null,
                    "valueBytes":{"bytes":"04"},
                    "provenance":null,
                    "tag":null,
                    "components":null
                }]
            },  # Property
            "location":null,  # Property
            "size":null, # Property
            "programPoint":null, # Property
            "properties":{
                "map":{
                    "host":"83c8ed1f-5045-dbcd-b39f-918f0df4f851",
                    "return_value":"0",
                    "fd":"4",
                    "exec":"python2.7",
                    "ppid":"1"
                }
            } # Property
        }
    },
    "CDMVersion":"18",  # Property
    "source":"SOURCE_FREEBSD_DTRACE_CADETS"  # Property
}