Streaming
Ablauf:
- Wähle Random CDM Daten aus einem firstK Datensatz.
- Publishe Daten an MQTT Broker
- Subscriber parsed CDM Daten in Query für Einfüge Operationen.
- Subscriber führen Query aus bei Erhalt neuer Nachricht.
Bauen eines Dockercontainers zu einem Publisher
Befehl: docker build -t image_name directory dockerfile
Hinzufügen existierender Nodes in Neo4j
In Neo4j werden Nodes bei CREATE Query nicht aktualisiert, sondern hinzugefügt. Das ist beim Streaming sehr nützlich, da beliebig neue Daten erzeugt werden können.
Publisher Mapping vom CDM zu DB Format
Wie werden die Daten vom CDM in das jeweilige Datenformat gemappt?
Neo4j
Im existierenden Ansatz aus dem MA Projekt wurde aus dem Typ des Eintrags (Event, FileObject, Host ... ) die Node-Label.
Mögliche Typen (Node Label) sind die "Klassen" des CDM: Event Subject Object (File, Memory, Netflow) Value Host Tag Principal FetFlowObject
Die Relationships zwischen Nodes ergeben sich aus den Beziehungen im CDM Modell
Event -isGeneratedBy-> Subject Event -affects-> Object Event -affects-> Value Event -affects-> Subject
Value -affects-> Event
Object -affects-> Event Object -residesOn-> Host Object -isPartOf-> Object Object -hasOwningPrincipal-> Principal Object -hasTag-> Tag
Subject -hasParent-> Subject Subject -affects-> Event Subject -runsOn-> Host Subject -hasLocalPrincipal-> Principal
Principal -hasAccountOn-> Host
Aufbau der Event als JSON
Netflow data = { "datum":{ "com.bbn.tc.schema.avro.cdm18.NetFlowObject":{ # Label "uuid":"6D6E3E08-37DD-11E8-BF66-D9AA8AFF4A69", # Property "baseObject":{ "hostId":"83C8ED1F-5045-DBCD-B39F-918F0DF4F851", "permission":null, "epoch":null, "properties":{"map":{}} }, # Property "localAddress":"localhost", # Property "localPort":-1, # Property "remoteAddress":"128.55.12.10", # Property "remotePort":53, # Property "ipProtocol":null, # Property "fileDescriptor":null # Property } }, "CDMVersion":"18", # Property "source":"SOURCE_FREEBSD_DTRACE_CADETS" # Property }
# SrcSinkObject
data = {
"datum":{
"com.bbn.tc.schema.avro.cdm18.SrcSinkObject":{ # Label
"uuid":"797A0981-37DD-11E8-BF66-D9AA8AFF4A69", # Property
"baseObject":{
"hostId":"83C8ED1F-5045-DBCD-B39F-918F0DF4F851",
"permission":null,
"epoch":null,
"properties":{"map":{}}
}, # Property
"type":"SRCSINK_IPC", # Property
"fileDescriptor":null # Property
}
},
"CDMVersion":"18", # Property
"source":"SOURCE_FREEBSD_DTRACE_CADETS" # Property
}
# UnnamedPipeObject
data = {
"datum":{
"com.bbn.tc.schema.avro.cdm18.UnnamedPipeObject":{ # Label
"uuid":"856390BF-CEB5-5426-A7FD-99CFC5AFB505", # Property
"baseObject":{
"hostId":"83C8ED1F-5045-DBCD-B39F-918F0DF4F851",
"permission":null,
"epoch":null,
"properties":{"map":{}}
}, # Property
"sourceFileDescriptor":null, # Property
"sinkFileDescriptor":null, # Property
"sourceUUID":{
"com.bbn.tc.schema.avro.cdm18.UUID":"6CAC3E51-37DD-11E8-BF66-D9AA8AFF4A69"
}, # affects
"sinkUUID":{
"com.bbn.tc.schema.avro.cdm18.UUID":"6CAC3E7B-37DD-11E8-BF66-D9AA8AFF4A69"
} # affects2
}
},
"CDMVersion":"18", # Property
"source":"SOURCE_FREEBSD_DTRACE_CADETS" # Property
}
# FileObject
data = {
"datum":{
"com.bbn.tc.schema.avro.cdm18.FileObject":{ # Label
"uuid":"42DD2C9E-36C2-11E8-BF66-D9AA8AFF4A69", # Property
"baseObject":{
"hostId":"83C8ED1F-5045-DBCD-B39F-918F0DF4F851",
"permission":null,
"epoch":null,
"properties":{"map":{}}
}, # Property
"type":"FILE_OBJECT_FILE", # Label??
"fileDescriptor":null, # Property
"localPrincipal":null, # Relationship: hasOwningPrincipal
"size":null, # Property
"peInfo":null, # Property
"hashes":null # Property
}
},
"CDMVersion":"18", # Property
"source":"SOURCE_FREEBSD_DTRACE_CADETS" # Property
}
# Principal
data = {
"datum":{
"com.bbn.tc.schema.avro.cdm18.Principal":{ # Label
"uuid":"7DCA248E-1BBA-59F5-9227-B25D5F253594", # Property
"type":"PRINCIPAL_LOCAL", # Property
"hostId":"83C8ED1F-5045-DBCD-B39F-918F0DF4F851", # Relationship: hasAccountOn
"userId":"0", # Property
"username":{"string":"root"}, # Property
"groupIds":[], # Property
"properties":{"map":{}} # Property
}
},
"CDMVersion":"18", # Property
"source":"SOURCE_FREEBSD_DTRACE_CADETS" # Property
}
# Host
data = {
"datum":{
"com.bbn.tc.schema.avro.cdm18.Host":{ # Label
"uuid":"83C8ED1F-5045-DBCD-B39F-918F0DF4F851", # Property
"hostName":"ta1-cadets", # Property
"hostIdentifiers":[], # Property
"osDetails":"FreeBSD 12.0-CURRENT FreeBSD 12.0-CURRENT #1 1863588dca9(HEAD)-dirty: Wed Feb 28 17:23:37 UTC 2018 root@ta1-cadets:/usr/obj/data/update/build-meta/freebsd/amd64.amd64/sys/CADETS amd64", # Property
"hostType":"HOST_DESKTOP", # Property
"interfaces":[
{"name":"vtnet0","macAddress":"52:54:00:f0:0d:23",
"ipAddresses":["fe80::5054:ff:fef0:d23%vtnet0","10.0.6.23"]},
{"name":"vtnet1","macAddress":"52:54:00:f0:08:23",
"ipAddresses":["fe80::5054:ff:fef0:823%vtnet1","128.55.12.73"]}
] # Property
}
},
"CDMVersion":"18", # Property
"source":"SOURCE_FREEBSD_DTRACE_CADETS" # Property
}
# Subject
data = {
"datum":{
"com.bbn.tc.schema.avro.cdm18.Subject":{ # Label
"uuid":"4633B892-36C2-11E8-BF66-D9AA8AFF4A69", # Property
"type":"SUBJECT_PROCESS", # Property
"cid":2807, # Property
"parentSubject":{"com.bbn.tc.schema.avro.cdm18.UUID":"4B42F341-36C0-11E8-BF66-D9AA8AFF4A69"}, # Relationship: hasParent
"hostId":"83C8ED1F-5045-DBCD-B39F-918F0DF4F851", # Relationship: runsOn
"localPrincipal":"26B62C58-E245-578A-AE04-495EBA139703", # Relationship: hasLocalPrincipal
"startTimestampNanos":1522706866413351857, # Property
"unitId":null, # Property
"iteration":null, # Property
"count":null, # Property
"cmdLine":null, # Property
"privilegeLevel":null # Property
,"importedLibraries":null # Property
,"exportedLibraries":null # Property
,"properties":{"map":{"host":"83c8ed1f-5045-dbcd-b39f-918f0df4f851"}} # Property
}
},
"CDMVersion":"18",
"source":"SOURCE_FREEBSD_DTRACE_CADETS"
}
# Event
data = {
"datum":{
"com.bbn.tc.schema.avro.cdm18.Event":{ # Label
"uuid":"08DB617B-6100-51F3-9742-902710EDCA7D", # Property
"sequence":{"long":4600591}, # Property
"type":"EVENT_FCNTL", # Property
"threadId":{"int":100117}, # Property
"hostId":"83C8ED1F-5045-DBCD-B39F-918F0DF4F851", # Relationship: runsOn
"subject":{"com.bbn.tc.schema.avro.cdm18.UUID":"72FB0406-3678-11E8-BF66-D9AA8AFF4A69"}, # Relationship: isGeneratedBy
"predicateObject":null, # Relationship: affects1
"predicateObjectPath":null, # Property
"predicateObject2":null, # Relationship: affects2
"predicateObject2Path":null, # Property
"timestampNanos":1522828473820631110, # Property
"name":{"string":"aue_fcntl"}, # Property
"parameters":{
"array":[{
"size":-1,
"type":"VALUE_TYPE_CONTROL",
"valueDataType":"VALUE_DATA_TYPE_INT",
"isNull":false,
"name":{"string":"cmd"},
"runtimeDataType":null,
"valueBytes":{"bytes":"04"},
"provenance":null,
"tag":null,
"components":null
}]
}, # Property
"location":null, # Property
"size":null, # Property
"programPoint":null, # Property
"properties":{
"map":{
"host":"83c8ed1f-5045-dbcd-b39f-918f0df4f851",
"return_value":"0",
"fd":"4",
"exec":"python2.7",
"ppid":"1"
}
} # Property
}
},
"CDMVersion":"18", # Property
"source":"SOURCE_FREEBSD_DTRACE_CADETS" # Property
}