diff --git a/.vscode/settings.json b/.vscode/settings.json old mode 100644 new mode 100755 diff --git a/README.md b/README.md old mode 100644 new mode 100755 diff --git a/code/eval/first_tests.md b/code/eval/first_tests.md old mode 100644 new mode 100755 diff --git a/code/eval/neo4j/streaming_cadvisor_data_firstK_1000rows_per_sec.png b/code/eval/neo4j/streaming_cadvisor_data_firstK_1000rows_per_sec.png old mode 100644 new mode 100755 diff --git a/code/eval/neo4j/streaming_neo4j_data_firstK_1000rows_per_sec.png b/code/eval/neo4j/streaming_neo4j_data_firstK_1000rows_per_sec.png old mode 100644 new mode 100755 diff --git a/code/infrastructure/build_push_container.sh b/code/infrastructure/build_push_container.sh old mode 100644 new mode 100755 diff --git a/code/infrastructure/monitoring/README.md b/code/infrastructure/monitoring/README.md old mode 100644 new mode 100755 diff --git a/code/infrastructure/monitoring/dashboards/12046_rev2_neo4j_v5.json b/code/infrastructure/monitoring/dashboards/12046_rev2_neo4j_v5.json old mode 100644 new mode 100755 diff --git a/code/infrastructure/monitoring/dashboards/README.md b/code/infrastructure/monitoring/dashboards/README.md old mode 100644 new mode 100755 diff --git a/code/infrastructure/monitoring/grafana-data/alerting/1/__default__.tmpl b/code/infrastructure/monitoring/grafana-data/alerting/1/__default__.tmpl old mode 100644 new mode 100755 diff --git a/code/infrastructure/monitoring/prune_prometheus_data.sh b/code/infrastructure/monitoring/prune_prometheus_data.sh old mode 100644 new mode 100755 diff --git a/code/infrastructure/streaming/clients/pub/Dockerfile b/code/infrastructure/streaming/clients/pub/Dockerfile old mode 100644 new mode 100755 diff --git a/code/infrastructure/streaming/clients/pub/README.md b/code/infrastructure/streaming/clients/pub/README.md old mode 100644 new mode 100755 diff --git a/code/infrastructure/streaming/clients/pub/pub_cdm.py b/code/infrastructure/streaming/clients/pub/pub_cdm.py old mode 100644 new mode 100755 index 4bcc5eb2edb14d3271e2050eeff8faa9ae921838..ee7f4d30cfad64722757d696b5c8991b64255c15 --- a/code/infrastructure/streaming/clients/pub/pub_cdm.py +++ b/code/infrastructure/streaming/clients/pub/pub_cdm.py @@ -9,8 +9,19 @@ port = int(os.getenv('mos_port',default="1883")) lines_per_window = int(os.getenv('lines_per_window',default="1000")) path = str(os.getenv('path_data',default='C:\\Studium_MIN\\05_Masterarbeit\\thesis\\ma_code\\code\\infrastructure\\streaming\\clients\\pub\\data\\')) -files = ['ta1-cadets-e3-official_1.json', 'ta1-cadets-e3-official_2.json', 'ta1-cadets-e3-official_3.json'] -line_count = [4999999,4999999,3911712] # line_count corresponding to each file +files = [ + 'ta1-cadets-e3-official_0.json', + 'ta1-cadets-e3-official_1.json', + 'ta1-cadets-e3-official_2.json', + 'ta1-cadets-e3-official-1_0.json', + 'ta1-cadets-e3-official-1_1.json', + 'ta1-cadets-e3-official-1_2.json', + 'ta1-cadets-e3-official-1_3.json', + 'ta1-cadets-e3-official-1_4.json', + 'ta1-cadets-e3-official-2_0.json', + 'ta1-cadets-e3-official-2_1.json' + ] +line_count = [4999999,4999999,3911712,4999999,4999999,4999999,4999999,4999999,2059063,4999999,34335661] # line_count corresponding to each file sleep_time = 2 client = mqtt.Client("Client1") topic = "neo4j" diff --git a/code/infrastructure/streaming/clients/spielwiese.ipynb b/code/infrastructure/streaming/clients/spielwiese.ipynb old mode 100644 new mode 100755 diff --git a/code/infrastructure/streaming/clients/sub/neo4j/Dockerfile b/code/infrastructure/streaming/clients/sub/neo4j/Dockerfile old mode 100644 new mode 100755 diff --git a/code/infrastructure/streaming/clients/sub/neo4j/sub_neo4j_cdm.py b/code/infrastructure/streaming/clients/sub/neo4j/sub_neo4j_cdm.py old mode 100644 new mode 100755 diff --git a/code/infrastructure/streaming/clients/sub/postgres/Dockerfile b/code/infrastructure/streaming/clients/sub/postgres/Dockerfile old mode 100644 new mode 100755 diff --git a/code/infrastructure/streaming/clients/sub/postgres/README.md b/code/infrastructure/streaming/clients/sub/postgres/README.md old mode 100644 new mode 100755 diff --git a/code/infrastructure/streaming/clients/sub/postgres/import_node_edge.txt b/code/infrastructure/streaming/clients/sub/postgres/import_node_edge.txt old mode 100644 new mode 100755 diff --git a/code/infrastructure/streaming/clients/sub/postgres/import_sql.txt b/code/infrastructure/streaming/clients/sub/postgres/import_sql.txt new file mode 100644 index 0000000000000000000000000000000000000000..ea618a58ad9938490d69c81ec7196d533142b180 --- /dev/null +++ b/code/infrastructure/streaming/clients/sub/postgres/import_sql.txt @@ -0,0 +1,252 @@ +DROP TABLE IF EXISTS event; + +CREATE TABLE event( + line NUMERIC, + line_no NUMERIC, + ts TIMESTAMP, + uuid VARCHAR(1024), + type VARCHAR(1024), + hostId VARCHAR(1024), + predicateObjectPath NUMERIC, + predicateObject2 NUMERIC, + predicateObject2Path NUMERIC, + timestampNanos NUMERIC, + location VARCHAR(1024), + size NUMERIC, + programPoint VARCHAR(1024), + sequence_long NUMERIC, + threadId_int NUMERIC, + subject_UUID VARCHAR(1024), + predicateObject_UUID VARCHAR(1024), + name_string VARCHAR(1024), + parameters_array TEXT, + properties_map_host VARCHAR(1024), + properties_map_return_value VARCHAR(1024), + properties_map_fd VARCHAR(1024), + properties_map_exec VARCHAR(1024), + properties_map_ppid VARCHAR(1024), + predicateObject2_UUID VARCHAR(1024), + properties_map_ret_fd2 VARCHAR(1024), + properties_map_ret_fd1 VARCHAR(1024), + predicateObject NUMERIC, + predicateObjectPath_string VARCHAR(1024), + size_long NUMERIC, + properties_map_partial_path VARCHAR(1024), + predicateObject2Path_string VARCHAR(1024), + properties_map_arg_pid VARCHAR(1024), + properties_map_cmdLine VARCHAR(1024), + properties_map_arg_mem_flags VARCHAR(1024), + properties_map_arg_euid VARCHAR(1024), + properties_map_arg_suid VARCHAR(1024), + properties_map_arg_ruid VARCHAR(1024), + properties_map_arg_rgid VARCHAR(1024), + properties_map_arg_egid VARCHAR(1024), + properties_map_arg_sgid VARCHAR(1024), + properties_map_address VARCHAR(1024), + properties_map_ret_msgid VARCHAR(1024), + properties_map_arg_uid VARCHAR(1024), + properties_map_arg_gid VARCHAR(1024), + properties_map_arg_miouuid VARCHAR(1024), + properties_map_port VARCHAR(1024), + subject NUMERIC, + name NUMERIC, + parameters NUMERIC, + properties_map_login VARCHAR(1024), + properties_map_ret_miouuid VARCHAR(1024) +) partition by range(ts); + +CREATE TABLE event_p2_0 PARTITION OF event FOR VALUES FROM ('2018-04-02 00:00:00') TO ('2018-04-02 06:00:00'); +CREATE TABLE event_p2_6 PARTITION OF event FOR VALUES FROM ('2018-04-02 06:00:00') TO ('2018-04-02 12:00:00'); +CREATE TABLE event_p2_12 PARTITION OF event FOR VALUES FROM ('2018-04-02 12:00:00') TO ('2018-04-02 18:00:00'); +CREATE TABLE event_p2_18 PARTITION OF event FOR VALUES FROM ('2018-04-02 18:00:00') TO ('2018-04-03 00:00:00'); + +CREATE TABLE event_p3_0 PARTITION OF event FOR VALUES FROM ('2018-04-03 00:00:00') TO ('2018-04-03 06:00:00'); +CREATE TABLE event_p3_6 PARTITION OF event FOR VALUES FROM ('2018-04-03 06:00:00') TO ('2018-04-03 12:00:00'); +CREATE TABLE event_p3_12 PARTITION OF event FOR VALUES FROM ('2018-04-03 12:00:00') TO ('2018-04-03 18:00:00'); +CREATE TABLE event_p3_18 PARTITION OF event FOR VALUES FROM ('2018-04-03 18:00:00') TO ('2018-04-04 00:00:00'); + +CREATE TABLE event_p4_0 PARTITION OF event FOR VALUES FROM ('2018-04-04 00:00:00') TO ('2018-04-04 06:00:00'); +CREATE TABLE event_p4_6 PARTITION OF event FOR VALUES FROM ('2018-04-04 06:00:00') TO ('2018-04-04 12:00:00'); +CREATE TABLE event_p4_12 PARTITION OF event FOR VALUES FROM ('2018-04-04 12:00:00') TO ('2018-04-04 18:00:00'); +CREATE TABLE event_p4_18 PARTITION OF event FOR VALUES FROM ('2018-04-04 18:00:00') TO ('2018-04-05 00:00:00'); + +CREATE TABLE event_p5_0 PARTITION OF event FOR VALUES FROM ('2018-04-05 00:00:00') TO ('2018-04-05 06:00:00'); +CREATE TABLE event_p5_6 PARTITION OF event FOR VALUES FROM ('2018-04-05 06:00:00') TO ('2018-04-05 12:00:00'); +CREATE TABLE event_p5_12 PARTITION OF event FOR VALUES FROM ('2018-04-05 12:00:00') TO ('2018-04-05 18:00:00'); +CREATE TABLE event_p5_18 PARTITION OF event FOR VALUES FROM ('2018-04-05 18:00:00') TO ('2018-04-06 00:00:00'); + +CREATE TABLE event_p6_0 PARTITION OF event FOR VALUES FROM ('2018-04-06 00:00:00') TO ('2018-04-06 06:00:00'); +CREATE TABLE event_p6_6 PARTITION OF event FOR VALUES FROM ('2018-04-06 06:00:00') TO ('2018-04-06 12:00:00'); +CREATE TABLE event_p6_12 PARTITION OF event FOR VALUES FROM ('2018-04-06 12:00:00') TO ('2018-04-06 18:00:00'); +CREATE TABLE event_p6_18 PARTITION OF event FOR VALUES FROM ('2018-04-06 18:00:00') TO ('2018-04-07 00:00:00'); + +CREATE TABLE event_p7_0 PARTITION OF event FOR VALUES FROM ('2018-04-07 00:00:00') TO ('2018-04-07 06:00:00'); +CREATE TABLE event_p7_6 PARTITION OF event FOR VALUES FROM ('2018-04-07 06:00:00') TO ('2018-04-07 12:00:00'); +CREATE TABLE event_p7_12 PARTITION OF event FOR VALUES FROM ('2018-04-07 12:00:00') TO ('2018-04-07 18:00:00'); +CREATE TABLE event_p7_18 PARTITION OF event FOR VALUES FROM ('2018-04-07 18:00:00') TO ('2018-04-08 00:00:00'); + +CREATE TABLE event_p8_0 PARTITION OF event FOR VALUES FROM ('2018-04-08 00:00:00') TO ('2018-04-08 06:00:00'); +CREATE TABLE event_p8_6 PARTITION OF event FOR VALUES FROM ('2018-04-08 06:00:00') TO ('2018-04-08 12:00:00'); +CREATE TABLE event_p8_12 PARTITION OF event FOR VALUES FROM ('2018-04-08 12:00:00') TO ('2018-04-08 18:00:00'); +CREATE TABLE event_p8_18 PARTITION OF event FOR VALUES FROM ('2018-04-08 18:00:00') TO ('2018-04-09 00:00:00'); + +CREATE TABLE event_p9_0 PARTITION OF event FOR VALUES FROM ('2018-04-09 00:00:00') TO ('2018-04-09 06:00:00'); +CREATE TABLE event_p9_6 PARTITION OF event FOR VALUES FROM ('2018-04-09 06:00:00') TO ('2018-04-09 12:00:00'); +CREATE TABLE event_p9_12 PARTITION OF event FOR VALUES FROM ('2018-04-09 12:00:00') TO ('2018-04-09 18:00:00'); +CREATE TABLE event_p9_18 PARTITION OF event FOR VALUES FROM ('2018-04-09 18:00:00') TO ('2018-04-10 00:00:00'); + +CREATE TABLE event_p10_0 PARTITION OF event FOR VALUES FROM ('2018-04-10 00:00:00') TO ('2018-04-10 06:00:00'); +CREATE TABLE event_p10_6 PARTITION OF event FOR VALUES FROM ('2018-04-10 06:00:00') TO ('2018-04-10 12:00:00'); +CREATE TABLE event_p10_12 PARTITION OF event FOR VALUES FROM ('2018-04-10 12:00:00') TO ('2018-04-10 18:00:00'); +CREATE TABLE event_p10_18 PARTITION OF event FOR VALUES FROM ('2018-04-10 18:00:00') TO ('2018-04-11 00:00:00'); + +CREATE TABLE event_p11_0 PARTITION OF event FOR VALUES FROM ('2018-04-11 00:00:00') TO ('2018-04-11 06:00:00'); +CREATE TABLE event_p11_6 PARTITION OF event FOR VALUES FROM ('2018-04-11 06:00:00') TO ('2018-04-11 12:00:00'); +CREATE TABLE event_p11_12 PARTITION OF event FOR VALUES FROM ('2018-04-11 12:00:00') TO ('2018-04-11 18:00:00'); +CREATE TABLE event_p11_18 PARTITION OF event FOR VALUES FROM ('2018-04-11 18:00:00') TO ('2018-04-12 00:00:00'); + +CREATE TABLE event_p12_0 PARTITION OF event FOR VALUES FROM ('2018-04-12 00:00:00') TO ('2018-04-12 06:00:00'); +CREATE TABLE event_p12_6 PARTITION OF event FOR VALUES FROM ('2018-04-12 06:00:00') TO ('2018-04-12 12:00:00'); +CREATE TABLE event_p12_12 PARTITION OF event FOR VALUES FROM ('2018-04-12 12:00:00') TO ('2018-04-12 18:00:00'); +CREATE TABLE event_p12_18 PARTITION OF event FOR VALUES FROM ('2018-04-12 18:00:00') TO ('2018-04-13 00:00:00'); + +CREATE TABLE event_p13_0 PARTITION OF event FOR VALUES FROM ('2018-04-13 00:00:00') TO ('2018-04-13 06:00:00'); +CREATE TABLE event_p13_6 PARTITION OF event FOR VALUES FROM ('2018-04-13 06:00:00') TO ('2018-04-13 12:00:00'); +CREATE TABLE event_p13_12 PARTITION OF event FOR VALUES FROM ('2018-04-13 12:00:00') TO ('2018-04-13 18:00:00'); +CREATE TABLE event_p13_18 PARTITION OF event FOR VALUES FROM ('2018-04-13 18:00:00') TO ('2018-04-14 00:00:00'); + +CREATE TABLE event_p14_0 PARTITION OF event FOR VALUES FROM ('2018-04-14 00:00:00') TO ('2018-04-14 06:00:00'); +CREATE TABLE event_p14_6 PARTITION OF event FOR VALUES FROM ('2018-04-14 06:00:00') TO ('2018-04-14 12:00:00'); +CREATE TABLE event_p14_12 PARTITION OF event FOR VALUES FROM ('2018-04-14 12:00:00') TO ('2018-04-14 18:00:00'); +CREATE TABLE event_p14_18 PARTITION OF event FOR VALUES FROM ('2018-04-14 18:00:00') TO ('2018-04-15 00:00:00'); + +DROP TABLE IF EXISTS fileobject; + +CREATE TABLE fileobject( + line_no NUMERIC, + line NUMERIC, + uuid VARCHAR(1024), + type VARCHAR(1024), + filedescriptor VARCHAR(1024), + localprincipal VARCHAR(1024), + size VARCHAR(1024), + peinfo VARCHAR(1024), + hashes VARCHAR(1024), + baseobject_hostid VARCHAR(1024), + baseobject_permission VARCHAR(1024), + baseobject_epoch VARCHAR(1024) +); + +DROP TABLE IF EXISTS principal; + +CREATE TABLE principal( + line_no NUMERIC, + line NUMERIC, + uuid VARCHAR(1024), + type VARCHAR(1024), + hostId VARCHAR(1024), + userId NUMERIC, + groupIds VARCHAR(1024), + username_string VARCHAR(1024) +); + +DROP TABLE IF EXISTS subject; + +CREATE TABLE subject( + line_no NUMERIC, + line NUMERIC, + ts TIMESTAMP, + uuid VARCHAR(1024), + type VARCHAR(1024), + cid NUMERIC, + hostId VARCHAR(1024), + localPrincipal VARCHAR(1024), + startTimestampNanos NUMERIC, + unitId VARCHAR(1024), + iteration VARCHAR(1024), + count VARCHAR(1024), + cmdLine VARCHAR(1024), + privilegeLevel VARCHAR(1024), + importedLibraries VARCHAR(1024), + exportedLibraries VARCHAR(1024), + parentSubject_UUID VARCHAR(1024), + properties_map_host VARCHAR(1024), + parentSubject VARCHAR(1024) +); + +DROP TABLE IF EXISTS host; + +CREATE TABLE host( + line_no NUMERIC, + line NUMERIC, + uuid VARCHAR(1024), + hostName VARCHAR(1024), + hostIdentifiers VARCHAR(1024), + osDetails VARCHAR(1024), + hostType VARCHAR(1024), + interfaces VARCHAR(1024) +); + +DROP TABLE IF EXISTS netflowobject; + +CREATE TABLE netflowobject( + line_no NUMERIC, + line NUMERIC, + uuid VARCHAR(1024), + localAddress VARCHAR(1024), + localPort NUMERIC, + remoteAddress VARCHAR(1024), + remotePort NUMERIC, + ipProtocol VARCHAR(1024), + fileDescriptor VARCHAR(1024), + baseObject_hostId VARCHAR(1024), + baseObject_permission VARCHAR(1024), + baseObject_epoch VARCHAR(1024) +); + +DROP TABLE IF EXISTS srcsinkobject; + +CREATE TABLE srcsinkobject( + line_no NUMERIC, + line NUMERIC, + uuid VARCHAR(1024), + type VARCHAR(1024), + fileDescriptor VARCHAR(1024), + baseObject_hostId VARCHAR(1024), + baseObject_permission VARCHAR(1024), + baseObject_epoch VARCHAR(1024) +); + +DROP TABLE IF EXISTS unnamedpipeobject; + +CREATE TABLE unnamedpipeobject( + line_no NUMERIC, + line NUMERIC, + uuid VARCHAR(1024), + sourceFileDescriptor VARCHAR(1024), + sinkFileDescriptor VARCHAR(1024), + baseObject_hostId VARCHAR(1024), + baseObject_permission VARCHAR(1024), + baseObject_epoch VARCHAR(1024), + sourceUUID_UUID VARCHAR(1024), + sinkUUID_UUID VARCHAR(1024) +); + +DROP TABLE IF EXISTS node_uuids; + +CREATE TABLE node_uuids( + uuid VARCHAR(1024), + line_no NUMERIC, + type VARCHAR(1024), + sub_type VARCHAR(1024) +); + +DROP TABLE IF EXISTS edge_list; +CREATE TABLE edge_list( + id NUMERIC, + source varchar(1024), + dest varchar(1024), + edge_type varchar(1024) +); + + diff --git a/code/infrastructure/streaming/clients/sub/postgres/schema_cmd18.sql b/code/infrastructure/streaming/clients/sub/postgres/schema_cmd18.sql old mode 100644 new mode 100755 diff --git a/code/infrastructure/streaming/clients/sub/postgres/sub_pg_cdm.py b/code/infrastructure/streaming/clients/sub/postgres/sub_pg_cdm.py old mode 100644 new mode 100755 diff --git a/code/infrastructure/streaming/mosquitto.conf b/code/infrastructure/streaming/mosquitto.conf old mode 100644 new mode 100755 diff --git a/code/infrastructure/streaming/obsolet/Sub_cdm.py b/code/infrastructure/streaming/obsolet/Sub_cdm.py old mode 100644 new mode 100755 diff --git a/code/infrastructure/streaming/obsolet/docker-compose.yml b/code/infrastructure/streaming/obsolet/docker-compose.yml old mode 100644 new mode 100755 diff --git a/code/infrastructure/streaming/obsolet/first_pub.py b/code/infrastructure/streaming/obsolet/first_pub.py old mode 100644 new mode 100755 diff --git a/code/infrastructure/streaming/obsolet/first_sub.py b/code/infrastructure/streaming/obsolet/first_sub.py old mode 100644 new mode 100755 diff --git a/code/infrastructure/streaming/obsolet/start_pub_sub.sh b/code/infrastructure/streaming/obsolet/start_pub_sub.sh old mode 100644 new mode 100755 diff --git a/code/infrastructure/streaming/obsolet/test.py b/code/infrastructure/streaming/obsolet/test.py old mode 100644 new mode 100755 diff --git a/code/infrastructure/streaming/readme.md b/code/infrastructure/streaming/readme.md old mode 100644 new mode 100755