removed unused shemas, split dbinit into multiple files for configurability,...

removed unused shemas, split dbinit into multiple files for configurability, moved data to top level of project, merged pub sub readmes

.gitignore

 .vscode/
 __pycache__
 .mypy_cache
-/streaming/pub/data/
-/postgres/data/
+/data/
 /output
\ No newline at end of file

docker-compose.yml

@@ -8,6 +8,8 @@ services:
       - BATCH_SIZE
     profiles:
       - experiment
+    configs:
+      - data.zip
     depends_on: 
       - mosquitto
       - sub_pg
@@ -71,10 +73,11 @@ services:
       start_period: 5s
       start_interval: 1s
     volumes:
-      - ./postgres/initdb:/docker-entrypoint-initdb.d
-      - ./streaming/pub/data:/docker-entrypoint-initdb.d/data:ro
+      - ./postgres/initdb/00-initdb.sql:/docker-entrypoint-initdb.d/00-initdb.sql:ro
+      - ./postgres/initdb/initdb.py:/docker-entrypoint-initdb.d/initdb.py:ro
     configs:
       - postgres_conf
+      - data.zip
     secrets:
       - postgres_db_pass
 
@@ -166,6 +169,8 @@ configs:
     file: ./grafana/9628_rev7.json
   pgadmin_server_conf:
     file: ./servers.json
+  data.zip:
+    file: ./data/ta1-cadets-e3-official.zip
 
 secrets:
   postgres_pass:

postgres/initdb/00-initdb.sql

+CREATE TYPE VERTEX_TYPE AS ENUM ('Event', 'FileObject', 'Principal', 'Subject', 'Host', 'NetFlowObject', 'SrcSinkObject', 'UnnamedPipeObject');
+CREATE TABLE vertex(
+    id INTEGER GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
+    type VERTEX_TYPE NOT NULL,
+    content JSONB NOT NULL
+);
+
+CREATE TYPE EDGE_TYPE AS ENUM ('is_generated_by', 'affects', 'affects_2', 'has_parent', 'has_owning_principal', 'ingests', 'outputs');

postgres/initdb/01-initdb_load_data.sql

+COPY vertex (type, content)
+FROM PROGRAM 'python3 /docker-entrypoint-initdb.d/initdb.py /data.zip'
+WITH (FORMAT csv, QUOTE '''');

postgres/initdb/02-initdb_edge_id.sql

+CREATE INDEX vertex_uuid_id ON vertex ((content->>'uuid')) INCLUDE (id);
+
+CREATE TABLE edge(
+    source INTEGER NOT NULL,
+    destination INTEGER NOT NULL,
+    type EDGE_TYPE NOT NULL
+);

postgres/initdb/02-initdb_edge_uuid.sql

+CREATE TABLE edge(
+    source UUID NOT NULL,
+    destination UUID NOT NULL,
+    type EDGE_TYPE NOT NULL
+);

postgres/initdb/03-initdb_fill_edge_id.sql

+with raw_edges as(
+    select id source, content#>>'{subject,UUID}' destination_uuid, 'is_generated_by' type
+    from vertex
+    where type='Event'
+    UNION ALL
+
+    select id, content#>>'{predicateObject,UUID}', 'affects'
+    from vertex
+    where type='Event'
+    UNION ALL
+
+    select id, content#>>'{predicateObject2,UUID}', 'affects_2'
+    from vertex
+    where type='Event'
+    UNION ALL
+
+    select id, content#>>'{parentSubject,UUID}', 'has_parent'
+    from vertex
+    where type='Subject'
+    UNION ALL
+
+    select id, content->>'localPrincipal', 'has_owning_principal'
+    from vertex
+    where type='Subject' or type='FileObject'
+    UNION ALL
+
+    select id, content#>>'{sourceUUID,UUID}', 'ingests'
+    from vertex
+    where type='UnnamedPipeObject'
+    UNION ALL
+
+    select id, content#>>'{sinkUUID,UUID}', 'outputs'
+    from vertex
+    where type='UnnamedPipeObject'
+), new_edges as(
+    select source, id destination, e.type::EDGE_TYPE
+    from raw_edges e
+    join vertex on (content->>'uuid')=destination_uuid
+    where destination_uuid is not null
+)
+insert into edge (source, destination, type)
+select * from new_edges;

postgres/initdb/03-initdb_fill_edge_uuid.sql

+with new_edges as(
+    select content->>'uuid' source, content#>>'{subject,UUID}' destination, 'is_generated_by' type
+    from vertex
+    where type='Event'
+    UNION ALL
+
+    select content->>'uuid', content#>>'{predicateObject,UUID}', 'affects'
+    from vertex
+    where type='Event'
+    UNION ALL
+
+    select content->>'uuid', content#>>'{predicateObject2,UUID}', 'affects_2'
+    from vertex
+    where type='Event'
+    UNION ALL
+
+    select content->>'uuid', content#>>'{parentSubject,UUID}', 'has_parent'
+    from vertex
+    where type='Subject'
+    UNION ALL
+
+    select content->>'uuid', content->>'localPrincipal', 'has_owning_principal'
+    from vertex
+    where type='Subject' or type='FileObject'
+    UNION ALL
+
+    select content->>'uuid', content#>>'{sourceUUID,UUID}', 'ingests'
+    from vertex
+    where type='UnnamedPipeObject'
+    UNION ALL
+
+    select content->>'uuid', content#>>'{sinkUUID,UUID}', 'outputs'
+    from vertex
+    where type='UnnamedPipeObject'
+)
+insert into edge (source, destination, type)
+select source, destination, type::EDGE_TYPE from new_edges;

postgres/initdb/initdb.sql → postgres/initdb/03-initdb_trigger_edge_id.sql

-CREATE TYPE VERTEX_TYPE AS ENUM ('Event', 'FileObject', 'Principal', 'Subject', 'Host', 'NetFlowObject', 'SrcSinkObject', 'UnnamedPipeObject');
-CREATE TABLE vertex(
-    id INTEGER GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
-    type VERTEX_TYPE NOT NULL,
-    content JSONB NOT NULL
-);
-
-CREATE TYPE EDGE_TYPE AS ENUM ('is_generated_by', 'affects', 'affects_2', 'has_parent', 'has_owning_principal', 'ingests', 'outputs');
-CREATE TABLE edge(
-    source INTEGER NOT NULL,
-    destination INTEGER NOT NULL,
-    type EDGE_TYPE NOT NULL
-);
-
 create function process_new_vertices() returns trigger as $proc_verts$
 begin
 	with raw_edges as(
@@ -64,12 +50,3 @@ create trigger process_vertex_insertions
 	after insert on vertex
 	referencing new table as new_vertices
 	for each statement execute function process_new_vertices();
-
--- COPY vertex (type, content)
--- FROM PROGRAM 'python3 /docker-entrypoint-initdb.d/initdb.py /docker-entrypoint-initdb.d/data/ta1-cadets-e3-official.zip'
--- WITH (FORMAT csv, QUOTE '''');
-
-CREATE INDEX vertex_uuid_id ON vertex ((content->>'uuid')) INCLUDE (id);
-
-CREATE INDEX edge_source ON edge (source) include (destination);
-CREATE INDEX edge_dest ON edge (destination) include (source);

postgres/initdb/03-initdb_trigger_edge_uuid.sql

+create function process_new_vertices() returns trigger as $proc_verts$
+begin
+	with new_edges as(
+		select content->>'uuid' source, content#>>'{subject,UUID}' destination, 'is_generated_by' type
+		from new_vertices
+		where type='Event'
+		UNION ALL
+
+		select content->>'uuid', content#>>'{predicateObject,UUID}', 'affects'
+		from new_vertices
+		where type='Event'
+		UNION ALL
+
+		select content->>'uuid', content#>>'{predicateObject2,UUID}', 'affects_2'
+		from new_vertices
+		where type='Event'
+		UNION ALL
+
+		select content->>'uuid', content#>>'{parentSubject,UUID}', 'has_parent'
+		from new_vertices
+		where type='Subject'
+		UNION ALL
+
+		select content->>'uuid', content->>'localPrincipal', 'has_owning_principal'
+		from new_vertices
+		where type='Subject' or type='FileObject'
+		UNION ALL
+
+		select content->>'uuid', content#>>'{sourceUUID,UUID}', 'ingests'
+		from new_vertices
+		where type='UnnamedPipeObject'
+		UNION ALL
+
+		select content->>'uuid', content#>>'{sinkUUID,UUID}', 'outputs'
+		from new_vertices
+		where type='UnnamedPipeObject'
+	)
+	insert into edge (source, destination, type)
+	select source, destination, type::EDGE_TYPE from new_edges;
+	return null;
+end;
+$proc_verts$ language plpgsql;
+
+create trigger process_vertex_insertions
+	after insert on vertex
+	referencing new table as new_vertices
+	for each statement execute function process_new_vertices();

postgres/initdb/04-initdb_edge_index.sql

+CREATE INDEX edge_source ON edge (source);
+CREATE INDEX edge_dest ON edge (destination);

postgres/initdb/04-initdb_edge_index_hash.sql

+CREATE INDEX edge_source ON edge USING HASH (source);
+CREATE INDEX edge_dest ON edge USING HASH (destination);

postgres/initdb/04-initdb_edge_index_multicolumn.sql

+CREATE INDEX edge_source ON edge (source) include (destination);
+CREATE INDEX edge_dest ON edge (destination) include (source);

streaming/readme.md → streaming/README.md

@@ -6,6 +6,10 @@ Ablauf:
 3. Subscriber parsed CDM Daten in Query für Einfüge Operationen.
 4. Subscriber führen Query aus bei Erhalt neuer Nachricht.
 
+## Path to Cadets
+Im folgenden Google Drive befinden sich die Datensätze der DARPA Performer in Engagement 3
+https://drive.google.com/drive/folders/1QlbUFWAGq3Hpl8wVdzOdIoZLFxkII4EK
+
 ## Bauen eines Dockercontainers zu einem Publisher
 Befehl: docker build -t <it>image_name</it> <it>directory dockerfile</it>
 

streaming/pub/Dockerfile

 FROM python:3
 WORKDIR /app
 RUN pip install paho-mqtt schedule
-COPY pub_cdm.py data/ta1-cadets-e3-official.zip ./
+COPY pub_cdm.py ./
 # ENV MOS_HOST= MOS_TOPIC= LINES_PER_SECOND= BATCH_SIZE=
 CMD ["python", "-u", "pub_cdm.py"]
\ No newline at end of file

streaming/pub/README.md

-# Path to Cadets
-Im folgenden Google Drive befinden sich die Datensätze der DARPA Performer in Engagement 3
-https://drive.google.com/drive/folders/1QlbUFWAGq3Hpl8wVdzOdIoZLFxkII4EK

streaming/pub/pub_cdm.py

@@ -52,7 +52,7 @@ if __name__ == "__main__":
     topic = environ["MOS_TOPIC"]
     num_lines = int(environ["LINES_PER_SECOND"])
     batch_size = max(min(num_lines, int(environ["BATCH_SIZE"])), 1)
-    file = "ta1-cadets-e3-official.zip"
+    file = "/data.zip"
     try:
         client = Client(CallbackAPIVersion.VERSION2, "Publisher")
         client.connect(**broker_conf)

streaming/sub/postgres/drop_schemas.sql

--- schema.sql
-DROP TABLE IF EXISTS event;
-DROP TABLE IF EXISTS fileobject;
-DROP TABLE IF EXISTS principal;
-DROP TABLE IF EXISTS subject;
-DROP TABLE IF EXISTS host;
-DROP TABLE IF EXISTS netflowobject;
-DROP TABLE IF EXISTS srcsinkobject;
-DROP TABLE IF EXISTS unnamedpipeobject;
-DROP TABLE IF EXISTS node CASCADE;
-DROP TABLE IF EXISTS edge CASCADE; 
-DROP INDEX IF EXISTS idx_node_uuid;
-DROP INDEX IF EXISTS idx_edge_source;
-DROP INDEX IF EXISTS idx_edge_dest;
-
--- procedures.sql
-DROP PROCEDURE IF EXISTS insert_host;
-DROP PROCEDURE IF EXISTS insert_event;
-DROP PROCEDURE IF EXISTS insert_subject; 
-DROP PROCEDURE IF EXISTS insert_fileobject;
-DROP PROCEDURE IF EXISTS insert_netflowobject;
-DROP PROCEDURE IF EXISTS insert_srcsinkobject;
-DROP PROCEDURE IF EXISTS insert_unnamedpipeobject;
-DROP PROCEDURE IF EXISTS insert_principal;
-
--- schema_json.sql
-DROP TABLE IF EXISTS edge CASCADE; 
-DROP TYPE IF EXISTS EDGE_TYPE;
-DROP TABLE IF EXISTS node CASCADE;
-DROP TYPE IF EXISTS NODE_TYPE;
-DROP INDEX IF EXISTS idx_node;
-DROP INDEX IF EXISTS idx_edge_source;
-DROP INDEX IF EXISTS idx_edge_dest;
-DROP INDEX IF EXISTS idx_edge;

streaming/sub/postgres/procedures.sql

-create procedure insert_host(
-	uuid host.uuid%type,
-    hostname host.hostname%type = null,
-    osdetails host.osdetails%type = null,
-    hosttype host.hosttype%type = null,
-    interfaces_0_name host.interfaces_0_name%type = null,
-    interfaces_0_macaddress host.interfaces_0_macaddress%type = null,
-    interfaces_0_ipaddresses_0 host.interfaces_0_ipaddresses_0%type = null,
-    interfaces_0_ipaddresses_1 host.interfaces_0_ipaddresses_1%type = null,
-    interfaces_1_name host.interfaces_1_name%type = null,
-    interfaces_1_macaddress host.interfaces_1_macaddress%type = null,
-    interfaces_1_ipaddresses_0 host.interfaces_1_ipaddresses_0%type = null,
-    interfaces_1_ipaddresses_1 host.interfaces_1_ipaddresses_1%type = null
-)
-language plpgsql
-as $$
-begin
-	insert into host (uuid, hostname, osdetails, hosttype, interfaces_0_name, interfaces_0_macaddress, interfaces_0_ipaddresses_0, interfaces_0_ipaddresses_1, interfaces_1_name, interfaces_1_macaddress, interfaces_1_ipaddresses_0, interfaces_1_ipaddresses_1)
-	values (uuid, hostname, osdetails, hosttype, interfaces_0_name, interfaces_0_macaddress, interfaces_0_ipaddresses_0, interfaces_0_ipaddresses_1, interfaces_1_name, interfaces_1_macaddress, interfaces_1_ipaddresses_0, interfaces_1_ipaddresses_1);
-	insert into node_list (uuid, type) values (uuid, 'Host');
-end;$$;
-
-create procedure insert_event(
-    uuid event.uuid%type,
-    ts event.ts%type = null,
-    type event.type%type = null,
-    hostid event.hostid%type = null,
-    predicateobjectpath event.predicateobjectpath%type = null,
-    predicateobject2 event.predicateobject2%type = null,
-    predicateobject2path event.predicateobject2path%type = null,
-    timestampnanos event.timestampnanos%type = null,
-    location event.location%type = null,
-    size event.size%type = null,
-    programpoint event.programpoint%type = null,
-    sequence_long event.sequence_long%type = null,
-    threadid_int event.threadid_int%type = null,
-    subject_uuid event.subject_uuid%type = null,
-    predicateobject_uuid event.predicateobject_uuid%type = null,
-    name_string event.name_string%type = null,
-    parameters_array_0_size event.parameters_array_0_size%type = null,
-    parameters_array_0_type event.parameters_array_0_type%type = null,
-    parameters_array_0_valuedatatype event.parameters_array_0_valuedatatype%type = null,
-    parameters_array_0_isnull event.parameters_array_0_isnull%type = null,
-    parameters_array_0_name_string event.parameters_array_0_name_string%type = null,
-    parameters_array_0_valuebytes_bytes event.parameters_array_0_valuebytes_bytes%type = null,
-    parameters_array_1_size event.parameters_array_1_size%type = null,
-    parameters_array_1_type event.parameters_array_1_type%type = null,
-    parameters_array_1_valuedatatype event.parameters_array_1_valuedatatype%type = null,
-    parameters_array_1_isnull event.parameters_array_1_isnull%type = null,
-    parameters_array_1_name_string event.parameters_array_1_name_string%type = null,
-    parameters_array_1_valuebytes_bytes event.parameters_array_1_valuebytes_bytes%type = null,
-    parameters_array_2_size event.parameters_array_2_size%type = null,
-    parameters_array_2_type event.parameters_array_2_type%type = null,
-    parameters_array_2_valuedatatype event.parameters_array_2_valuedatatype%type = null,
-    parameters_array_2_isnull event.parameters_array_2_isnull%type = null,
-    parameters_array_2_name_string event.parameters_array_2_name_string%type = null,
-    parameters_array_2_valuebytes_bytes event.parameters_array_2_valuebytes_bytes%type = null,
-    properties_map_host event.properties_map_host%type = null,
-    properties_map_return_value event.properties_map_return_value%type = null,
-    properties_map_fd event.properties_map_fd%type = null,
-    properties_map_exec event.properties_map_exec%type = null,
-    properties_map_ppid event.properties_map_ppid%type = null,
-    predicateobject2_uuid event.predicateobject2_uuid%type = null,
-    properties_map_ret_fd2 event.properties_map_ret_fd2%type = null,
-    properties_map_ret_fd1 event.properties_map_ret_fd1%type = null,
-    predicateobject event.predicateobject%type = null,
-    predicateobjectpath_string event.predicateobjectpath_string%type = null,
-    size_long event.size_long%type = null,
-    properties_map_partial_path event.properties_map_partial_path%type = null,
-    predicateobject2path_string event.predicateobject2path_string%type = null,
-    properties_map_arg_pid event.properties_map_arg_pid%type = null,
-    properties_map_cmdline event.properties_map_cmdline%type = null,
-    properties_map_arg_mem_flags event.properties_map_arg_mem_flags%type = null,
-    properties_map_arg_euid event.properties_map_arg_euid%type = null,
-    properties_map_arg_suid event.properties_map_arg_suid%type = null,
-    properties_map_arg_ruid event.properties_map_arg_ruid%type = null,
-    properties_map_arg_rgid event.properties_map_arg_rgid%type = null,
-    properties_map_arg_egid event.properties_map_arg_egid%type = null,
-    properties_map_arg_sgid event.properties_map_arg_sgid%type = null,
-    properties_map_address event.properties_map_address%type = null,
-    properties_map_ret_msgid event.properties_map_ret_msgid%type = null,
-    properties_map_arg_uid event.properties_map_arg_uid%type = null,
-    properties_map_arg_gid event.properties_map_arg_gid%type = null,
-    properties_map_arg_miouuid event.properties_map_arg_miouuid%type = null,
-    properties_map_port event.properties_map_port%type = null,
-    subject event.subject%type = null,
-    name event.name%type = null,
-    parameters event.parameters%type = null,
-    properties_map_login event.properties_map_login%type = null,
-    properties_map_ret_miouuid event.properties_map_ret_miouuid%type = null
-)
-language plpgsql
-as $$
-begin
-	insert into event (ts, uuid, type, hostid, predicateobjectpath, predicateobject2, predicateobject2path, timestampnanos, location, size, programpoint, sequence_long, threadid_int, subject_uuid, predicateobject_uuid, name_string, parameters_array_0_size, parameters_array_0_type, parameters_array_0_valuedatatype, parameters_array_0_isnull, parameters_array_0_name_string, parameters_array_0_valuebytes_bytes, parameters_array_1_size, parameters_array_1_type, parameters_array_1_valuedatatype, parameters_array_1_isnull, parameters_array_1_name_string, parameters_array_1_valuebytes_bytes, parameters_array_2_size, parameters_array_2_type, parameters_array_2_valuedatatype, parameters_array_2_isnull, parameters_array_2_name_string, parameters_array_2_valuebytes_bytes, properties_map_host, properties_map_return_value, properties_map_fd, properties_map_exec, properties_map_ppid, predicateobject2_uuid, properties_map_ret_fd2, properties_map_ret_fd1, predicateobject, predicateobjectpath_string, size_long, properties_map_partial_path, predicateobject2path_string, properties_map_arg_pid, properties_map_cmdline, properties_map_arg_mem_flags, properties_map_arg_euid, properties_map_arg_suid, properties_map_arg_ruid, properties_map_arg_rgid, properties_map_arg_egid, properties_map_arg_sgid, properties_map_address, properties_map_ret_msgid, properties_map_arg_uid, properties_map_arg_gid, properties_map_arg_miouuid, properties_map_port, subject, name, parameters, properties_map_login, properties_map_ret_miouuid)
-	values (ts, uuid, type, hostid, predicateobjectpath, predicateobject2, predicateobject2path, timestampnanos, location, size, programpoint, sequence_long, threadid_int, subject_uuid, predicateobject_uuid, name_string, parameters_array_0_size, parameters_array_0_type, parameters_array_0_valuedatatype, parameters_array_0_isnull, parameters_array_0_name_string, parameters_array_0_valuebytes_bytes, parameters_array_1_size, parameters_array_1_type, parameters_array_1_valuedatatype, parameters_array_1_isnull, parameters_array_1_name_string, parameters_array_1_valuebytes_bytes, parameters_array_2_size, parameters_array_2_type, parameters_array_2_valuedatatype, parameters_array_2_isnull, parameters_array_2_name_string, parameters_array_2_valuebytes_bytes, properties_map_host, properties_map_return_value, properties_map_fd, properties_map_exec, properties_map_ppid, predicateobject2_uuid, properties_map_ret_fd2, properties_map_ret_fd1, predicateobject, predicateobjectpath_string, size_long, properties_map_partial_path, predicateobject2path_string, properties_map_arg_pid, properties_map_cmdline, properties_map_arg_mem_flags, properties_map_arg_euid, properties_map_arg_suid, properties_map_arg_ruid, properties_map_arg_rgid, properties_map_arg_egid, properties_map_arg_sgid, properties_map_address, properties_map_ret_msgid, properties_map_arg_uid, properties_map_arg_gid, properties_map_arg_miouuid, properties_map_port, subject, name, parameters, properties_map_login, properties_map_ret_miouuid);
-	insert into node_list (uuid, type) values (uuid, 'Event');
-	if subject_uuid is not null then insert into edge_list (source, dest, edge_type) values (uuid, subject_uuid, 'is_generated_by'); end if;
-	if predicateobject_uuid is not null then insert into edge_list (source, dest, edge_type) values (uuid, predicateobject_uuid, 'affects'); end if;
-	if predicateobject2_uuid is not null then insert into edge_list (source, dest, edge_type) values (uuid, predicateobject2_uuid, 'affects_2'); end if;
-end;$$;
-
-create procedure insert_subject(
-    uuid subject.uuid%type,
-    ts subject.ts%type = null,
-    type subject.type%type = null,
-    cid subject.cid%type = null,
-    hostid subject.hostid%type = null,
-    localprincipal subject.localprincipal%type = null,
-    starttimestampnanos subject.starttimestampnanos%type = null,
-    unitid subject.unitid%type = null,
-    iteration subject.iteration%type = null,
-    count subject.count%type = null,
-    cmdline subject.cmdline%type = null,
-    privilegelevel subject.privilegelevel%type = null,
-    importedlibraries subject.importedlibraries%type = null,
-    exportedlibraries subject.exportedlibraries%type = null,
-    parentsubject_uuid subject.parentsubject_uuid%type = null,
-    properties_map_host subject.properties_map_host%type = null,
-    parentsubject subject.parentsubject%type = null
-)
-language plpgsql
-as $$
-begin
-	insert into subject (ts, uuid, type, cid, hostid, localprincipal, starttimestampnanos, unitid, iteration, count, cmdline, privilegelevel, importedlibraries, exportedlibraries, parentsubject_uuid, properties_map_host, parentsubject)
-	values (ts, uuid, type, cid, hostid, localprincipal, starttimestampnanos, unitid, iteration, count, cmdline, privilegelevel, importedlibraries, exportedlibraries, parentsubject_uuid, properties_map_host, parentsubject);
-	insert into node_list (uuid, type) values (uuid, 'Subject');
-	if parentsubject_uuid is not null then insert into edge_list (source, dest, edge_type) values (uuid, parentsubject_uuid, 'has_parent'); end if; 
-	if localprincipal is not null then insert into edge_list (source, dest, edge_type) values (uuid, localprincipal, 'has_local_principal'); end if; 
-	if hostid is not null then insert into edge_list (source, dest, edge_type) values (uuid, hostid, 'runs_on'); end if;
-end;$$;
-
-create procedure insert_fileobject(
-    uuid fileobject.uuid%type,
-    type fileobject.type%type = null,
-    filedescriptor fileobject.filedescriptor%type = null,
-    localprincipal fileobject.localprincipal%type = null,
-    size fileobject.size%type = null,
-    peinfo fileobject.peinfo%type = null,
-    hashes fileobject.hashes%type = null,
-    baseobject_hostid fileobject.baseobject_hostid%type = null,
-    baseobject_permission fileobject.baseobject_permission%type = null,
-    baseobject_epoch fileobject.baseobject_epoch%type = null
-)
-language plpgsql
-as $$
-begin
-	insert into fileobject (uuid, type, filedescriptor, localprincipal, size, peinfo, hashes, baseobject_hostid, baseobject_permission, baseobject_epoch)
-	values (uuid, type, filedescriptor, localprincipal, size, peinfo, hashes, baseobject_hostid, baseobject_permission, baseobject_epoch);
-	insert into node_list (uuid, type) values (uuid, 'FileObject');
-	if baseobject_hostid is not null then insert into edge_list (source, dest, edge_type) values (uuid, baseobject_hostid, 'resides_on'); end if; 
-	if localprincipal is not null then insert into edge_list (source, dest, edge_type) values (uuid, localprincipal, 'has_owning_principal'); end if;
-end;$$;
-
-create procedure insert_netflowobject(
-    uuid netflowobject.uuid%type,
-    localaddress netflowobject.localaddress%type = null,
-    localport netflowobject.localport%type = null,
-    remoteaddress netflowobject.remoteaddress%type = null,
-    remoteport netflowobject.remoteport%type = null,
-    ipprotocol netflowobject.ipprotocol%type = null,
-    filedescriptor netflowobject.filedescriptor%type = null,
-    baseobject_hostid netflowobject.baseobject_hostid%type = null,
-    baseobject_permission netflowobject.baseobject_permission%type = null,
-    baseobject_epoch netflowobject.baseobject_epoch%type = null
-)
-language plpgsql
-as $$
-begin
-	insert into netflowobject (uuid, localaddress, localport, remoteaddress, remoteport, ipprotocol, filedescriptor, baseobject_hostid, baseobject_permission, baseobject_epoch)
-	values (uuid, localaddress, localport, remoteaddress, remoteport, ipprotocol, filedescriptor, baseobject_hostid, baseobject_permission, baseobject_epoch);
-	insert into node_list (uuid, type) values (uuid, 'NetFlowObject');
-	if baseobject_hostid is not null then insert into edge_list (source, dest, edge_type) values (uuid, baseobject_hostid, 'resides_on'); end if;
-end;$$;
-
-create procedure insert_srcsinkobject(
-    uuid srcsinkobject.uuid%type,
-    type srcsinkobject.type%type = null,
-    filedescriptor srcsinkobject.filedescriptor%type = null,
-    baseobject_hostid srcsinkobject.baseobject_hostid%type = null,
-    baseobject_permission srcsinkobject.baseobject_permission%type = null,
-    baseobject_epoch srcsinkobject.baseobject_epoch%type = null
-)
-language plpgsql
-as $$
-begin
-	insert into srcsinkobject (uuid, type, filedescriptor, baseobject_hostid, baseobject_permission, baseobject_epoch)
-	values (uuid, type, filedescriptor, baseobject_hostid, baseobject_permission, baseobject_epoch);
-	insert into node_list (uuid, type) values (uuid, 'SrcSinkObject');
-	if baseobject_hostid is not null then insert into edge_list (source, dest, edge_type) values (uuid, baseobject_hostid, 'resides_on'); end if;
-end;$$;
-
-create procedure insert_unnamedpipeobject(
-    uuid unnamedpipeobject.uuid%type,
-    sourcefiledescriptor unnamedpipeobject.sourcefiledescriptor%type = null,
-    sinkfiledescriptor unnamedpipeobject.sinkfiledescriptor%type = null,
-    baseobject_hostid unnamedpipeobject.baseobject_hostid%type = null,
-    baseobject_permission unnamedpipeobject.baseobject_permission%type = null,
-    baseobject_epoch unnamedpipeobject.baseobject_epoch%type = null,
-    sourceuuid_uuid unnamedpipeobject.sourceuuid_uuid%type = null,
-    sinkuuid_uuid unnamedpipeobject.sinkuuid_uuid%type = null
-)
-language plpgsql
-as $$
-begin
-	insert into unnamedpipeobject (uuid, sourcefiledescriptor, sinkfiledescriptor, baseobject_hostid, baseobject_permission, baseobject_epoch, sourceuuid_uuid, sinkuuid_uuid)
-	values (uuid, sourcefiledescriptor, sinkfiledescriptor, baseobject_hostid, baseobject_permission, baseobject_epoch, sourceuuid_uuid, sinkuuid_uuid);
-	insert into node_list (uuid, type) values (uuid, 'UnnamedPipeObject');
-	if baseobject_hostid is not null then insert into edge_list (source, dest, edge_type) values (uuid, baseobject_hostid, 'resides_on'); end if; 
-	if sourceuuid_uuid is not null then insert into edge_list (source, dest, edge_type) values (uuid, sourceuuid_uuid, 'affects'); end if; 
-	if sinkuuid_uuid is not null then insert into edge_list (source, dest, edge_type) values (uuid, sinkuuid_uuid, 'affects_2'); end if;
-end;$$;
-
-create procedure insert_principal(
-    uuid principal.uuid%type,
-    type principal.type%type = null,
-    hostid principal.hostid%type = null,
-    userid principal.userid%type = null,
-    groupids principal.groupids%type = null,
-    username_string principal.username_string%type = null
-)
-language plpgsql
-as $$
-begin
-	insert into principal (uuid, type, hostid, userid, groupids, username_string)
-	values (uuid, type, hostid, userid, groupids, username_string);
-	insert into node_list (uuid, type) values (uuid, 'Principal');
-	if hostid is not null then insert into edge_list (source, dest, edge_type) values (uuid, hostid, 'has_account_on'); end if;
-end;$$;