made some changes to accomodate theia dataset

.env

@@ -8,9 +8,6 @@ BATCH_SIZE=10000
 EDGE_INSERTS=EDGE_INSERTS
 UUID_EDGES=
 
-# query
-QUERY_INTERVAL=1
-
 # pgadmin
 PGADMIN_CONFIG_SERVER_MODE=False
 PGADMIN_CONFIG_MASTER_PASSWORD_REQUIRED=False

compose.yml

@@ -11,8 +11,8 @@ services:
       - PGPASSFILE=/run/secrets/pgpass
     profiles:
       - experiment
-    configs:
-      - data.zip
+    volumes:
+      - ./data/cadets.zip:/data.zip:ro
     secrets:
       - pgpass
     depends_on:
@@ -27,7 +27,6 @@ services:
       - PGDATABASE
       - PGUSER
       - PGPASSFILE=/run/secrets/pgpass
-      - QUERY_INTERVAL
     profiles:
       - experiment
     secrets:
@@ -50,11 +49,11 @@ services:
       start_period: 5s
       start_interval: 1s
     volumes:
+      - ./data/theia_part1.zip:/data.zip:ro
       - ./postgres/initdb/initdb.py:/docker-entrypoint-initdb.d/initdb.py:ro
       - ./postgres/initdb/00-initdb.sql:/docker-entrypoint-initdb.d/00-initdb.sql:ro
     configs:
       - postgres_conf
-      - data.zip
     secrets:
       - postgres_db_pass
 
@@ -77,12 +76,8 @@ services:
 configs:
   postgres_conf:
     file: ./postgresql.conf
-  mosquitto_conf:
-    file: ./mosquitto.conf
   pgadmin_server_conf:
     file: ./servers.json
-  data.zip:
-    file: ./data/ta1-cadets-e3-official.zip
 
 secrets:
   pgpass:

fill-edge-id.yml

+services:
+  postgres:
+    volumes:
+      - ./postgres/initdb/03-initdb_fill_edge_id.sql:/docker-entrypoint-initdb.d/03-initdb.sql:ro

insert/Dockerfile

@@ -2,5 +2,5 @@ FROM python:3
 WORKDIR /app
 RUN pip install "psycopg[binary,pool]"
 COPY pg_insert.py ./
-# ENV PGHOST= PGDATABASE= PGUSER= PGPASSWORD= BATCH_SIZE= EDGE_INSERTS= UUID_EDGES=
+# ENV PGHOST= PGDATABASE= PGUSER= PGPASSFILE= BATCH_SIZE= EDGE_INSERTS= UUID_EDGES=
 CMD ["python", "-u", "pg_insert.py"]

insert/pg_insert.py

@@ -27,6 +27,7 @@ NODE_TYPES: Final[dict] = {
     ),
     "NetFlowObject": (),
     "SrcSinkObject": (),
+    "MemoryObject": (),
     "Principal": (),
     "Host": (),
 }

load-data.yml

+services:
+  postgres:
+    volumes:
+      - ./postgres/initdb/01-initdb_load_data.sql:/docker-entrypoint-initdb.d/01-initdb.sql:ro

postgres/initdb/00-initdb.sql

-CREATE TYPE VERTEX_TYPE AS ENUM ('Event', 'FileObject', 'Principal', 'Subject', 'Host', 'NetFlowObject', 'SrcSinkObject', 'UnnamedPipeObject');
+CREATE TYPE VERTEX_TYPE AS ENUM ('Event', 'FileObject', 'Principal', 'Subject', 'Host', 'NetFlowObject', 'SrcSinkObject', 'UnnamedPipeObject', 'MemoryObject');
 CREATE TABLE vertex(
     id INTEGER GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
     type VERTEX_TYPE NOT NULL,

query/Dockerfile

@@ -2,5 +2,5 @@ FROM python:3
 WORKDIR /app
 RUN pip install "psycopg[binary,pool]" schedule
 COPY cdm_sql_queries.py queries.sql ./
-# ENV PGHOST= PGDATABASE= PGUSER= PGPASSWORD= QUERY_INTERVAL=
+# ENV PGHOST= PGDATABASE= PGUSER= PGPASSFILE=
 CMD ["python", "-u", "cdm_sql_queries.py"]
\ No newline at end of file

query/cdm_sql_queries.py

 from datetime import datetime
 from signal import SIGINT, SIGTERM, signal
-from typing import Sequence, Tuple, Any, Deque
-from time import strftime, perf_counter, sleep
-from os import environ, path, makedirs
+from typing import Tuple, Any, Deque
+from time import perf_counter, sleep
+from os import path, makedirs
 from logging import info, basicConfig, INFO
 from psycopg import connect
 from datetime import timedelta