prepare idacs experiments

code/build_push_container.sh

-#!/bin/bash
+#!/usr/bin/env bash
 
 # Docker login
 echo 'Logging into docker...'
@@ -17,24 +17,25 @@ docker login docker.gitlab.gwdg.de
 #     ["sql_queries"]="./eval/queries/sql"
 # )
 
-declare -A paths=(
-    ["sub_mem"]="./infrastructure/streaming/clients/sub/memgraph"
-    ["pub_cdm"]="./infrastructure/streaming/clients/pub"
-    ["memgraph_queries"]="./eval/queries/cypher/memgraph"
+paths=(
+    "sub_mem:./infrastructure/streaming/clients/sub/memgraph"
+    "pub_cdm:./infrastructure/streaming/clients/pub"
+    "memgraph_queries:./eval/queries/cypher/memgraph"
 )
 
-for name in "${!paths[@]}"; do
-    path=${paths[$name]}
+for entry in "${paths[@]}"; do
+    name="${entry%%:*}"
+    path="${entry#*:}"
     if [[ -z "$name" || -z "$path" ]]; then
         echo "Error: 'name' or 'path' is empty. Skipping..."
         continue
     fi
     echo ''
-    echo 'Build docker image for '${name}'...' 
-    docker build -t docker.gitlab.gwdg.de/sven-ove.haensel/ma_haensel/${name} ${path}
+    echo "Build docker image for ${name}..." 
+    docker build -t "docker.gitlab.gwdg.de/sven-ove.haensel/ma_haensel/${name}" "${path}"
     echo ''
-    echo 'Push docker image for '${name}'...' 
-    docker push docker.gitlab.gwdg.de/sven-ove.haensel/ma_haensel/${name}
+    echo "Push docker image for ${name}..." 
+    docker push "docker.gitlab.gwdg.de/sven-ove.haensel/ma_haensel/${name}"
 done
 
 echo "Finished Building..."
\ No newline at end of file

code/infrastructure/docker-compose_memgraph_idacs.yml

+# version: '3'
+
+services:
+  # Dashboard für Metriken
+  grafana:
+    image: grafana/grafana-oss:latest
+    container_name: grafana
+    ports:
+      - "127.0.0.1:3000:3000"
+    volumes:
+      - ./monitoring/grafana-data:/var/lib/grafana
+
+  # TSDB zum Speichern der Metriken
+  prometheus:
+    image: prom/prometheus:v2.37.9
+    container_name: prometheus
+    ports:
+      - 127.0.0.1:9090:9090
+    command: 
+      - "--config.file=/etc/prometheus/prometheus.yml"
+      - '--web.enable-admin-api'  
+      - '--storage.tsdb.retention.time=1y'
+    volumes:
+      - ./monitoring/prometheus/config:/etc/prometheus
+      - ./monitoring/prometheus/data:/prometheus
+    
+    
+  # Container Advisor - Docker Compose Metriken zu scrapen
+  cadvisor:
+    image: gcr.io/cadvisor/cadvisor:v0.47.0     
+    container_name: cadvisor
+    ports:
+      - 8080:8080
+    volumes:
+      - /:/rootfs:ro
+      - /var/run:/var/run:ro
+      - /sys:/sys:ro
+      - /var/lib/docker/:/var/lib/docker:ro
+      - /dev/disk/:/dev/disk:ro
+    devices:
+      - /dev/kmsg
+    privileged: true
+
+  # DB Container memgraph (testweise)
+  memgraph:
+    image: memgraph/memgraph-platform
+    container_name: memgraph
+    ports:
+      - "8687:7687"
+      - "8444:7444"
+      - "3001:3000"
+      - "9091:9091"
+      # - "9115:9115" only for mem_exporter
+    environment: 
+      - MEMGRAPH="--query-modules-directory=/usr/lib/memgraph/query_modules --monitoring-address=0.0.0.0 --monitoring-port=7444 --log-file=/var/log/memgraph/memgraph.log --log-level=TRACE --config=/etc/memgraph/memgraph.conf --memory-limit=5120 --query-execution-timeout-sec=6000 --storage-transaction-timeout-sec=600 --cartesian-product-enabled=false"
+    volumes: # mybe adding --memory-limit=6G # above
+      - ./memgraph/mg_lib:/var/lib/memgraph
+      - ./memgraph/mg_log:/var/log/memgraph:rw
+      - ./memgraph/mg_etc:/etc/memgraph:rw
+    entrypoint: ["/usr/bin/supervisord"]
+    healthcheck:
+      test: ["CMD", "mgconsole"]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+      start_period: 30s  
+    deploy:
+      resources:
+        limits:
+          memory: 12G
+          cpus: '4.0'  # Limiting to 4 CPU cores
+        reservations:
+          memory: 12G
+          cpus: '2.0'  # Reserving 2 CPU cores
+    mem_limit: 12G  # Set a hard memory limit
+    memswap_limit: 0G  # Set a swap limit to prevent excessive swapping -> export RAM to Harddrive (Paging)
+   
+
+  #mem_exporter:
+  #  image: mem_exporter
+  #  container_name: mem_exporter
+  #  ports: 
+  #    - "9115:9115"
+
+  
+  mosquitto:
+    container_name: mos1
+    image: eclipse-mosquitto
+    ports:
+      - 1883:1883
+      - 9001:9001
+      - 8883:8883
+    volumes:
+      - ./streaming/broker/mosquitto.conf:/mosquitto/config/mosquitto.conf
+      - ./streaming/broker/log/:/mosquitto/log/    
+      - ./streaming/broker/data/:/mosquitto/data/    
+
+  sub_mem:   # don t start sub
+    container_name: sub_mem
+    image: lab.it.hs-hannover.de:4567/cwy-p8d-u1/ma_code/sub_mem
+    environment:
+      - abort_time_limit=999999
+      - mem_host=bolt://memgraph:7687 
+      - mos_host=mos1
+      - mos_port=1883
+    depends_on:
+      memgraph:
+        condition: service_healthy
+        restart: true
+
+  pub: 
+    container_name: pub_cdm
+    image: docker.gitlab.gwdg.de/sven-ove.haensel/ma_haensel/pub_cdm:latest
+    # image: lab.it.hs-hannover.de:4567/cwy-p8d-u1/ma_code/pub_cdm
+    environment:
+      - lines_per_window= 45 #${WINDOW_SIZE}
+      # $Winndow from python start
+      - path_to_firstK=/var/lib/import/first1k.json
+      - path_data=/var/lib/import/theia/
+      - sleeptime=1
+      - original=false
+      - mos_host=mos1
+      - mos_port=1883
+    volumes: 
+      - ./streaming/clients/pub/data/theia/:/var/lib/import/theia/
+    depends_on:      #don't start sub
+      sub_mem:
+        condition: service_started
+      mosquitto:
+        condition: service_started    
+
+  
+  query_memgraph:
+    container_name: query_memgraph
+    image: lab.it.hs-hannover.de:4567/cwy-p8d-u1/ma_code/memgraph_queries
+    environment:
+      - interval=2
+      - db_host=memgraph 
+      - db_port=7687
+      - query_file_path=/app/queries/queries.txt
+    volumes:
+      - ./eval/query/memgraph/log:/app/log
+      - ./eval/query/memgraph/queries:/app/queries
+    depends_on:
+      - pub  
+      - memgraph #added
+
+volumes:
+  prometheus-data:
+    driver: local
+  mg_lib:
+  mg_log:
+  mg_etc:
+  memgraph_data:
+    driver: local
+    driver_opts:
+      size: 10GB  # Limit the volume size to 5GB 
+
+# networks:
+#   postgres:
+#     external: true

code/infrastructure/streaming/broker/mosquitto.conf

-
 log_dest file /mosquitto/log/mosquitto.log
 allow_anonymous true
 listener 1883 0.0.0.0

code/infrastructure/streaming/clients/pub/Dockerfile

@@ -7,7 +7,7 @@ WORKDIR /app
 # install dependencies
 RUN pip install paho-mqtt 
 
-ENV DOCKERIZE_VERSION v0.6.1
+ENV DOCKERIZE_VERSION=v0.6.1
 
 # Copy the Python script into the container
 COPY pub_cdm.py /app/