Skip to content
Snippets Groups Projects
Commit bed8d6e3 authored by Sven-Ove Hänsel's avatar Sven-Ove Hänsel
Browse files

update docker for queries from txt

parent 8b8e8f84
Branches
Tags
No related merge requests found
...@@ -11,7 +11,7 @@ declare -A paths=( ...@@ -11,7 +11,7 @@ declare -A paths=(
["sub_pg"]="./infrastructure/streaming/clients/sub/postgres" ["sub_pg"]="./infrastructure/streaming/clients/sub/postgres"
["sub_neo4j"]="./infrastructure/streaming/clients/sub/neo4j" ["sub_neo4j"]="./infrastructure/streaming/clients/sub/neo4j"
["pub_cdm"]="./infrastructure/streaming/clients/pub" ["pub_cdm"]="./infrastructure/streaming/clients/pub"
["cypher_queries"]="./eval/queries/cypher" ["cypher_queries"]="./eval/queries/cypher/driver_latest"
["sql_queries"]="./eval/queries/sql" ["sql_queries"]="./eval/queries/sql"
) )
......
# Use an official Python runtime as a base image
FROM python:3.8
# Set working directory in the container
WORKDIR /app
# install dependencies
RUN pip install neo4j
RUN pip install schedule
ENV DOCKERIZE_VERSION v0.6.1
# Copy the Python script into the container
COPY cdm_cypher_queries.py /app/
# Set environment variable 'chunk_size' and 'number_of_insertions'
ENV db_port='7687'
ENV db_host='localhost'
ENV interval=15
ENV query_file_path='/app/queries/queries.txt'
CMD [ "python","cdm_cypher_queries.py" ]
\ No newline at end of file
from neo4j import GraphDatabase
import schedule
import time
import logging
import os
PORT = int(os.getenv('db_port',default=7687))
HOST = str(os.getenv('db_host',default='localhost'))
INTERVAL = int(os.getenv('interval',default=15))
log_directory = "log"
queries_file_path = str(os.getenv('query_file_path',default="C:\\Studium_MIN\\05_Masterarbeit\\thesis\\ma_code\\code\\infrastructure\\eval\\query\\neo4j\\queries.txt"))
# Connection details
uri = f"bolt://{HOST}:{PORT}"
auth = ("","") # Use this for no authentication, or (USERNAME, PASSWORD) tuple for authentication
cypher_queries = {
"anc": "MATCH path=(node:Host)-[*]->(ancestor) RETURN ancestor, length(path) AS distance_upstream ORDER BY distance_upstream;",
"desc": "MATCH path=(node:Host)<-[*]-(descendant) RETURN descendant, length(path) AS distance_downstream ORDER BY distance_downstream;",
"path": "MATCH path = shortestPath((a:Subject)-[*]-(b:Event)) where a._uuid='0CF2BB3E-36B8-11E8-BF66-D9AA8AFF4A69' RETURN path;",
"2-hop": "MATCH (startNode: Host)-[*1..2]-(neighborhood) RETURN neighborhood;"
}
def load_queries_from_file(file_path):
cypher_queries = {}
with open(file_path, 'r') as file:
for line in file:
if '|' in line: # Simple validation to ignore malformed lines
key, query = line.strip().split('|', 1)
cypher_queries[key] = query
return cypher_queries
# Function to execute a specific Cypher query and log response time
def execute_and_log_query(query_key, query):
start_time = time.time()
with driver.session() as session:
result = session.run(query)
summary = result.consume()
end_time = time.time()
# Depending on the driver version, you might directly get the execution time from the summary
execution_time = summary.result_available_after + summary.result_consumed_after
logging.info(f"Results for Query {query_key}:")
logging.info(f"Python executed in: {end_time - start_time}s")
logging.info(f"DBMS answered in: {execution_time} ms\n")
# Function to schedule and execute all queries
def schedule_and_execute_queries():
for query_key, query in cypher_queries.items():
execute_and_log_query(query_key=query_key, query=query)
# Example scheduling: execute immediately then every minute for demonstration
# TODO: Create the schedule here
schedule.every(INTERVAL).minutes.do(execute_and_log_query, query_key=query_key, query=query)
# schedule.every(15).minutes.do(execute_and_log_query, query_key=query_key, query=query)
# schedule.every(30).minutes.do()
while True:
schedule.run_pending()
time.sleep(1)
# create log directory
if not os.path.exists(log_directory):
os.makedirs(log_directory)
# Setup logging to file
formatted_time = time.strftime("%Y-%m-%d_%H-%M-%S")
logging.basicConfig(level=logging.INFO,
filename=f"{log_directory}/{formatted_time}_query_log.log",
filemode='a', # Append to the log file if it exists, 'w' to overwrite
format='%(asctime)s - %(levelname)s - %(message)s')
cypher_queries = load_queries_from_file(queries_file_path)
# Initialize the Neo4j connection
driver = GraphDatabase.driver(uri=uri, auth=auth,encrypted=False)
logging.info(f"Connected to db... {uri}")
# Main execution block with graceful shutdown
if __name__ == "__main__":
try:
schedule_and_execute_queries()
except KeyboardInterrupt:
logging.info("Script interrupted by user")
finally:
driver.close()
logging.info("Database connection closed.")
...@@ -20,5 +20,6 @@ ENV PGUSER='localhost' ...@@ -20,5 +20,6 @@ ENV PGUSER='localhost'
ENV PGPASSWORD='localhost' ENV PGPASSWORD='localhost'
ENV PGPORT='localhost' ENV PGPORT='localhost'
ENV interval=15 ENV interval=15
ENV query_file_path=/app/queries
CMD [ "python","cdm_sql_queries.py" ] CMD [ "python","cdm_sql_queries.py" ]
\ No newline at end of file
...@@ -11,6 +11,7 @@ username = os.getenv('PGUSER', 'postgres') ...@@ -11,6 +11,7 @@ username = os.getenv('PGUSER', 'postgres')
password = os.getenv('PGPASSWORD', 'postgres') password = os.getenv('PGPASSWORD', 'postgres')
port_id = os.getenv('PGPORT', '5432') # Default to 5432 if not set port_id = os.getenv('PGPORT', '5432') # Default to 5432 if not set
interval = int(os.getenv('interval',default=15)) interval = int(os.getenv('interval',default=15))
queries_file_path = str(os.getenv('query_file_path',default="C:\\Studium_MIN\\05_Masterarbeit\\thesis\\ma_code\\code\\infrastructure\\eval\\query\\pg\\queries.txt"))
def connect_to_db(): def connect_to_db():
try: try:
...@@ -28,8 +29,20 @@ def connect_to_db(): ...@@ -28,8 +29,20 @@ def connect_to_db():
logging.error(f"Error connecting to PostgreSQL database: {e}") logging.error(f"Error connecting to PostgreSQL database: {e}")
return None return None
def load_queries_from_file(file_path):
queries = {}
with open(file_path, 'r') as file:
content = file.read()
query_blocks = content.split('---') # Assuming '---' is used as separator between queries
for block in query_blocks:
if '|' in block:
key, query = block.strip().split('|', 1)
queries[key.strip()] = query.strip()
return queries
def execute_queries(conn): def execute_queries(conn):
try: try:
queries = load_queries_from_file(queries_file_path)
# Create a cursor object # Create a cursor object
cur = conn.cursor() cur = conn.cursor()
......
...@@ -175,7 +175,7 @@ services: ...@@ -175,7 +175,7 @@ services:
test: ["CMD","ls"] test: ["CMD","ls"]
interval: 30s interval: 30s
timeout: 10s timeout: 10s
retries: 2 retries: 5
start_period: 10s start_period: 10s
depends_on: depends_on:
ongdb: ongdb:
...@@ -217,7 +217,7 @@ services: ...@@ -217,7 +217,7 @@ services:
test: ["CMD","ls"] test: ["CMD","ls"]
interval: 30s interval: 30s
timeout: 10s timeout: 10s
retries: 2 retries: 5
start_period: 10s start_period: 10s
depends_on: depends_on:
neo4j: neo4j:
...@@ -268,8 +268,10 @@ services: ...@@ -268,8 +268,10 @@ services:
- PGPASSWORD=postgres - PGPASSWORD=postgres
- PGPORT=5432 - PGPORT=5432
- interval=1 - interval=1
- query_file_path=/app/queries/queries.txt
volumes: volumes:
- ./eval/query/pg/log:/app/log - ./eval/query/pg/log:/app/log
- ./eval/query/pg/queries:/app/queries
depends_on: depends_on:
- pub - pub
...@@ -280,8 +282,10 @@ services: ...@@ -280,8 +282,10 @@ services:
- interval=1 - interval=1
- db_host=neo4j - db_host=neo4j
- db_port=7687 - db_port=7687
- query_file_path=/app/queries/queries.txt
volumes: volumes:
- ./eval/query/neo4j/log:/app/log - ./eval/query/neo4j/log:/app/log
- ./eval/query/neo4j/queries:/app/queries
depends_on: depends_on:
- pub - pub
...@@ -292,8 +296,10 @@ services: ...@@ -292,8 +296,10 @@ services:
- interval=1 - interval=1
- db_host=ongdb - db_host=ongdb
- db_port=7687 - db_port=7687
- query_file_path='/app/queries/queries.txt'
volumes: volumes:
- ./eval/query/ongdb/log:/app/log - ./eval/query/ongdb/log:/app/log
- ./eval/query/ongdb/queries:/app/queries
depends_on: depends_on:
- pub - pub
...@@ -304,8 +310,10 @@ services: ...@@ -304,8 +310,10 @@ services:
- interval=1 - interval=1
- db_host=memgraph - db_host=memgraph
- db_port=7687 - db_port=7687
- query_file_path=/app/queries/queries.txt
volumes: volumes:
- ./eval/query/memgraph/log:/app/log - ./eval/query/memgraph/log:/app/log
- ./eval/query/memgraph/queries:/app/queries
depends_on: depends_on:
- pub - pub
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment