Skip to content
Snippets Groups Projects
Commit ebda5f5a authored by julian's avatar julian
Browse files

added some helpful info to experiment output, moved insertion config to compose files

parent 65750c8d
No related branches found
No related tags found
No related merge requests found
...@@ -2,3 +2,8 @@ services: ...@@ -2,3 +2,8 @@ services:
postgres: postgres:
volumes: volumes:
- ./postgres/initdb/02-initdb_edge_id_no_uuid_index.sql:/docker-entrypoint-initdb.d/02-initdb.sql:ro - ./postgres/initdb/02-initdb_edge_id_no_uuid_index.sql:/docker-entrypoint-initdb.d/02-initdb.sql:ro
sub_pg:
environment:
- EDGE_INSERTS=EDGE_INSERTS
- UUID_EDGES=
...@@ -3,3 +3,8 @@ services: ...@@ -3,3 +3,8 @@ services:
volumes: volumes:
- ./postgres/initdb/02-initdb_edge_id.sql:/docker-entrypoint-initdb.d/02-initdb.sql:ro - ./postgres/initdb/02-initdb_edge_id.sql:/docker-entrypoint-initdb.d/02-initdb.sql:ro
- ./postgres/initdb/03-initdb_trigger_edge_id.sql:/docker-entrypoint-initdb.d/03-initdb.sql:ro - ./postgres/initdb/03-initdb_trigger_edge_id.sql:/docker-entrypoint-initdb.d/03-initdb.sql:ro
sub_pg:
environment:
- EDGE_INSERTS=
- UUID_EDGES=
...@@ -2,3 +2,8 @@ services: ...@@ -2,3 +2,8 @@ services:
postgres: postgres:
volumes: volumes:
- ./postgres/initdb/02-initdb_edge_id.sql:/docker-entrypoint-initdb.d/02-initdb.sql:ro - ./postgres/initdb/02-initdb_edge_id.sql:/docker-entrypoint-initdb.d/02-initdb.sql:ro
sub_pg:
environment:
- EDGE_INSERTS=EDGE_INSERTS
- UUID_EDGES=
...@@ -2,3 +2,8 @@ services: ...@@ -2,3 +2,8 @@ services:
postgres: postgres:
volumes: volumes:
- ./postgres/initdb/02-initdb_edge_uuid.sql:/docker-entrypoint-initdb.d/02-initdb.sql:ro - ./postgres/initdb/02-initdb_edge_uuid.sql:/docker-entrypoint-initdb.d/02-initdb.sql:ro
sub_pg:
environment:
- EDGE_INSERTS=EDGE_INSERTS
- UUID_EDGES=UUID_EDGES
from logging import INFO, basicConfig, info
from os import environ, makedirs, path from os import environ, makedirs, path
from subprocess import run from subprocess import run
from typing import Sequence from typing import Sequence
...@@ -11,18 +12,19 @@ def run_experiment(name: str, compose_files: Sequence[str], env: dict = {}): ...@@ -11,18 +12,19 @@ def run_experiment(name: str, compose_files: Sequence[str], env: dict = {}):
run(("docker", "compose", "down")) run(("docker", "compose", "down"))
f = ["-f"] * len(compose_files) f = ["-f"] * len(compose_files)
interleaved = (val for pair in zip(f, compose_files) for val in pair) interleaved = (val for pair in zip(f, compose_files) for val in pair)
print("Start compose environment...") info(f"Starting compose environment for {name}")
run(("docker", "compose", *interleaved, "up", "-d", "--build")) run(("docker", "compose", *interleaved, "up", "-d", "--build"))
print("waiting for data run to finish") info(f"waiting for {name} to finish")
run(("docker", "compose", "wait", "query_pg")) run(("docker", "compose", "wait", "query_pg"))
print("data run finished, saving result") info(f"{name} finished, saving result")
run(("docker", "compose", "cp", "query_pg:/app/log/data.csv", f"result/{name}.csv")) run(("docker", "compose", "cp", "query_pg:/app/log/data.csv", f"result/{name}.csv"))
run(("docker", "compose", "down")) run(("docker", "compose", "down"))
if __name__ == "__main__": if __name__ == "__main__":
basicConfig(level=INFO, format="%(asctime)s - %(levelname)s - %(message)s")
result_dir = "result" result_dir = "result"
if not path.exists(result_dir): if not path.exists(result_dir):
makedirs(result_dir) makedirs(result_dir)
...@@ -30,202 +32,102 @@ if __name__ == "__main__": ...@@ -30,202 +32,102 @@ if __name__ == "__main__":
{ {
"name": "experiment_batch1_10000", "name": "experiment_batch1_10000",
"compose_files": ("compose.yml", "edge-id.yml", "index.yml"), "compose_files": ("compose.yml", "edge-id.yml", "index.yml"),
"env": { "env": {"LINES_PER_SECOND": 10000, "BATCH_SIZE": 1},
"EDGE_INSERTS": "EDGE_INSERTS",
"UUID_EDGES": "",
"LINES_PER_SECOND": 10000,
"BATCH_SIZE": 1,
},
}, },
{ {
"name": "experiment_batch10_10000", "name": "experiment_batch10_10000",
"compose_files": ("compose.yml", "edge-id.yml", "index.yml"), "compose_files": ("compose.yml", "edge-id.yml", "index.yml"),
"env": { "env": {"LINES_PER_SECOND": 10000, "BATCH_SIZE": 10},
"EDGE_INSERTS": "EDGE_INSERTS",
"UUID_EDGES": "",
"LINES_PER_SECOND": 10000,
"BATCH_SIZE": 10,
},
}, },
{ {
"name": "experiment_batch10_20000", "name": "experiment_batch10_20000",
"compose_files": ("compose.yml", "edge-id.yml", "index.yml"), "compose_files": ("compose.yml", "edge-id.yml", "index.yml"),
"env": { "env": {"LINES_PER_SECOND": 20000, "BATCH_SIZE": 10},
"EDGE_INSERTS": "EDGE_INSERTS",
"UUID_EDGES": "",
"LINES_PER_SECOND": 20000,
"BATCH_SIZE": 10,
},
}, },
{ {
"name": "experiment_batch100_10000", "name": "experiment_batch100_10000",
"compose_files": ("compose.yml", "edge-id.yml", "index.yml"), "compose_files": ("compose.yml", "edge-id.yml", "index.yml"),
"env": { "env": {"LINES_PER_SECOND": 10000, "BATCH_SIZE": 100},
"EDGE_INSERTS": "EDGE_INSERTS",
"UUID_EDGES": "",
"LINES_PER_SECOND": 10000,
"BATCH_SIZE": 100,
},
}, },
{ {
"name": "experiment_batch100_20000", "name": "experiment_batch100_20000",
"compose_files": ("compose.yml", "edge-id.yml", "index.yml"), "compose_files": ("compose.yml", "edge-id.yml", "index.yml"),
"env": { "env": {"LINES_PER_SECOND": 20000, "BATCH_SIZE": 100},
"EDGE_INSERTS": "EDGE_INSERTS",
"UUID_EDGES": "",
"LINES_PER_SECOND": 20000,
"BATCH_SIZE": 100,
},
}, },
{ {
"name": "experiment_batch1000_10000", "name": "experiment_batch1000_10000",
"compose_files": ("compose.yml", "edge-id.yml", "index.yml"), "compose_files": ("compose.yml", "edge-id.yml", "index.yml"),
"env": { "env": {"LINES_PER_SECOND": 10000, "BATCH_SIZE": 1000},
"EDGE_INSERTS": "EDGE_INSERTS",
"UUID_EDGES": "",
"LINES_PER_SECOND": 10000,
"BATCH_SIZE": 1000,
},
}, },
{ {
"name": "experiment_batch1000_20000", "name": "experiment_batch1000_20000",
"compose_files": ("compose.yml", "edge-id.yml", "index.yml"), "compose_files": ("compose.yml", "edge-id.yml", "index.yml"),
"env": { "env": {"LINES_PER_SECOND": 20000, "BATCH_SIZE": 1000},
"EDGE_INSERTS": "EDGE_INSERTS",
"UUID_EDGES": "",
"LINES_PER_SECOND": 20000,
"BATCH_SIZE": 1000,
},
}, },
{ {
"name": "experiment_no_uuid_index", "name": "experiment_no_uuid_index",
"compose_files": ("compose.yml", "edge-id-no-index.yml", "index.yml"), "compose_files": ("compose.yml", "edge-id-no-index.yml", "index.yml"),
"env": { "env": {"LINES_PER_SECOND": 1500, "BATCH_SIZE": 1500},
"EDGE_INSERTS": "EDGE_INSERTS",
"UUID_EDGES": "",
"LINES_PER_SECOND": 1500,
"BATCH_SIZE": 1500,
},
}, },
{ {
"name": "experiment_default_10000", "name": "experiment_default_10000",
"compose_files": ("compose.yml", "edge-id.yml", "index.yml"), "compose_files": ("compose.yml", "edge-id.yml", "index.yml"),
"env": { "env": {"LINES_PER_SECOND": 10000, "BATCH_SIZE": 5000},
"EDGE_INSERTS": "EDGE_INSERTS",
"UUID_EDGES": "",
"LINES_PER_SECOND": 10000,
"BATCH_SIZE": 5000,
},
}, },
{ {
"name": "experiment_default_20000", "name": "experiment_default_20000",
"compose_files": ("compose.yml", "edge-id.yml", "index.yml"), "compose_files": ("compose.yml", "edge-id.yml", "index.yml"),
"env": { "env": {"LINES_PER_SECOND": 20000, "BATCH_SIZE": 5000},
"EDGE_INSERTS": "EDGE_INSERTS",
"UUID_EDGES": "",
"LINES_PER_SECOND": 20000,
"BATCH_SIZE": 5000,
},
}, },
{ {
"name": "experiment_default_30000", "name": "experiment_default_30000",
"compose_files": ("compose.yml", "edge-id.yml", "index.yml"), "compose_files": ("compose.yml", "edge-id.yml", "index.yml"),
"env": { "env": {"LINES_PER_SECOND": 30000, "BATCH_SIZE": 5000},
"EDGE_INSERTS": "EDGE_INSERTS",
"UUID_EDGES": "",
"LINES_PER_SECOND": 30000,
"BATCH_SIZE": 5000,
},
}, },
{ {
"name": "experiment_hash_index_15000", "name": "experiment_hash_index_15000",
"compose_files": ("compose.yml", "edge-id.yml", "index-hash.yml"), "compose_files": ("compose.yml", "edge-id.yml", "index-hash.yml"),
"env": { "env": {"LINES_PER_SECOND": 15000, "BATCH_SIZE": 5000},
"EDGE_INSERTS": "EDGE_INSERTS",
"UUID_EDGES": "",
"LINES_PER_SECOND": 15000,
"BATCH_SIZE": 5000,
},
}, },
{ {
"name": "experiment_hash_index_25000", "name": "experiment_hash_index_25000",
"compose_files": ("compose.yml", "edge-id.yml", "index-hash.yml"), "compose_files": ("compose.yml", "edge-id.yml", "index-hash.yml"),
"env": { "env": {"LINES_PER_SECOND": 25000, "BATCH_SIZE": 5000},
"EDGE_INSERTS": "EDGE_INSERTS",
"UUID_EDGES": "",
"LINES_PER_SECOND": 25000,
"BATCH_SIZE": 5000,
},
}, },
{ {
"name": "experiment_multicolumn_index_15000", "name": "experiment_multicolumn_index_15000",
"compose_files": ("compose.yml", "edge-id.yml", "index-multi.yml"), "compose_files": ("compose.yml", "edge-id.yml", "index-multi.yml"),
"env": { "env": {"LINES_PER_SECOND": 15000, "BATCH_SIZE": 5000},
"EDGE_INSERTS": "EDGE_INSERTS",
"UUID_EDGES": "",
"LINES_PER_SECOND": 15000,
"BATCH_SIZE": 5000,
},
}, },
{ {
"name": "experiment_multicolumn_index_25000", "name": "experiment_multicolumn_index_25000",
"compose_files": ("compose.yml", "edge-id.yml", "index-multi.yml"), "compose_files": ("compose.yml", "edge-id.yml", "index-multi.yml"),
"env": { "env": {"LINES_PER_SECOND": 25000, "BATCH_SIZE": 5000},
"EDGE_INSERTS": "EDGE_INSERTS",
"UUID_EDGES": "",
"LINES_PER_SECOND": 25000,
"BATCH_SIZE": 5000,
},
}, },
{ {
"name": "experiment_triggers_10000", "name": "experiment_triggers_10000",
"compose_files": ("compose.yml", "edge-id-triggers.yml", "index.yml"), "compose_files": ("compose.yml", "edge-id-triggers.yml", "index.yml"),
"env": { "env": {"LINES_PER_SECOND": 10000, "BATCH_SIZE": 10000},
"EDGE_INSERTS": "",
"UUID_EDGES": "",
"LINES_PER_SECOND": 10000,
"BATCH_SIZE": 10000,
},
}, },
{ {
"name": "experiment_triggers_20000", "name": "experiment_triggers_20000",
"compose_files": ("compose.yml", "edge-id-triggers.yml", "index.yml"), "compose_files": ("compose.yml", "edge-id-triggers.yml", "index.yml"),
"env": { "env": {"LINES_PER_SECOND": 20000, "BATCH_SIZE": 20000},
"EDGE_INSERTS": "",
"UUID_EDGES": "",
"LINES_PER_SECOND": 20000,
"BATCH_SIZE": 20000,
},
}, },
{ {
"name": "experiment_triggers_30000", "name": "experiment_triggers_30000",
"compose_files": ("compose.yml", "edge-id-triggers.yml", "index.yml"), "compose_files": ("compose.yml", "edge-id-triggers.yml", "index.yml"),
"env": { "env": {"LINES_PER_SECOND": 30000, "BATCH_SIZE": 30000},
"EDGE_INSERTS": "",
"UUID_EDGES": "",
"LINES_PER_SECOND": 30000,
"BATCH_SIZE": 30000,
},
}, },
{ {
"name": "experiment_uuid_10000", "name": "experiment_uuid_10000",
"compose_files": ("compose.yml", "edge-uuid.yml", "index.yml"), "compose_files": ("compose.yml", "edge-uuid.yml", "index.yml"),
"env": { "env": {"LINES_PER_SECOND": 10000, "BATCH_SIZE": 10000},
"EDGE_INSERTS": "EDGE_INSERTS",
"UUID_EDGES": "UUID_EDGES",
"LINES_PER_SECOND": 10000,
"BATCH_SIZE": 10000,
},
}, },
{ {
"name": "experiment_uuid_20000", "name": "experiment_uuid_20000",
"compose_files": ("compose.yml", "edge-uuid.yml", "index.yml"), "compose_files": ("compose.yml", "edge-uuid.yml", "index.yml"),
"env": { "env": {"LINES_PER_SECOND": 20000, "BATCH_SIZE": 20000},
"EDGE_INSERTS": "EDGE_INSERTS",
"UUID_EDGES": "UUID_EDGES",
"LINES_PER_SECOND": 20000,
"BATCH_SIZE": 20000,
},
}, },
) )
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment