Skip to content
Snippets Groups Projects
Commit c6dcda0f authored by julian's avatar julian
Browse files

fixed some bugs with the experiments

parent d0bffb4f
No related branches found
No related tags found
No related merge requests found
......@@ -13,7 +13,7 @@ MOS_TOPIC=ta1-cadets-e3-official
# pub
LINES_PER_SECOND=10000
BATCH_SIZE=1000000
BATCH_SIZE=10000
# query
QUERY_INTERVAL=1
......
......@@ -55,7 +55,7 @@ if __name__ == "__main__":
csv.writer(file).writerow(("TIMESTAMP", "VERTEX_COUNT"))
info("Starting...")
end_time = timedelta(seconds=15)
end_time = timedelta(hours=1)
schedule.every(5).seconds.until(end_time).do(perf_job, csv_path)
while (sleep_time := schedule.idle_seconds()) is not None and not interrupted:
sleep(max(sleep_time, 0))
......
......@@ -12,7 +12,7 @@ def run_experiment(name: str, compose_files: Sequence[str], env: dict = {}):
f = ["-f"] * len(compose_files)
interleaved = (val for pair in zip(f, compose_files) for val in pair)
print("Start compose environment...")
run(("docker", "compose", *interleaved, "up", "-d"))
run(("docker", "compose", *interleaved, "up", "-d", "--build"))
print("waiting for data run to finish")
run(("docker", "compose", "wait", "query_pg"))
......@@ -27,100 +27,215 @@ if __name__ == "__main__":
if not path.exists(result_dir):
makedirs(result_dir)
experiments: Sequence[dict] = (
{
"name": "experiment_uuid_10000",
"compose_files": ("compose.yml", "edge-uuid.yml", "index.yml"),
"env": {"UUID_EDGES": "UUID_EDGES"},
},
{
"name": "experiment_uuid_20000",
"compose_files": ("compose.yml", "edge-uuid.yml", "index.yml"),
"env": {"UUID_EDGES": "UUID_EDGES", "LINES_PER_SECOND": 20000},
},
{
"name": "experiment_batch1_10000",
"compose_files": ("compose.yml", "edge-id.yml", "index.yml"),
"env": {"BATCH_SIZE": 1},
"env": {
"EDGE_INSERTS": "EDGE_INSERTS",
"UUID_EDGES": "",
"LINES_PER_SECOND": 10000,
"BATCH_SIZE": 1,
},
},
{
"name": "experiment_batch1_20000",
"compose_files": ("compose.yml", "edge-id.yml", "index.yml"),
"env": {"BATCH_SIZE": 1, "LINES_PER_SECOND": 20000},
"env": {
"EDGE_INSERTS": "EDGE_INSERTS",
"UUID_EDGES": "",
"LINES_PER_SECOND": 20000,
"BATCH_SIZE": 1,
},
},
{
"name": "experiment_batch10_10000",
"compose_files": ("compose.yml", "edge-id.yml", "index.yml"),
"env": {"BATCH_SIZE": 10},
"env": {
"EDGE_INSERTS": "EDGE_INSERTS",
"UUID_EDGES": "",
"LINES_PER_SECOND": 10000,
"BATCH_SIZE": 10,
},
},
{
"name": "experiment_batch10_20000",
"compose_files": ("compose.yml", "edge-id.yml", "index.yml"),
"env": {"BATCH_SIZE": 10, "LINES_PER_SECOND": 20000},
"env": {
"EDGE_INSERTS": "EDGE_INSERTS",
"UUID_EDGES": "",
"LINES_PER_SECOND": 20000,
"BATCH_SIZE": 10,
},
},
{
"name": "experiment_batch100_10000",
"compose_files": ("compose.yml", "edge-id.yml", "index.yml"),
"env": {"BATCH_SIZE": 100},
"env": {
"EDGE_INSERTS": "EDGE_INSERTS",
"UUID_EDGES": "",
"LINES_PER_SECOND": 10000,
"BATCH_SIZE": 100,
},
},
{
"name": "experiment_batch100_20000",
"compose_files": ("compose.yml", "edge-id.yml", "index.yml"),
"env": {"BATCH_SIZE": 100, "LINES_PER_SECOND": 20000},
"env": {
"EDGE_INSERTS": "EDGE_INSERTS",
"UUID_EDGES": "",
"LINES_PER_SECOND": 20000,
"BATCH_SIZE": 100,
},
},
{
"name": "experiment_batch1000_10000",
"compose_files": ("compose.yml", "edge-id.yml", "index.yml"),
"env": {"BATCH_SIZE": 1000},
"env": {
"EDGE_INSERTS": "EDGE_INSERTS",
"UUID_EDGES": "",
"LINES_PER_SECOND": 10000,
"BATCH_SIZE": 1000,
},
},
{
"name": "experiment_batch1000_20000",
"compose_files": ("compose.yml", "edge-id.yml", "index.yml"),
"env": {"BATCH_SIZE": 1000, "LINES_PER_SECOND": 20000},
"env": {
"EDGE_INSERTS": "EDGE_INSERTS",
"UUID_EDGES": "",
"LINES_PER_SECOND": 20000,
"BATCH_SIZE": 1000,
},
},
{
"name": "experiment_no_uuid_index",
"compose_files": ("compose.yml", "edge-id-no-index.yml", "index.yml"),
"env": {"LINES_PER_SECOND": 1500},
"env": {
"EDGE_INSERTS": "EDGE_INSERTS",
"UUID_EDGES": "",
"LINES_PER_SECOND": 1500,
"BATCH_SIZE": 1500,
},
},
{
"name": "experiment_default_10000",
"compose_files": ("compose.yml", "edge-id.yml", "index.yml"),
"env": {},
"env": {
"EDGE_INSERTS": "EDGE_INSERTS",
"UUID_EDGES": "",
"LINES_PER_SECOND": 10000,
"BATCH_SIZE": 5000,
},
},
{
"name": "experiment_default_20000",
"compose_files": ("compose.yml", "edge-id.yml", "index.yml"),
"env": {"LINES_PER_SECOND": 20000},
"env": {
"EDGE_INSERTS": "EDGE_INSERTS",
"UUID_EDGES": "",
"LINES_PER_SECOND": 20000,
"BATCH_SIZE": 5000,
},
},
{
"name": "experiment_hash_index_10000",
"name": "experiment_default_30000",
"compose_files": ("compose.yml", "edge-id.yml", "index.yml"),
"env": {
"EDGE_INSERTS": "EDGE_INSERTS",
"UUID_EDGES": "",
"LINES_PER_SECOND": 30000,
"BATCH_SIZE": 5000,
},
},
{
"name": "experiment_hash_index_15000",
"compose_files": ("compose.yml", "edge-id.yml", "index-hash.yml"),
"env": {},
"env": {
"EDGE_INSERTS": "EDGE_INSERTS",
"UUID_EDGES": "",
"LINES_PER_SECOND": 15000,
"BATCH_SIZE": 5000,
},
},
{
"name": "experiment_hash_index_20000",
"name": "experiment_hash_index_25000",
"compose_files": ("compose.yml", "edge-id.yml", "index-hash.yml"),
"env": {"LINES_PER_SECOND": 20000},
"env": {
"EDGE_INSERTS": "EDGE_INSERTS",
"UUID_EDGES": "",
"LINES_PER_SECOND": 25000,
"BATCH_SIZE": 5000,
},
},
{
"name": "experiment_multicolumn_index_10000",
"name": "experiment_multicolumn_index_15000",
"compose_files": ("compose.yml", "edge-id.yml", "index-multi.yml"),
"env": {},
"env": {
"EDGE_INSERTS": "EDGE_INSERTS",
"UUID_EDGES": "",
"LINES_PER_SECOND": 15000,
"BATCH_SIZE": 5000,
},
},
{
"name": "experiment_multicolumn_index_20000",
"name": "experiment_multicolumn_index_25000",
"compose_files": ("compose.yml", "edge-id.yml", "index-multi.yml"),
"env": {"LINES_PER_SECOND": 20000},
"env": {
"EDGE_INSERTS": "EDGE_INSERTS",
"UUID_EDGES": "",
"LINES_PER_SECOND": 25000,
"BATCH_SIZE": 5000,
},
},
{
"name": "experiment_triggers_10000",
"compose_files": ("compose.yml", "edge-id-triggers.yml", "index.yml"),
"env": {"EDGE_INSERTS": ""},
"env": {
"EDGE_INSERTS": "",
"UUID_EDGES": "",
"LINES_PER_SECOND": 10000,
"BATCH_SIZE": 10000,
},
},
{
"name": "experiment_triggers_20000",
"compose_files": ("compose.yml", "edge-id-triggers.yml", "index.yml"),
"env": {"EDGE_INSERTS": "", "LINES_PER_SECOND": 20000},
"env": {
"EDGE_INSERTS": "",
"UUID_EDGES": "",
"LINES_PER_SECOND": 20000,
"BATCH_SIZE": 20000,
},
},
{
"name": "experiment_triggers_30000",
"compose_files": ("compose.yml", "edge-id-triggers.yml", "index.yml"),
"env": {
"EDGE_INSERTS": "",
"UUID_EDGES": "",
"LINES_PER_SECOND": 30000,
"BATCH_SIZE": 30000,
},
},
{
"name": "experiment_uuid_10000",
"compose_files": ("compose.yml", "edge-uuid.yml", "index.yml"),
"env": {
"EDGE_INSERTS": "EDGE_INSERTS",
"UUID_EDGES": "UUID_EDGES",
"LINES_PER_SECOND": 10000,
"BATCH_SIZE": 10000,
},
},
{
"name": "experiment_uuid_20000",
"compose_files": ("compose.yml", "edge-uuid.yml", "index.yml"),
"env": {
"EDGE_INSERTS": "EDGE_INSERTS",
"UUID_EDGES": "UUID_EDGES",
"LINES_PER_SECOND": 20000,
"BATCH_SIZE": 20000,
},
},
)
......
......@@ -108,7 +108,9 @@ class Postgres_inserter:
if len(getenv("UUID_EDGES", "")) > 0:
cursor.executemany(*format_edge_uuid(nodes))
else:
cursor.execute(*format_edge_id(nodes))
query, params = format_edge_id(nodes)
if len(params) > 0:
cursor.execute(query, params)
self.db_conn.commit()
def on_connect(self, client: Client, _2, _3, reason_code: ReasonCode, _4):
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment