{ "protocol": "mixed-mutations-v2", "generated_utc": "2026-10-29T16:00:39.596245+00:00", "completed": 60, "failed": 0, "planned": 60, "status": "complete", "track": "reference", "experiment": { "protocol": "mixed-mutations-v2", "arguments": { "out": "artifacts/publication-v2", "scenarios": [ "scifact-small", "nq-10k", "nq-100k", "nq-100k-burst" ], "backends": [ "qdrant", "weaviate", "pgvector" ], "trials": 5, "duration": 30.0, "seed": 20261028 }, "order": [ [ 4, "weaviate", "nq-100k" ], [ 1, "weaviate", "nq-10k" ], [ 3, "weaviate", "nq-100k" ], [ 3, "qdrant", "scifact-small" ], [ 4, "pgvector", "nq-100k-burst" ], [ 1, "pgvector", "scifact-small" ], [ 2, "pgvector", "scifact-small" ], [ 1, "pgvector", "nq-100k" ], [ 1, "weaviate", "nq-100k-burst" ], [ 1, "qdrant", "nq-100k-burst" ], [ 2, "pgvector", "nq-100k-burst" ], [ 5, "qdrant", "nq-10k" ], [ 3, "pgvector", "nq-10k" ], [ 2, "weaviate", "nq-100k" ], [ 2, "qdrant", "scifact-small" ], [ 5, "pgvector", "scifact-small" ], [ 3, "weaviate", "scifact-small" ], [ 1, "weaviate", "scifact-small" ], [ 1, "pgvector", "nq-100k-burst" ], [ 2, "weaviate", "nq-10k" ], [ 3, "qdrant", "nq-10k" ], [ 2, "qdrant", "nq-100k-burst" ], [ 3, "qdrant", "nq-100k-burst" ], [ 5, "pgvector", "nq-100k" ], [ 2, "pgvector", "nq-10k" ], [ 1, "qdrant", "nq-10k" ], [ 1, "weaviate", "nq-100k" ], [ 5, "qdrant", "scifact-small" ], [ 2, "weaviate", "nq-100k-burst" ], [ 5, "pgvector", "nq-100k-burst" ], [ 5, "weaviate", "scifact-small" ], [ 3, "pgvector", "scifact-small" ], [ 4, "qdrant", "scifact-small" ], [ 2, "pgvector", "nq-100k" ], [ 3, "qdrant", "nq-100k" ], [ 5, "pgvector", "nq-10k" ], [ 4, "weaviate", "nq-10k" ], [ 4, "pgvector", "nq-10k" ], [ 4, "weaviate", "scifact-small" ], [ 5, "qdrant", "nq-100k" ], [ 5, "weaviate", "nq-10k" ], [ 5, "qdrant", "nq-100k-burst" ], [ 4, "qdrant", "nq-100k" ], [ 2, "qdrant", "nq-100k" ], [ 3, "weaviate", "nq-10k" ], [ 1, "qdrant", "scifact-small" ], [ 2, "weaviate", "scifact-small" ], [ 1, "pgvector", "nq-10k" ], [ 3, "pgvector", "nq-100k" ], [ 3, "weaviate", "nq-100k-burst" ], [ 5, "weaviate", "nq-100k" ], [ 2, "qdrant", "nq-10k" ], [ 4, "weaviate", "nq-100k-burst" ], [ 5, "weaviate", "nq-100k-burst" ], [ 1, "qdrant", "nq-100k" ], [ 4, "qdrant", "nq-10k" ], [ 4, "pgvector", "nq-100k" ], [ 4, "pgvector", "scifact-small" ], [ 3, "pgvector", "nq-100k-burst" ], [ 4, "qdrant", "nq-100k-burst" ] ], "host": { "platform": "macOS-26.6.2-arm64-arm-64bit", "cpu_count": 10, "cpu_model": "Apple M4", "memory_bytes": 17179869184 }, "source": { "src/freshness_benchmark/__init__.py": "9ebd67f8a0f5ee0cba85c72a89c11d8e4676137dd4b935ae3363567149aeaf4a", "src/freshness_benchmark/backends.py": "feda3cfba4645f84fc193f9357a95150608db0c5b98305bf67c774df60c81dac", "src/freshness_benchmark/benchmark_pack.py": "9f0f4c90028d8058521a1e8500dc7d4a6cecdeaebd5fcffa5735baadae9bc76d", "src/freshness_benchmark/calibration.py": "50110387665da94e6b84f433b1dd2648a22b7a738dcc20b9dd5ea6741fe03cbb", "src/freshness_benchmark/cli.py": "ec3f6ed5774d5687e93b1de69de8a5079d421860fd3006b2d6862f20a127a784", "src/freshness_benchmark/config.py": "8b99bee8fd8e2f782f0d6140f518b706e5f8b9847aa0af1fb8bd71631ffcae91", "src/freshness_benchmark/embeddings.py": "36fd973fddb3ecad4c23ecfe0585008a0db958452686fd3fc182ea6920d49ca8", "src/freshness_benchmark/models.py": "a7069bffd5dd931d4db16cc92c5f7fb31591f64d97df907c7f982cf0ab642d53", "src/freshness_benchmark/pgvector_backend.py": "9f8d3ebf562c1b0023add32c8bea09ee28066be5171b4f353717fd3d9bfe4eb7", "src/freshness_benchmark/probing.py": "64d6cfa5522128cfc37fa19d192671d76dac772bdf2fe9345022b9ee1abc2d28", "src/freshness_benchmark/qdrant_backend.py": "214e21e0a2d7ccd2d04456b1af7bdcee68a08027b739ce60eb61ce7803dd3f58", "src/freshness_benchmark/quality.py": "faa67a046826f995a373bd09e67d968ddd9f3d2ddb5b9c9fc01be645a8d412df", "src/freshness_benchmark/reporting.py": "849084af584f13d45840408bd914836e88b251bc4892d1a03154fb4a27e91a06", "src/freshness_benchmark/runner.py": "98bca7326badbdadc41513c79d0c68a82dd489073ee7d48cf207bba35e8858bd", "src/freshness_benchmark/streaming.py": "e997e44ef65baded334e2432a9320c0d0d70a8415f90c4e399994b266c2c5b24", "src/freshness_benchmark/streaming_backends.py": "c18dae89475f7e3059bcc5168c0275d76decf56e172be0c4fccc140dee68358c", "src/freshness_benchmark/visualize.py": "ba6d5614f309dcfaa2de5920a8d34293b85d06e82972ccba4f24cc56d72e1810", "src/freshness_benchmark/weaviate_backend.py": "dac140dec600511cf4bfb3401c1bffdc1ff2cfd2528a92b2fc344810e750cae1", "src/freshness_benchmark/workload.py": "8aeb73c89eb5e11b04c44b0be3142c6828dc8f0d8828965ba74125544d823978", "scripts/analyze_publication.py": "0e7ac4538e4131391f6e5bbe0c6bfc6118fa449c5741f71df9c1df57d266c03c", "scripts/build_website.py": "57d40f0b8602c0e31c382053f35768892baf4affcd545eae50105dbe4ccae63d", "scripts/bundle_publication.py": "420b68694e4f595143d84e1000b582969116cd56928d51282ccff8a5974bc72b", "scripts/export_v2.py": "f60ef12ef0eb33acea7d71a32c9f00ecf8894b0c0a7613c912c4e233b1e22d25", "scripts/prepare_publication_data.py": "5165bd76f4538c9cf44ae609e857f0dbf42429f142906e5e6054be2201aa5f95", "scripts/prepare_v2_data.py": "faf58acbac65454bd9b83c846144209dab40f77fee7e906af9f5e03a81020994", "scripts/run_publication.py": "6a7ef26e313f8de4181823bf0febd11cddaef8b4b1b56d698f378d9547a571c4", "scripts/run_streaming.py": "0ef376f3e7eaa18d39e83551935bfe04816277e7045f25bcfe6ad41fd035b0c4", "scripts/verify_publication.py": "98908613c84cc6a454dad8b76f158144c39df6e17dc6a9ca470d7890f7e3801d", "scripts/verify_v2.py": "8539026d07204c1bf18eaae58a93002d90e0b622805de1378cc46fbe68aa55c3", "pyproject.toml": "288fc3c00500a7df91380b0e2ccaef456c8218bee3820d62cbf9d21333536c35", "docker-compose.v2.yml": "90c8220172e1453df154e472f5106a22a9df1b67aa974cf08ca01dd253632ff1", "benchmarks/protocol-v2.json": "79493a0255e3cf6ce5b3560b3b86aa7265b63a4f7ccdbd42575ac2e05b0520cb", "benchmarks/nq-v2.lock.json": "f516dcafe7189bee2008186674baa54ccf91d5237ec8d6164895456954ecc0d9", "benchmarks/scifact-v2.lock.json": "3ec7ee4bff001b3d8370a254eef5799775ddaedb3e43e748b88ea969f83f41e5" }, "datasets": { "scifact": { "dataset": "scifact", "source": "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/scifact.zip", "source_sha256": "536e14446a0ba56ed1398ab1055f39fe852686ecad24a6306c80c490fa8e0165", "source_md5": "5f7d1de60b170fc8027bb7898e2efca1", "records": 5183, "queries": 100, "relevant_documents_retained": 107, "seed": 20261028, "license": "Corpus: ODC-By-1.0; claims/evidence: CC-BY-4.0", "license_source": "https://raw.githubusercontent.com/allenai/scifact/master/LICENSE.md", "attribution": "BEIR and original dataset authors; Wikipedia contributors for NQ, SciFact authors and cited article authors for SciFact.", "transformation": "200-word title+text prefix; seeded reservoir of documents with relevant documents for selected test queries retained at front. Simulated mutations; no historical replay.", "model": "sentence-transformers/all-MiniLM-L6-v2", "model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "dimensions": 384, "normalized": true, "max_tokens": 256, "device": "mps", "files": { "corpus.jsonl": "189d58f273468352c9f73c6333c5120dcfb7ba1695984f073f0ebb6c100ed0cd", "queries.json": "c8b333c73353a8c4aeb7d6b63067c9a241aa5d39a0314f0c315ec63019ee9558", "vectors.npy": "b6b21404977151ce565e7d6d57d9931a4679e1ecf4aa44fcd9b29eb54955d09e", "query_vectors.npy": "02324eb7c3b3d2e2f0ec3208453e2ec0efd5d61d9fc845b1f8b7950e913f2730", "UPSTREAM-LICENSE.md": "56ee69bc639cbab8f0b2e4b67091ec5e9390aa99a1d65f0a35e4274686c9a806" } }, "nq": { "dataset": "nq", "source": "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/nq.zip", "source_sha256": "2553bf7bfbab47b1436ca00a34bce57320e18e611fd00999a2a3a1b4714be752", "source_md5": "d4d3d2e48787a744b6f6e691ff534307", "records": 124000, "queries": 100, "relevant_documents_retained": 120, "seed": 20261028, "license": "CC-BY-SA-3.0", "license_source": "https://raw.githubusercontent.com/google-research-datasets/natural-questions/master/nq_open/README.md", "attribution": "BEIR and original dataset authors; Wikipedia contributors for NQ, SciFact authors and cited article authors for SciFact.", "transformation": "200-word title+text prefix; seeded reservoir of documents with relevant documents for selected test queries retained at front. Simulated mutations; no historical replay.", "model": "sentence-transformers/all-MiniLM-L6-v2", "model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "dimensions": 384, "normalized": true, "max_tokens": 256, "device": "mps", "files": { "corpus.jsonl": "9dc487d785e6ee493eccaf764daa3bf8720a8f4fc8c406246065e1ad33a91f5d", "queries.json": "5603f52876ada5d36118b65dea490f1d43900bc159e1804e637796eb9115e999", "vectors.npy": "cb7421eb6ca5964231403f0a922843088f6fd0e01cbf67f2a1076cc7101005db", "query_vectors.npy": "5e666f8d966d8db6b0fd93d4963234d2dffa0ee3910b8fad069a8ee201e8b5d4", "UPSTREAM-LICENSE.md": "8b675521da3fa142ac6bda38c9bcfcc8cff0e2d6128be53a062ed55fb53f7b87" } } }, "dependencies": "annotated-doc==0.0.5\nanyio==4.15.1\ncertifi==2026.10.23\nclick==8.5.0\ncloudpickle==3.1.2\ncontourpy==1.4.0\ncycler==0.12.1\nfilelock==4.2.4\nfonttools==4.68.0\nfsspec==2026.9.0\nh11==0.16.0\nhf-xet==1.7.1\nhttpcore==1.0.9\nhttpcore2==2.16.0\nhttpx==0.28.1\nhttpx2==2.16.0\nhuggingface-hub==2.3.0\nidna==3.20\njinja2==3.1.6\njoblib==1.6.0\nkiwisolver==1.5.1\nmarkdown-it-py==4.2.0\nmarkupsafe==3.0.4\nmatplotlib==3.11.2\nmdurl==0.1.2\nmpmath==1.3.0\nnarwhals==2.28.0\nnetworkx==3.7\nnumpy==2.5.4\npackaging==26.3\npillow==13.0.0\npsycopg==3.3.6\npsycopg-binary==3.3.6\npygments==2.21.0\npyparsing==3.3.3\npython-dateutil==3.0.0\npyyaml==6.0.3\nregex==2026.9.29\nrich==15.0.0\nsafetensors==0.8.0\nscikit-learn==1.9.1\nscipy==1.18.1\nsentence-transformers==6.2.0\nsetuptools==84.2.1\nshellingham==1.5.4\nsix==1.17.0\nsympy==1.14.0\nthreadpoolctl==3.7.0\ntokenizers==0.23.3\ntorch==2.15.0\ntqdm==4.70.1\ntransformers==5.21.0\ntruststore==0.10.5\ntyper==0.27.3\ntyping-extensions==4.16.0\ntzdata==2026.6", "docker_host": { "NCPU": 10, "MemTotal": 8217448448, "KernelVersion": "6.12.54-linuxkit", "OperatingSystem": "Docker Desktop", "ServerVersion": "29.1.2" }, "containers": [ { "name": "/freshness-v2-qdrant", "image_id": "sha256:9bfc89f9914fc14876a88c05270685921a55b8877b254a98bcc1cc2f0a424cd2", "image": "qdrant/qdrant:v1.20.0", "memory_bytes": 3221225472, "shm_bytes": 67108864, "cpu_quota": 2000000000, "async_indexing": null }, { "name": "/freshness-v2-weaviate", "image_id": "sha256:e96e2e04ee6f401a8dcdf89adb80b610516e1491a33769dfed6d1318801e1129", "image": "cr.weaviate.io/semitechnologies/weaviate:1.40.3", "memory_bytes": 3221225472, "shm_bytes": 67108864, "cpu_quota": 2000000000, "async_indexing": "ASYNC_INDEXING=false" }, { "name": "/freshness-v2-pgvector", "image_id": "sha256:7b822b0aac60967beb1ea5e576b8602c94c300a157d187f385ae3e0da199b90a", "image": "pgvector/pgvector:pg16@sha256:7b822b0aac60967beb1ea5e576b8602c94c300a157d187f385ae3e0da199b90a", "memory_bytes": 3221225472, "shm_bytes": 1073741824, "cpu_quota": 2000000000, "async_indexing": null } ] }, "groups": [ { "scenario": "nq-100k", "backend": "pgvector", "operation": "insert", "trials": 5, "expected_trials": 5, "failed_runs": 0, "events": 500, "success": 500, "success_rate": 1.0, "first_p95": 35.69996252772398, "first_ci": [ 34.604297328041866, 36.606266745366156 ], "confirmed_p95": 97.7326333464589, "confirmed_ci": [ 96.78471490042284, 101.72839760780334 ], "query_p95": 15.796393709024414, "query_ci": [ 14.819730957970023, 16.50958546670154 ], "ann_recall": 0.994, "ndcg": 0.7931268675492565, "idle_query_p95": 22.15062077739276, "exact_ndcg": 0.7931268675492565, "within_100ms": 0.976, "within_1000ms": 1.0, "regressions": 0, "eligible": true, "failed_gates": [], "run_ids": [ "fb2_429e9a_007", "fb2_429e9a_023", "fb2_429e9a_033", "fb2_429e9a_048", "fb2_429e9a_056" ] }, { "scenario": "nq-100k", "backend": "pgvector", "operation": "update", "trials": 5, "expected_trials": 5, "failed_runs": 0, "events": 500, "success": 500, "success_rate": 1.0, "first_p95": 33.677660051034756, "first_ci": [ 33.07623571599834, 34.444959729444236 ], "confirmed_p95": 95.30571732320823, "confirmed_ci": [ 94.33021912700497, 96.57452050596476 ], "query_p95": 15.796393709024414, "query_ci": [ 14.819730957970023, 16.50958546670154 ], "ann_recall": 0.994, "ndcg": 0.7931268675492565, "idle_query_p95": 22.15062077739276, "exact_ndcg": 0.7931268675492565, "within_100ms": 0.994, "within_1000ms": 1.0, "regressions": 0, "eligible": true, "failed_gates": [], "run_ids": [ "fb2_429e9a_007", "fb2_429e9a_023", "fb2_429e9a_033", "fb2_429e9a_048", "fb2_429e9a_056" ] }, { "scenario": "nq-100k", "backend": "pgvector", "operation": "delete", "trials": 5, "expected_trials": 5, "failed_runs": 0, "events": 500, "success": 500, "success_rate": 1.0, "first_p95": 31.244569044793025, "first_ci": [ 29.924027115339413, 31.789086025673896 ], "confirmed_p95": 95.1784104516264, "confirmed_ci": [ 94.00878751184791, 97.07282468443736 ], "query_p95": 15.796393709024414, "query_ci": [ 14.819730957970023, 16.50958546670154 ], "ann_recall": 0.994, "ndcg": 0.7931268675492565, "idle_query_p95": 22.15062077739276, "exact_ndcg": 0.7931268675492565, "within_100ms": 0.984, "within_1000ms": 1.0, "regressions": 0, "eligible": true, "failed_gates": [], "run_ids": [ "fb2_429e9a_007", "fb2_429e9a_023", "fb2_429e9a_033", "fb2_429e9a_048", "fb2_429e9a_056" ] }, { "scenario": "nq-100k", "backend": "qdrant", "operation": "insert", "trials": 5, "expected_trials": 5, "failed_runs": 0, "events": 500, "success": 500, "success_rate": 1.0, "first_p95": 72.88419574615546, "first_ci": [ 72.10925590479746, 105.23606811184435 ], "confirmed_p95": 222.8782527614385, "confirmed_ci": [ 218.97600833908655, 233.09823964373203 ], "query_p95": 12.25551397074014, "query_ci": [ 11.14022918045521, 47.87565842270851 ], "ann_recall": 0.989, "ndcg": 0.7931268675492565, "idle_query_p95": 12.67668814980425, "exact_ndcg": 0.7931268675492565, "within_100ms": 0.0, "within_1000ms": 1.0, "regressions": 0, "eligible": true, "failed_gates": [], "run_ids": [ "fb2_429e9a_034", "fb2_429e9a_039", "fb2_429e9a_042", "fb2_429e9a_043", "fb2_429e9a_054" ] }, { "scenario": "nq-100k", "backend": "qdrant", "operation": "update", "trials": 5, "expected_trials": 5, "failed_runs": 0, "events": 500, "success": 500, "success_rate": 1.0, "first_p95": 71.59317740588449, "first_ci": [ 70.96087315003388, 74.27814853726886 ], "confirmed_p95": 222.3006479733158, "confirmed_ci": [ 221.1335648142267, 224.4344085920602 ], "query_p95": 12.25551397074014, "query_ci": [ 11.14022918045521, 47.87565842270851 ], "ann_recall": 0.989, "ndcg": 0.7931268675492565, "idle_query_p95": 12.67668814980425, "exact_ndcg": 0.7931268675492565, "within_100ms": 0.0, "within_1000ms": 1.0, "regressions": 0, "eligible": true, "failed_gates": [], "run_ids": [ "fb2_429e9a_034", "fb2_429e9a_039", "fb2_429e9a_042", "fb2_429e9a_043", "fb2_429e9a_054" ] }, { "scenario": "nq-100k", "backend": "qdrant", "operation": "delete", "trials": 5, "expected_trials": 5, "failed_runs": 0, "events": 500, "success": 500, "success_rate": 1.0, "first_p95": 21.21363806654699, "first_ci": [ 20.734089275356382, 47.99626526655628 ], "confirmed_p95": 161.32449613069184, "confirmed_ci": [ 159.60381933837198, 181.20786278159355 ], "query_p95": 12.25551397074014, "query_ci": [ 11.14022918045521, 47.87565842270851 ], "ann_recall": 0.989, "ndcg": 0.7931268675492565, "idle_query_p95": 12.67668814980425, "exact_ndcg": 0.7931268675492565, "within_100ms": 0.0, "within_1000ms": 1.0, "regressions": 0, "eligible": true, "failed_gates": [], "run_ids": [ "fb2_429e9a_034", "fb2_429e9a_039", "fb2_429e9a_042", "fb2_429e9a_043", "fb2_429e9a_054" ] }, { "scenario": "nq-100k", "backend": "weaviate", "operation": "insert", "trials": 5, "expected_trials": 5, "failed_runs": 0, "events": 500, "success": 500, "success_rate": 1.0, "first_p95": 34.05072229797952, "first_ci": [ 32.879156223498285, 35.34893480245955 ], "confirmed_p95": 106.31492319516838, "confirmed_ci": [ 105.1834607613273, 106.84662737767212 ], "query_p95": 15.99512090906501, "query_ci": [ 14.928012469317766, 16.254522872623056 ], "ann_recall": 0.991, "ndcg": 0.7931268675492565, "idle_query_p95": 13.481597660575062, "exact_ndcg": 0.7931268675492565, "within_100ms": 0.786, "within_1000ms": 1.0, "regressions": 0, "eligible": true, "failed_gates": [], "run_ids": [ "fb2_429e9a_000", "fb2_429e9a_002", "fb2_429e9a_013", "fb2_429e9a_026", "fb2_429e9a_050" ] }, { "scenario": "nq-100k", "backend": "weaviate", "operation": "update", "trials": 5, "expected_trials": 5, "failed_runs": 0, "events": 500, "success": 500, "success_rate": 1.0, "first_p95": 32.30716179823503, "first_ci": [ 30.47465423005633, 33.65717075066641 ], "confirmed_p95": 104.0129562432412, "confirmed_ci": [ 103.17273990367539, 106.19459378067404 ], "query_p95": 15.99512090906501, "query_ci": [ 14.928012469317766, 16.254522872623056 ], "ann_recall": 0.991, "ndcg": 0.7931268675492565, "idle_query_p95": 13.481597660575062, "exact_ndcg": 0.7931268675492565, "within_100ms": 0.862, "within_1000ms": 1.0, "regressions": 0, "eligible": true, "failed_gates": [], "run_ids": [ "fb2_429e9a_000", "fb2_429e9a_002", "fb2_429e9a_013", "fb2_429e9a_026", "fb2_429e9a_050" ] }, { "scenario": "nq-100k", "backend": "weaviate", "operation": "delete", "trials": 5, "expected_trials": 5, "failed_runs": 0, "events": 500, "success": 500, "success_rate": 1.0, "first_p95": 20.256677718134597, "first_ci": [ 20.212427078513425, 25.11313746799715 ], "confirmed_p95": 87.68574438290669, "confirmed_ci": [ 86.34342438890599, 94.06139792408794 ], "query_p95": 15.99512090906501, "query_ci": [ 14.928012469317766, 16.254522872623056 ], "ann_recall": 0.991, "ndcg": 0.7931268675492565, "idle_query_p95": 13.481597660575062, "exact_ndcg": 0.7931268675492565, "within_100ms": 0.978, "within_1000ms": 1.0, "regressions": 0, "eligible": true, "failed_gates": [], "run_ids": [ "fb2_429e9a_000", "fb2_429e9a_002", "fb2_429e9a_013", "fb2_429e9a_026", "fb2_429e9a_050" ] }, { "scenario": "nq-100k-burst", "backend": "pgvector", "operation": "insert", "trials": 5, "expected_trials": 5, "failed_runs": 0, "events": 650, "success": 650, "success_rate": 1.0, "first_p95": 34.40463942242786, "first_ci": [ 33.20730993291363, 35.7725894311443 ], "confirmed_p95": 96.82332848897204, "confirmed_ci": [ 95.13019393780269, 97.59422272909433 ], "query_p95": 14.714500034460801, "query_ci": [ 14.25962975481525, 15.092180948704483 ], "ann_recall": 0.994, "ndcg": 0.7931268675492565, "idle_query_p95": 22.09521274198778, "exact_ndcg": 0.7931268675492565, "within_100ms": 0.9876923076923076, "within_1000ms": 1.0, "regressions": 0, "eligible": true, "failed_gates": [], "run_ids": [ "fb2_429e9a_004", "fb2_429e9a_010", "fb2_429e9a_018", "fb2_429e9a_029", "fb2_429e9a_058" ] }, { "scenario": "nq-100k-burst", "backend": "pgvector", "operation": "update", "trials": 5, "expected_trials": 5, "failed_runs": 0, "events": 650, "success": 650, "success_rate": 1.0, "first_p95": 30.92455990263261, "first_ci": [ 30.471292044967413, 31.7727543937508 ], "confirmed_p95": 92.13067924138159, "confirmed_ci": [ 91.92856453591958, 93.06218500132672 ], "query_p95": 14.714500034460801, "query_ci": [ 14.25962975481525, 15.092180948704483 ], "ann_recall": 0.994, "ndcg": 0.7931268675492565, "idle_query_p95": 22.09521274198778, "exact_ndcg": 0.7931268675492565, "within_100ms": 1.0, "within_1000ms": 1.0, "regressions": 0, "eligible": true, "failed_gates": [], "run_ids": [ "fb2_429e9a_004", "fb2_429e9a_010", "fb2_429e9a_018", "fb2_429e9a_029", "fb2_429e9a_058" ] }, { "scenario": "nq-100k-burst", "backend": "pgvector", "operation": "delete", "trials": 5, "expected_trials": 5, "failed_runs": 0, "events": 650, "success": 650, "success_rate": 1.0, "first_p95": 29.47893098462373, "first_ci": [ 28.83260215166956, 30.597235780442134 ], "confirmed_p95": 93.75949619570747, "confirmed_ci": [ 91.41639804583974, 94.38110638875514 ], "query_p95": 14.714500034460801, "query_ci": [ 14.25962975481525, 15.092180948704483 ], "ann_recall": 0.994, "ndcg": 0.7931268675492565, "idle_query_p95": 22.09521274198778, "exact_ndcg": 0.7931268675492565, "within_100ms": 0.9923076923076923, "within_1000ms": 1.0, "regressions": 0, "eligible": true, "failed_gates": [], "run_ids": [ "fb2_429e9a_004", "fb2_429e9a_010", "fb2_429e9a_018", "fb2_429e9a_029", "fb2_429e9a_058" ] }, { "scenario": "nq-100k-burst", "backend": "qdrant", "operation": "insert", "trials": 5, "expected_trials": 5, "failed_runs": 0, "events": 650, "success": 650, "success_rate": 1.0, "first_p95": 105.21178120397963, "first_ci": [ 103.72933360631578, 106.35504775564186 ], "confirmed_p95": 246.85336668626405, "confirmed_ci": [ 243.3339831593912, 251.59784343559295 ], "query_p95": 48.03217915468849, "query_ci": [ 47.127238987013705, 49.616958398837596 ], "ann_recall": 0.991, "ndcg": 0.7931268675492565, "idle_query_p95": 11.264227709034456, "exact_ndcg": 0.7931268675492565, "within_100ms": 0.0, "within_1000ms": 1.0, "regressions": 0, "eligible": true, "failed_gates": [], "run_ids": [ "fb2_429e9a_009", "fb2_429e9a_021", "fb2_429e9a_022", "fb2_429e9a_041", "fb2_429e9a_059" ] }, { "scenario": "nq-100k-burst", "backend": "qdrant", "operation": "update", "trials": 5, "expected_trials": 5, "failed_runs": 0, "events": 650, "success": 650, "success_rate": 1.0, "first_p95": 102.87115379469469, "first_ci": [ 102.69042053259909, 104.7482599853538 ], "confirmed_p95": 244.14518572739325, "confirmed_ci": [ 243.98482901160605, 248.60607528244145 ], "query_p95": 48.03217915468849, "query_ci": [ 47.127238987013705, 49.616958398837596 ], "ann_recall": 0.991, "ndcg": 0.7931268675492565, "idle_query_p95": 11.264227709034456, "exact_ndcg": 0.7931268675492565, "within_100ms": 0.0, "within_1000ms": 1.0, "regressions": 0, "eligible": true, "failed_gates": [], "run_ids": [ "fb2_429e9a_009", "fb2_429e9a_021", "fb2_429e9a_022", "fb2_429e9a_041", "fb2_429e9a_059" ] }, { "scenario": "nq-100k-burst", "backend": "qdrant", "operation": "delete", "trials": 5, "expected_trials": 5, "failed_runs": 0, "events": 650, "success": 650, "success_rate": 1.0, "first_p95": 47.999368741875514, "first_ci": [ 46.640510723227635, 51.40850858297199 ], "confirmed_p95": 189.79566668276675, "confirmed_ci": [ 187.9890499811154, 201.17622331017628 ], "query_p95": 48.03217915468849, "query_ci": [ 47.127238987013705, 49.616958398837596 ], "ann_recall": 0.991, "ndcg": 0.7931268675492565, "idle_query_p95": 11.264227709034456, "exact_ndcg": 0.7931268675492565, "within_100ms": 0.0, "within_1000ms": 1.0, "regressions": 0, "eligible": true, "failed_gates": [], "run_ids": [ "fb2_429e9a_009", "fb2_429e9a_021", "fb2_429e9a_022", "fb2_429e9a_041", "fb2_429e9a_059" ] }, { "scenario": "nq-100k-burst", "backend": "weaviate", "operation": "insert", "trials": 5, "expected_trials": 5, "failed_runs": 0, "events": 650, "success": 650, "success_rate": 1.0, "first_p95": 31.982566096121445, "first_ci": [ 31.26471490249969, 32.53313969471492 ], "confirmed_p95": 104.35666070552543, "confirmed_ci": [ 103.5491458082106, 105.7334188837558 ], "query_p95": 15.96772765624337, "query_ci": [ 14.804920769529417, 16.295252030249685 ], "ann_recall": 0.991, "ndcg": 0.7931268675492565, "idle_query_p95": 13.225008343579246, "exact_ndcg": 0.7931268675492565, "within_100ms": 0.9, "within_1000ms": 1.0, "regressions": 0, "eligible": true, "failed_gates": [], "run_ids": [ "fb2_429e9a_008", "fb2_429e9a_028", "fb2_429e9a_049", "fb2_429e9a_052", "fb2_429e9a_053" ] }, { "scenario": "nq-100k-burst", "backend": "weaviate", "operation": "update", "trials": 5, "expected_trials": 5, "failed_runs": 0, "events": 650, "success": 650, "success_rate": 1.0, "first_p95": 32.93857527896762, "first_ci": [ 32.38528749789111, 33.66597680142149 ], "confirmed_p95": 104.62289359420538, "confirmed_ci": [ 103.3983581524808, 105.57187323574908 ], "query_p95": 15.96772765624337, "query_ci": [ 14.804920769529417, 16.295252030249685 ], "ann_recall": 0.991, "ndcg": 0.7931268675492565, "idle_query_p95": 13.225008343579246, "exact_ndcg": 0.7931268675492565, "within_100ms": 0.8646153846153846, "within_1000ms": 1.0, "regressions": 0, "eligible": true, "failed_gates": [], "run_ids": [ "fb2_429e9a_008", "fb2_429e9a_028", "fb2_429e9a_049", "fb2_429e9a_052", "fb2_429e9a_053" ] }, { "scenario": "nq-100k-burst", "backend": "weaviate", "operation": "delete", "trials": 5, "expected_trials": 5, "failed_runs": 0, "events": 650, "success": 650, "success_rate": 1.0, "first_p95": 19.055741681950167, "first_ci": [ 18.518729164497927, 20.836546184727922 ], "confirmed_p95": 86.66508560418151, "confirmed_ci": [ 85.46466057305224, 87.09083738503978 ], "query_p95": 15.96772765624337, "query_ci": [ 14.804920769529417, 16.295252030249685 ], "ann_recall": 0.991, "ndcg": 0.7931268675492565, "idle_query_p95": 13.225008343579246, "exact_ndcg": 0.7931268675492565, "within_100ms": 0.9907692307692307, "within_1000ms": 1.0, "regressions": 0, "eligible": true, "failed_gates": [], "run_ids": [ "fb2_429e9a_008", "fb2_429e9a_028", "fb2_429e9a_049", "fb2_429e9a_052", "fb2_429e9a_053" ] }, { "scenario": "nq-10k", "backend": "pgvector", "operation": "insert", "trials": 5, "expected_trials": 5, "failed_runs": 0, "events": 500, "success": 500, "success_rate": 1.0, "first_p95": 25.949552131351084, "first_ci": [ 25.324377499055117, 28.122695168713108 ], "confirmed_p95": 91.47805799148045, "confirmed_ci": [ 89.19969765702263, 94.8239017801825 ], "query_p95": 7.819029240636155, "query_ci": [ 6.980877689784393, 8.229849993949758 ], "ann_recall": 0.999, "ndcg": 0.9481505328688746, "idle_query_p95": 8.941639313707128, "exact_ndcg": 0.9481505328688745, "within_100ms": 0.994, "within_1000ms": 1.0, "regressions": 0, "eligible": true, "failed_gates": [], "run_ids": [ "fb2_429e9a_012", "fb2_429e9a_024", "fb2_429e9a_035", "fb2_429e9a_037", "fb2_429e9a_047" ] }, { "scenario": "nq-10k", "backend": "pgvector", "operation": "update", "trials": 5, "expected_trials": 5, "failed_runs": 0, "events": 500, "success": 500, "success_rate": 1.0, "first_p95": 25.98592469585128, "first_ci": [ 25.759424740681425, 26.793889555847272 ], "confirmed_p95": 91.28019412746653, "confirmed_ci": [ 90.40831254096702, 93.3352726220619 ], "query_p95": 7.819029240636155, "query_ci": [ 6.980877689784393, 8.229849993949758 ], "ann_recall": 0.999, "ndcg": 0.9481505328688746, "idle_query_p95": 8.941639313707128, "exact_ndcg": 0.9481505328688745, "within_100ms": 1.0, "within_1000ms": 1.0, "regressions": 0, "eligible": true, "failed_gates": [], "run_ids": [ "fb2_429e9a_012", "fb2_429e9a_024", "fb2_429e9a_035", "fb2_429e9a_037", "fb2_429e9a_047" ] }, { "scenario": "nq-10k", "backend": "pgvector", "operation": "delete", "trials": 5, "expected_trials": 5, "failed_runs": 0, "events": 500, "success": 500, "success_rate": 1.0, "first_p95": 24.235108389984816, "first_ci": [ 23.484032740816474, 25.059871160192415 ], "confirmed_p95": 89.95891908416525, "confirmed_ci": [ 87.45332955149934, 90.4052707424853 ], "query_p95": 7.819029240636155, "query_ci": [ 6.980877689784393, 8.229849993949758 ], "ann_recall": 0.999, "ndcg": 0.9481505328688746, "idle_query_p95": 8.941639313707128, "exact_ndcg": 0.9481505328688745, "within_100ms": 1.0, "within_1000ms": 1.0, "regressions": 0, "eligible": true, "failed_gates": [], "run_ids": [ "fb2_429e9a_012", "fb2_429e9a_024", "fb2_429e9a_035", "fb2_429e9a_037", "fb2_429e9a_047" ] }, { "scenario": "nq-10k", "backend": "qdrant", "operation": "insert", "trials": 5, "expected_trials": 5, "failed_runs": 0, "events": 500, "success": 500, "success_rate": 1.0, "first_p95": 71.65691102272831, "first_ci": [ 70.81292299553752, 80.69105662871146 ], "confirmed_p95": 218.27049792627804, "confirmed_ci": [ 216.64947674726136, 219.56211106735282 ], "query_p95": 11.17367711267434, "query_ci": [ 10.658175317803398, 15.280519053339978 ], "ann_recall": 0.999, "ndcg": 0.948294881680092, "idle_query_p95": 9.784028783906251, "exact_ndcg": 0.9481505328688745, "within_100ms": 0.0, "within_1000ms": 1.0, "regressions": 0, "eligible": true, "failed_gates": [], "run_ids": [ "fb2_429e9a_011", "fb2_429e9a_020", "fb2_429e9a_025", "fb2_429e9a_051", "fb2_429e9a_055" ] }, { "scenario": "nq-10k", "backend": "qdrant", "operation": "update", "trials": 5, "expected_trials": 5, "failed_runs": 0, "events": 500, "success": 500, "success_rate": 1.0, "first_p95": 69.54829166061245, "first_ci": [ 69.16556035284884, 70.77660204377025 ], "confirmed_p95": 216.60936185508035, "confirmed_ci": [ 216.19951814063825, 219.40074206795543 ], "query_p95": 11.17367711267434, "query_ci": [ 10.658175317803398, 15.280519053339978 ], "ann_recall": 0.999, "ndcg": 0.948294881680092, "idle_query_p95": 9.784028783906251, "exact_ndcg": 0.9481505328688745, "within_100ms": 0.0, "within_1000ms": 1.0, "regressions": 0, "eligible": true, "failed_gates": [], "run_ids": [ "fb2_429e9a_011", "fb2_429e9a_020", "fb2_429e9a_025", "fb2_429e9a_051", "fb2_429e9a_055" ] }, { "scenario": "nq-10k", "backend": "qdrant", "operation": "delete", "trials": 5, "expected_trials": 5, "failed_runs": 0, "events": 500, "success": 500, "success_rate": 1.0, "first_p95": 23.390629212371934, "first_ci": [ 21.049419423798096, 59.267306269612156 ], "confirmed_p95": 159.06769998837262, "confirmed_ci": [ 157.74073152570054, 203.17447943380103 ], "query_p95": 11.17367711267434, "query_ci": [ 10.658175317803398, 15.280519053339978 ], "ann_recall": 0.999, "ndcg": 0.948294881680092, "idle_query_p95": 9.784028783906251, "exact_ndcg": 0.9481505328688745, "within_100ms": 0.0, "within_1000ms": 1.0, "regressions": 0, "eligible": true, "failed_gates": [], "run_ids": [ "fb2_429e9a_011", "fb2_429e9a_020", "fb2_429e9a_025", "fb2_429e9a_051", "fb2_429e9a_055" ] }, { "scenario": "nq-10k", "backend": "weaviate", "operation": "insert", "trials": 5, "expected_trials": 5, "failed_runs": 0, "events": 500, "success": 500, "success_rate": 1.0, "first_p95": 30.464852188015357, "first_ci": [ 30.209300003480163, 31.550645863171663 ], "confirmed_p95": 103.42967112665065, "confirmed_ci": [ 98.07594587909989, 103.87551870662719 ], "query_p95": 14.887731295311827, "query_ci": [ 14.259504532674333, 15.589810459641745 ], "ann_recall": 0.996, "ndcg": 0.9481505328688746, "idle_query_p95": 12.605095619801432, "exact_ndcg": 0.9481505328688745, "within_100ms": 0.934, "within_1000ms": 1.0, "regressions": 0, "eligible": true, "failed_gates": [], "run_ids": [ "fb2_429e9a_001", "fb2_429e9a_019", "fb2_429e9a_036", "fb2_429e9a_040", "fb2_429e9a_044" ] }, { "scenario": "nq-10k", "backend": "weaviate", "operation": "update", "trials": 5, "expected_trials": 5, "failed_runs": 0, "events": 500, "success": 500, "success_rate": 1.0, "first_p95": 31.648829899495468, "first_ci": [ 29.868601850466803, 34.30093359784223 ], "confirmed_p95": 103.14910382730886, "confirmed_ci": [ 98.47702120896429, 103.83872464881279 ], "query_p95": 14.887731295311827, "query_ci": [ 14.259504532674333, 15.589810459641745 ], "ann_recall": 0.996, "ndcg": 0.9481505328688746, "idle_query_p95": 12.605095619801432, "exact_ndcg": 0.9481505328688745, "within_100ms": 0.914, "within_1000ms": 1.0, "regressions": 0, "eligible": true, "failed_gates": [], "run_ids": [ "fb2_429e9a_001", "fb2_429e9a_019", "fb2_429e9a_036", "fb2_429e9a_040", "fb2_429e9a_044" ] }, { "scenario": "nq-10k", "backend": "weaviate", "operation": "delete", "trials": 5, "expected_trials": 5, "failed_runs": 0, "events": 500, "success": 500, "success_rate": 1.0, "first_p95": 21.343398588942357, "first_ci": [ 19.993846188299358, 26.37408274458721 ], "confirmed_p95": 86.32707701181062, "confirmed_ci": [ 84.78239559917711, 93.39297685655765 ], "query_p95": 14.887731295311827, "query_ci": [ 14.259504532674333, 15.589810459641745 ], "ann_recall": 0.996, "ndcg": 0.9481505328688746, "idle_query_p95": 12.605095619801432, "exact_ndcg": 0.9481505328688745, "within_100ms": 0.994, "within_1000ms": 1.0, "regressions": 0, "eligible": true, "failed_gates": [], "run_ids": [ "fb2_429e9a_001", "fb2_429e9a_019", "fb2_429e9a_036", "fb2_429e9a_040", "fb2_429e9a_044" ] }, { "scenario": "scifact-small", "backend": "pgvector", "operation": "insert", "trials": 5, "expected_trials": 5, "failed_runs": 0, "events": 500, "success": 500, "success_rate": 1.0, "first_p95": 23.143760720267892, "first_ci": [ 22.381539863999933, 25.18198542529717 ], "confirmed_p95": 85.2607398875989, "confirmed_ci": [ 83.67925615166314, 86.06329791946337 ], "query_p95": 6.18157065473497, "query_ci": [ 5.788843223126605, 6.378347240388393 ], "ann_recall": 0.9993333333333334, "ndcg": 0.6567157762068453, "idle_query_p95": 7.320660422556102, "exact_ndcg": 0.6567157762068453, "within_100ms": 1.0, "within_1000ms": 1.0, "regressions": 0, "eligible": true, "failed_gates": [], "run_ids": [ "fb2_429e9a_005", "fb2_429e9a_006", "fb2_429e9a_015", "fb2_429e9a_031", "fb2_429e9a_057" ] }, { "scenario": "scifact-small", "backend": "pgvector", "operation": "update", "trials": 5, "expected_trials": 5, "failed_runs": 0, "events": 500, "success": 500, "success_rate": 1.0, "first_p95": 22.381274751387537, "first_ci": [ 22.239129821537063, 23.853521165437993 ], "confirmed_p95": 84.7152583883144, "confirmed_ci": [ 84.26738956477493, 85.30728297191672 ], "query_p95": 6.18157065473497, "query_ci": [ 5.788843223126605, 6.378347240388393 ], "ann_recall": 0.9993333333333334, "ndcg": 0.6567157762068453, "idle_query_p95": 7.320660422556102, "exact_ndcg": 0.6567157762068453, "within_100ms": 0.998, "within_1000ms": 1.0, "regressions": 0, "eligible": true, "failed_gates": [], "run_ids": [ "fb2_429e9a_005", "fb2_429e9a_006", "fb2_429e9a_015", "fb2_429e9a_031", "fb2_429e9a_057" ] }, { "scenario": "scifact-small", "backend": "pgvector", "operation": "delete", "trials": 5, "expected_trials": 5, "failed_runs": 0, "events": 500, "success": 500, "success_rate": 1.0, "first_p95": 21.36173989274539, "first_ci": [ 21.1818712297827, 22.03007848584093 ], "confirmed_p95": 82.703517266782, "confirmed_ci": [ 81.88061274122447, 83.92241245601326 ], "query_p95": 6.18157065473497, "query_ci": [ 5.788843223126605, 6.378347240388393 ], "ann_recall": 0.9993333333333334, "ndcg": 0.6567157762068453, "idle_query_p95": 7.320660422556102, "exact_ndcg": 0.6567157762068453, "within_100ms": 1.0, "within_1000ms": 1.0, "regressions": 0, "eligible": true, "failed_gates": [], "run_ids": [ "fb2_429e9a_005", "fb2_429e9a_006", "fb2_429e9a_015", "fb2_429e9a_031", "fb2_429e9a_057" ] }, { "scenario": "scifact-small", "backend": "qdrant", "operation": "insert", "trials": 5, "expected_trials": 5, "failed_runs": 0, "events": 500, "success": 500, "success_rate": 1.0, "first_p95": 104.41014791140333, "first_ci": [ 69.95584963588044, 104.77418363443576 ], "confirmed_p95": 213.808337575756, "confirmed_ci": [ 211.82654576841742, 217.67850241158155 ], "query_p95": 10.314891714369878, "query_ci": [ 9.489391755778342, 45.82351447897963 ], "ann_recall": 1.0, "ndcg": 0.6567157762068453, "idle_query_p95": 8.48379306262359, "exact_ndcg": 0.6567157762068453, "within_100ms": 0.0, "within_1000ms": 1.0, "regressions": 0, "eligible": true, "failed_gates": [], "run_ids": [ "fb2_429e9a_003", "fb2_429e9a_014", "fb2_429e9a_027", "fb2_429e9a_032", "fb2_429e9a_045" ] }, { "scenario": "scifact-small", "backend": "qdrant", "operation": "update", "trials": 5, "expected_trials": 5, "failed_runs": 0, "events": 500, "success": 500, "success_rate": 1.0, "first_p95": 67.50240036053583, "first_ci": [ 65.92168157803826, 75.39194647106342 ], "confirmed_p95": 212.79386245878413, "confirmed_ci": [ 211.2761639582459, 213.29157467698678 ], "query_p95": 10.314891714369878, "query_ci": [ 9.489391755778342, 45.82351447897963 ], "ann_recall": 1.0, "ndcg": 0.6567157762068453, "idle_query_p95": 8.48379306262359, "exact_ndcg": 0.6567157762068453, "within_100ms": 0.0, "within_1000ms": 1.0, "regressions": 0, "eligible": true, "failed_gates": [], "run_ids": [ "fb2_429e9a_003", "fb2_429e9a_014", "fb2_429e9a_027", "fb2_429e9a_032", "fb2_429e9a_045" ] }, { "scenario": "scifact-small", "backend": "qdrant", "operation": "delete", "trials": 5, "expected_trials": 5, "failed_runs": 0, "events": 500, "success": 500, "success_rate": 1.0, "first_p95": 60.77254583942703, "first_ci": [ 58.19412565906532, 63.427677791332826 ], "confirmed_p95": 204.8482791869901, "confirmed_ci": [ 198.76028327271342, 205.9482375276275 ], "query_p95": 10.314891714369878, "query_ci": [ 9.489391755778342, 45.82351447897963 ], "ann_recall": 1.0, "ndcg": 0.6567157762068453, "idle_query_p95": 8.48379306262359, "exact_ndcg": 0.6567157762068453, "within_100ms": 0.0, "within_1000ms": 1.0, "regressions": 0, "eligible": true, "failed_gates": [], "run_ids": [ "fb2_429e9a_003", "fb2_429e9a_014", "fb2_429e9a_027", "fb2_429e9a_032", "fb2_429e9a_045" ] }, { "scenario": "scifact-small", "backend": "weaviate", "operation": "insert", "trials": 5, "expected_trials": 5, "failed_runs": 0, "events": 500, "success": 500, "success_rate": 1.0, "first_p95": 30.845760786905885, "first_ci": [ 29.90001904545352, 31.52760628145188 ], "confirmed_p95": 102.79049620730802, "confirmed_ci": [ 96.51153514860198, 103.16983934026212 ], "query_p95": 14.374172559473664, "query_ci": [ 13.32284351810813, 15.721488045528533 ], "ann_recall": 1.0, "ndcg": 0.6567157762068453, "idle_query_p95": 12.13232510490343, "exact_ndcg": 0.6567157762068453, "within_100ms": 0.942, "within_1000ms": 1.0, "regressions": 0, "eligible": true, "failed_gates": [], "run_ids": [ "fb2_429e9a_016", "fb2_429e9a_017", "fb2_429e9a_030", "fb2_429e9a_038", "fb2_429e9a_046" ] }, { "scenario": "scifact-small", "backend": "weaviate", "operation": "update", "trials": 5, "expected_trials": 5, "failed_runs": 0, "events": 500, "success": 500, "success_rate": 1.0, "first_p95": 30.44567526085302, "first_ci": [ 28.83286604192108, 32.10251426207833 ], "confirmed_p95": 102.80484800459817, "confirmed_ci": [ 96.13994136452675, 103.50165865966119 ], "query_p95": 14.374172559473664, "query_ci": [ 13.32284351810813, 15.721488045528533 ], "ann_recall": 1.0, "ndcg": 0.6567157762068453, "idle_query_p95": 12.13232510490343, "exact_ndcg": 0.6567157762068453, "within_100ms": 0.95, "within_1000ms": 1.0, "regressions": 0, "eligible": true, "failed_gates": [], "run_ids": [ "fb2_429e9a_016", "fb2_429e9a_017", "fb2_429e9a_030", "fb2_429e9a_038", "fb2_429e9a_046" ] }, { "scenario": "scifact-small", "backend": "weaviate", "operation": "delete", "trials": 5, "expected_trials": 5, "failed_runs": 0, "events": 500, "success": 500, "success_rate": 1.0, "first_p95": 24.4229735690169, "first_ci": [ 20.77167949173599, 25.018210418056693 ], "confirmed_p95": 88.75638328609057, "confirmed_ci": [ 82.89536254596896, 91.46205619326791 ], "query_p95": 14.374172559473664, "query_ci": [ 13.32284351810813, 15.721488045528533 ], "ann_recall": 1.0, "ndcg": 0.6567157762068453, "idle_query_p95": 12.13232510490343, "exact_ndcg": 0.6567157762068453, "within_100ms": 0.996, "within_1000ms": 1.0, "regressions": 0, "eligible": true, "failed_gates": [], "run_ids": [ "fb2_429e9a_016", "fb2_429e9a_017", "fb2_429e9a_030", "fb2_429e9a_038", "fb2_429e9a_046" ] } ], "runs": [ { "run": "fb2_429e9a_000", "trial": 4, "backend": "weaviate", "scenario": "nq-100k", "status": "complete", "eligible": true, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "summary": { "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "wall_s": 30.00902225007303, "operations": { "insert": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 7.88487505633384, "p50": 18.87104200432077, "p95": 34.05072229797952, "p99": 35.269569309893996, "max": 35.29295907355845 }, "confirmed_ms": { "min": 67.08837498445064, "p50": 88.62952055642381, "p95": 106.31492319516838, "p99": 108.75600087339991, "max": 110.9506250359118 }, "ack_ms": { "min": 4.206290934234858, "p50": 11.539437458850443, "p95": 17.08471248857677, "p99": 18.211111709242694, "max": 19.53808404505253 }, "search_minus_ack_ms": { "min": 0.1116669736802578, "p50": 5.644917022436857, "p95": 19.253752409713343, "p99": 20.004585674032572, "max": 20.42549999896437 }, "regressions": 0, "confirmed_within_100ms": 0.75, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "update": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 5.476334015838802, "p50": 18.295458517968655, "p95": 32.41296010091901, "p99": 34.576516243396334, "max": 35.881666000932455 }, "confirmed_ms": { "min": 62.65733402688056, "p50": 84.20881256461143, "p95": 103.17273990367539, "p99": 105.1829495362472, "max": 107.72720794193447 }, "ack_ms": { "min": 4.071375005878508, "p50": 10.79650002066046, "p95": 14.597099978709593, "p99": 16.792410014895722, "max": 17.12999993469566 }, "search_minus_ack_ms": { "min": -0.28029107488691807, "p50": 4.17952099815011, "p95": 19.209108385257423, "p99": 20.65382783650422, "max": 23.39641598518938 }, "regressions": 0, "confirmed_within_100ms": 0.89, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "delete": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 6.684166961349547, "p50": 16.567416489124298, "p95": 21.208108228165653, "p99": 32.49419363215567, "max": 35.66483303438872 }, "confirmed_ms": { "min": 66.00666709709913, "p50": 82.49662449816242, "p95": 86.34342438890599, "p99": 103.76788299297915, "max": 106.92845797166228 }, "ack_ms": { "min": 2.151833032257855, "p50": 5.36060455488041, "p95": 9.620868711499492, "p99": 13.444468787638472, "max": 13.853915967047215 }, "search_minus_ack_ms": { "min": -0.7155000930652022, "p50": 6.173270987346768, "p95": 12.412727653281761, "p99": 19.426182801835253, "max": 22.22450007684529 }, "regressions": 0, "confirmed_within_100ms": 0.98, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 } }, "total_operations": 300, "statuses": { "success": 300, "timeout": 0, "error": 0, "invalid_precondition": 0, "client_overload": 0 }, "query": { "count": 300, "success": 300, "ann_recall_at_10": 0.99, "ndcg_at_10": 0.7931268675492565, "latency_ms": { "min": 4.106208099983633, "p50": 11.387582984752953, "p95": 15.656735398806632, "p99": 19.875673433998596, "max": 22.35387498512864 }, "dispatch_delay_ms": { "min": 0.09670911822468042, "p50": 1.4645210467278957, "p95": 2.2543631959706545, "p99": 2.6902425428852426, "max": 3.036209032870829 } }, "diagnostics": { "schedule_delay_ms": { "min": 0.005167024209384863, "p50": 0.31637505162507296, "p95": 0.6020295841146606, "p99": 0.6470882594571491, "max": 1.8107090611003684 }, "probe_dispatch_ms": { "min": 0.036334036849439144, "p50": 0.4104588879272342, "p95": 2.087733126245439, "p99": 2.620524452067913, "max": 3.508959081955254 }, "probe_service_ms": { "min": 0.839458080008626, "p50": 5.155291990377009, "p95": 11.926528601907192, "p99": 15.190272883046426, "max": 17.459583934396505 }, "write_service_ms": { "min": 1.9018331076949835, "p50": 9.107520978432149, "p95": 15.296018839580942, "p99": 16.711552640190344, "max": 18.74504203442484 } }, "actual_submission_rate": 9.999996683285753, "metadata": { "backend": "weaviate", "collection": "fb2_429e9a_000", "dataset": { "dataset": "nq", "source": "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/nq.zip", "source_sha256": "2553bf7bfbab47b1436ca00a34bce57320e18e611fd00999a2a3a1b4714be752", "source_md5": "d4d3d2e48787a744b6f6e691ff534307", "records": 124000, "queries": 100, "relevant_documents_retained": 120, "seed": 20261028, "license": "CC-BY-SA-3.0", "license_source": "https://raw.githubusercontent.com/google-research-datasets/natural-questions/master/nq_open/README.md", "attribution": "BEIR and original dataset authors; Wikipedia contributors for NQ, SciFact authors and cited article authors for SciFact.", "transformation": "200-word title+text prefix; seeded reservoir of documents with relevant documents for selected test queries retained at front. Simulated mutations; no historical replay.", "model": "sentence-transformers/all-MiniLM-L6-v2", "model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "dimensions": 384, "normalized": true, "max_tokens": 256, "device": "mps", "files": { "corpus.jsonl": "9dc487d785e6ee493eccaf764daa3bf8720a8f4fc8c406246065e1ad33a91f5d", "queries.json": "5603f52876ada5d36118b65dea490f1d43900bc159e1804e637796eb9115e999", "vectors.npy": "cb7421eb6ca5964231403f0a922843088f6fd0e01cbf67f2a1076cc7101005db", "query_vectors.npy": "5e666f8d966d8db6b0fd93d4963234d2dffa0ee3910b8fad069a8ee201e8b5d4", "UPSTREAM-LICENSE.md": "8b675521da3fa142ac6bda38c9bcfcc8cff0e2d6128be53a062ed55fb53f7b87" } }, "base_count": 100000, "profile": { "dataset": "nq", "base_count": 100000, "rate": 10, "query_rate": 10, "arrival": "steady" }, "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "preloaded_mutable": 200, "started_utc": "2026-10-28T18:18:34Z", "setup_s": 37.42437158396933, "backend_before": { "version": "1.40.3", "schema": { "class": "Ffb2429e9a000", "invertedIndexConfig": { "bm25": { "b": 0.75, "k1": 1.2 }, "cleanupIntervalSeconds": 60, "stopwords": { "additions": null, "preset": "en", "removals": null }, "usingBlockMaxWAND": true }, "multiTenancyConfig": { "autoTenantActivation": false, "autoTenantCreation": false, "enabled": false }, "properties": [ { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "doc_id", "tokenization": "field" }, { "dataType": [ "int" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": false, "name": "version" }, { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "cohort", "tokenization": "field" }, { "dataType": [ "text" ], "indexFilterable": false, "indexRangeFilters": false, "indexSearchable": false, "name": "text", "tokenization": "word" } ], "shardingConfig": { "actualCount": 1, "actualVirtualCount": 128, "desiredCount": 1, "desiredVirtualCount": 128, "function": "murmur3", "key": "_id", "strategy": "hash", "virtualPerPhysical": 128 }, "vectorIndexConfig": { "bq": { "enabled": false }, "cleanupIntervalSeconds": 300, "distance": "cosine", "dynamicEfFactor": 8, "dynamicEfMax": 500, "dynamicEfMin": 100, "ef": 128, "efConstruction": 128, "filterStrategy": "acorn", "flatSearchCutoff": 0, "maxConnections": 16, "multivector": { "aggregation": "maxSim", "enabled": false, "muvera": { "dprojections": 16, "enabled": false, "ksim": 4, "repetitions": 10 } }, "pq": { "bitCompression": false, "centroids": 256, "enabled": false, "encoder": { "distribution": "log-normal", "type": "kmeans" }, "segments": 0, "trainingLimit": 100000 }, "rq": { "bits": 8, "centering": false, "enabled": false, "rescoreLimit": 20, "trainingLimit": 10000 }, "skip": false, "skipDefaultQuantization": false, "sq": { "enabled": false, "rescoreLimit": 20, "trainingLimit": 100000 }, "trackDefaultQuantization": false, "vectorCacheMaxObjects": 1000000000000 }, "vectorIndexType": "hnsw", "vectorizer": "none", "replicationConfig": { "deletionStrategy": "TimeBasedResolution", "factor": 1, "asyncEnabled": false } }, "shards": [ { "name": "CepVIOAUth6k", "per_node_status": { "node1": "READY" }, "status": "READY", "vectorQueueSize": 0 } ], "ack": "synchronous indexing, object PUT/POST/DELETE", "transport": "HTTP REST writes + GraphQL search, pooled httpx" }, "count_before": 100200, "idle_duration_s": 10.0, "idle_queries": { "count": 100, "ann_recall": 0.99, "ndcg": 0.7931268675492565, "latency_ms": { "min": 4.246582975611091, "p50": 11.943437450099736, "p95": 14.125371124828234, "p99": 15.323938862420686, "max": 28.402416966855526 } }, "exact_ndcg": 0.7931268675492565, "resources_before": { "usage_usec": 489259893, "user_usec": 475541365, "system_usec": 13718527, "nice_usec": 0, "nr_periods": 9805, "nr_throttled": 742, "throttled_usec": 2044668, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 1119281152, "pids": 11 }, "resources_after": { "usage_usec": 499169613, "user_usec": 485040903, "system_usec": 14128709, "nice_usec": 0, "nr_periods": 10107, "nr_throttled": 742, "throttled_usec": 2044668, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 1014206464, "pids": 11 }, "count_after": 100200, "backend_after": { "version": "1.40.3", "schema": { "class": "Ffb2429e9a000", "invertedIndexConfig": { "bm25": { "b": 0.75, "k1": 1.2 }, "cleanupIntervalSeconds": 60, "stopwords": { "additions": null, "preset": "en", "removals": null }, "usingBlockMaxWAND": true }, "multiTenancyConfig": { "autoTenantActivation": false, "autoTenantCreation": false, "enabled": false }, "properties": [ { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "doc_id", "tokenization": "field" }, { "dataType": [ "int" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": false, "name": "version" }, { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "cohort", "tokenization": "field" }, { "dataType": [ "text" ], "indexFilterable": false, "indexRangeFilters": false, "indexSearchable": false, "name": "text", "tokenization": "word" } ], "shardingConfig": { "actualCount": 1, "actualVirtualCount": 128, "desiredCount": 1, "desiredVirtualCount": 128, "function": "murmur3", "key": "_id", "strategy": "hash", "virtualPerPhysical": 128 }, "vectorIndexConfig": { "bq": { "enabled": false }, "cleanupIntervalSeconds": 300, "distance": "cosine", "dynamicEfFactor": 8, "dynamicEfMax": 500, "dynamicEfMin": 100, "ef": 128, "efConstruction": 128, "filterStrategy": "acorn", "flatSearchCutoff": 0, "maxConnections": 16, "multivector": { "aggregation": "maxSim", "enabled": false, "muvera": { "dprojections": 16, "enabled": false, "ksim": 4, "repetitions": 10 } }, "pq": { "bitCompression": false, "centroids": 256, "enabled": false, "encoder": { "distribution": "log-normal", "type": "kmeans" }, "segments": 0, "trainingLimit": 100000 }, "rq": { "bits": 8, "centering": false, "enabled": false, "rescoreLimit": 20, "trainingLimit": 10000 }, "skip": false, "skipDefaultQuantization": false, "sq": { "enabled": false, "rescoreLimit": 20, "trainingLimit": 100000 }, "trackDefaultQuantization": false, "vectorCacheMaxObjects": 1000000000000 }, "vectorIndexType": "hnsw", "vectorizer": "none", "replicationConfig": { "deletionStrategy": "TimeBasedResolution", "factor": 1, "asyncEnabled": false } }, "shards": [ { "name": "CepVIOAUth6k", "per_node_status": { "node1": "READY" }, "status": "READY", "vectorQueueSize": 0 } ], "ack": "synchronous indexing, object PUT/POST/DELETE", "transport": "HTTP REST writes + GraphQL search, pooled httpx" } }, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "eligible": true, "protocol": "mixed-mutations-v2" }, "examples": [ { "event": 0, "operation": "insert", "doc_id": "mutation-000000", "scheduled_s": 0.0, "submitted_s": 0.00021279207430779934, "schedule_delay_ms": 0.21279207430779934, "precondition": true, "status": "success", "ack_ms": 5.454999976791441, "write_started_ms": 1.3557919301092625, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 14.1731669427827, "stable_since_ms": 14.1731669427827, "confirmed_ms": 67.08837498445064, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 7.298916927538812, "stable_since_ms": 7.298916927538812, "confirmed_ms": 65.44037500862032, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 14.1731669427827, "confirmed_ms": 67.08837498445064, "search_minus_ack_ms": 1.8439169507473707 }, { "event": 1, "operation": "update", "doc_id": "mutation-000001", "scheduled_s": 0.1, "submitted_s": 0.10020487499423325, "schedule_delay_ms": 0.20487499423324507, "precondition": true, "status": "success", "ack_ms": 7.0368340238928795, "write_started_ms": 0.5179590079933405, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 16.580334049649537, "stable_since_ms": 16.580334049649537, "confirmed_ms": 72.3756670486182, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 10.0116670364514, "stable_since_ms": 10.0116670364514, "confirmed_ms": 76.96262502577156, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 16.580334049649537, "confirmed_ms": 76.96262502577156, "search_minus_ack_ms": 2.97483301255852 }, { "event": 2, "operation": "delete", "doc_id": "mutation-000002", "scheduled_s": 0.2, "submitted_s": 0.20057800004724413, "schedule_delay_ms": 0.5780000472441205, "precondition": true, "status": "success", "ack_ms": 5.263749975711107, "write_started_ms": 1.1557090329006314, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 11.558500002138317, "stable_since_ms": 11.558500002138317, "confirmed_ms": 70.79112494830042, "consecutive": 3, "regressions": 0, "requests": 3 }, "search": { "first_ms": 11.380666983313859, "stable_since_ms": 11.380666983313859, "confirmed_ms": 82.85791694652289, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 11.558500002138317, "confirmed_ms": 82.85791694652289, "search_minus_ack_ms": 6.116917007602751 } ] }, { "run": "fb2_429e9a_001", "trial": 1, "backend": "weaviate", "scenario": "nq-10k", "status": "complete", "eligible": true, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "summary": { "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "wall_s": 30.001108875032514, "operations": { "insert": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 8.207708015106618, "p50": 17.302416497841477, "p95": 31.550645863171663, "p99": 38.054978108266376, "max": 39.95450004003942 }, "confirmed_ms": { "min": 66.42654188908637, "p50": 81.08327054651454, "p95": 101.97203818242997, "p99": 104.79345753905365, "max": 108.19237504620105 }, "ack_ms": { "min": 4.252917016856372, "p50": 9.462417045142502, "p95": 13.730108638992531, "p99": 22.49200217775076, "max": 35.83245805930346 }, "search_minus_ack_ms": { "min": -0.2662079641595483, "p50": 2.637249999679625, "p95": 17.12999966111965, "p99": 19.823118508793424, "max": 21.880875108763576 }, "regressions": 0, "confirmed_within_100ms": 0.93, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "update": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 7.968166959472001, "p50": 17.18116650590673, "p95": 29.868601850466803, "p99": 31.42738932045177, "max": 32.30337495915592 }, "confirmed_ms": { "min": 70.20379090681672, "p50": 81.55202103080228, "p95": 98.47702120896429, "p99": 102.40720042958857, "max": 104.13895803503692 }, "ack_ms": { "min": 4.903708933852613, "p50": 9.9588125012815, "p95": 13.404891651589422, "p99": 20.499089170480165, "max": 23.849165998399258 }, "search_minus_ack_ms": { "min": 0.09970890823751688, "p50": 3.619771043304354, "p95": 17.968581209424883, "p99": 18.74931250931695, "max": 18.792625050991774 }, "regressions": 0, "confirmed_within_100ms": 0.97, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "delete": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 6.353207980282605, "p50": 16.229958506301045, "p95": 20.513577066594735, "p99": 25.8477300964296, "max": 28.593000024557114 }, "confirmed_ms": { "min": 70.1048340415582, "p50": 79.17837501736358, "p95": 85.84081283770502, "p99": 92.7069153392222, "max": 98.25074998661876 }, "ack_ms": { "min": 3.2432080479338765, "p50": 5.711666424758732, "p95": 10.250627360073848, "p99": 15.210727002704516, "max": 15.229040989652276 }, "search_minus_ack_ms": { "min": -0.4272080259397626, "p50": 4.892375553026795, "p95": 10.46679965802468, "p99": 18.14853297080845, "max": 18.70866701938212 }, "regressions": 0, "confirmed_within_100ms": 1.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 } }, "total_operations": 300, "statuses": { "success": 300, "timeout": 0, "error": 0, "invalid_precondition": 0, "client_overload": 0 }, "query": { "count": 300, "success": 300, "ann_recall_at_10": 0.996, "ndcg_at_10": 0.9481505328688746, "latency_ms": { "min": 4.184874938800931, "p50": 10.076103964820504, "p95": 14.887731295311827, "p99": 21.01614663610233, "max": 35.09912500157952 }, "dispatch_delay_ms": { "min": 0.12524996418505907, "p50": 1.4489790191873908, "p95": 2.3550476355012506, "p99": 2.981148477410897, "max": 3.754124976694584 } }, "diagnostics": { "schedule_delay_ms": { "min": 0.005374965258653219, "p50": 0.3554170136337298, "p95": 0.6017920386497623, "p99": 0.6472096580549855, "max": 1.0396670084453774 }, "probe_dispatch_ms": { "min": 0.033207936212420464, "p50": 0.42629195377230644, "p95": 2.1973124705255014, "p99": 2.777687390334904, "max": 28.033083071932197 }, "probe_service_ms": { "min": 0.9422500152140856, "p50": 5.598582909442484, "p95": 11.240638070739806, "p99": 13.956366304773828, "max": 35.76591704040766 }, "write_service_ms": { "min": 2.575875027105212, "p50": 8.12210445292294, "p95": 12.67996832029894, "p99": 20.245741829276067, "max": 35.1395410252735 } }, "actual_submission_rate": 10.000043129952484, "metadata": { "backend": "weaviate", "collection": "fb2_429e9a_001", "dataset": { "dataset": "nq", "source": "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/nq.zip", "source_sha256": "2553bf7bfbab47b1436ca00a34bce57320e18e611fd00999a2a3a1b4714be752", "source_md5": "d4d3d2e48787a744b6f6e691ff534307", "records": 124000, "queries": 100, "relevant_documents_retained": 120, "seed": 20261028, "license": "CC-BY-SA-3.0", "license_source": "https://raw.githubusercontent.com/google-research-datasets/natural-questions/master/nq_open/README.md", "attribution": "BEIR and original dataset authors; Wikipedia contributors for NQ, SciFact authors and cited article authors for SciFact.", "transformation": "200-word title+text prefix; seeded reservoir of documents with relevant documents for selected test queries retained at front. Simulated mutations; no historical replay.", "model": "sentence-transformers/all-MiniLM-L6-v2", "model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "dimensions": 384, "normalized": true, "max_tokens": 256, "device": "mps", "files": { "corpus.jsonl": "9dc487d785e6ee493eccaf764daa3bf8720a8f4fc8c406246065e1ad33a91f5d", "queries.json": "5603f52876ada5d36118b65dea490f1d43900bc159e1804e637796eb9115e999", "vectors.npy": "cb7421eb6ca5964231403f0a922843088f6fd0e01cbf67f2a1076cc7101005db", "query_vectors.npy": "5e666f8d966d8db6b0fd93d4963234d2dffa0ee3910b8fad069a8ee201e8b5d4", "UPSTREAM-LICENSE.md": "8b675521da3fa142ac6bda38c9bcfcc8cff0e2d6128be53a062ed55fb53f7b87" } }, "base_count": 10000, "profile": { "dataset": "nq", "base_count": 10000, "rate": 10, "query_rate": 10, "arrival": "steady" }, "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "preloaded_mutable": 200, "started_utc": "2026-10-28T18:19:56Z", "setup_s": 2.5337713750777766, "backend_before": { "version": "1.40.3", "schema": { "class": "Ffb2429e9a001", "invertedIndexConfig": { "bm25": { "b": 0.75, "k1": 1.2 }, "cleanupIntervalSeconds": 60, "stopwords": { "additions": null, "preset": "en", "removals": null }, "usingBlockMaxWAND": true }, "multiTenancyConfig": { "autoTenantActivation": false, "autoTenantCreation": false, "enabled": false }, "properties": [ { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "doc_id", "tokenization": "field" }, { "dataType": [ "int" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": false, "name": "version" }, { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "cohort", "tokenization": "field" }, { "dataType": [ "text" ], "indexFilterable": false, "indexRangeFilters": false, "indexSearchable": false, "name": "text", "tokenization": "word" } ], "shardingConfig": { "actualCount": 1, "actualVirtualCount": 128, "desiredCount": 1, "desiredVirtualCount": 128, "function": "murmur3", "key": "_id", "strategy": "hash", "virtualPerPhysical": 128 }, "vectorIndexConfig": { "bq": { "enabled": false }, "cleanupIntervalSeconds": 300, "distance": "cosine", "dynamicEfFactor": 8, "dynamicEfMax": 500, "dynamicEfMin": 100, "ef": 128, "efConstruction": 128, "filterStrategy": "acorn", "flatSearchCutoff": 0, "maxConnections": 16, "multivector": { "aggregation": "maxSim", "enabled": false, "muvera": { "dprojections": 16, "enabled": false, "ksim": 4, "repetitions": 10 } }, "pq": { "bitCompression": false, "centroids": 256, "enabled": false, "encoder": { "distribution": "log-normal", "type": "kmeans" }, "segments": 0, "trainingLimit": 100000 }, "rq": { "bits": 8, "centering": false, "enabled": false, "rescoreLimit": 20, "trainingLimit": 10000 }, "skip": false, "skipDefaultQuantization": false, "sq": { "enabled": false, "rescoreLimit": 20, "trainingLimit": 100000 }, "trackDefaultQuantization": false, "vectorCacheMaxObjects": 1000000000000 }, "vectorIndexType": "hnsw", "vectorizer": "none", "replicationConfig": { "deletionStrategy": "TimeBasedResolution", "factor": 1, "asyncEnabled": false } }, "shards": [ { "name": "s5fQF41BcNRq", "per_node_status": { "node1": "READY" }, "status": "READY", "vectorQueueSize": 0 } ], "ack": "synchronous indexing, object PUT/POST/DELETE", "transport": "HTTP REST writes + GraphQL search, pooled httpx" }, "count_before": 10200, "idle_duration_s": 10.0, "idle_queries": { "count": 100, "ann_recall": 0.996, "ndcg": 0.9481505328688745, "latency_ms": { "min": 3.691916004754603, "p50": 10.69387496681884, "p95": 12.605095619801432, "p99": 16.328193652443595, "max": 17.890083021484315 } }, "exact_ndcg": 0.9481505328688745, "resources_before": { "usage_usec": 504745904, "user_usec": 490376657, "system_usec": 14369247, "nice_usec": 0, "nr_periods": 10261, "nr_throttled": 744, "throttled_usec": 2046346, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 377606144, "pids": 11 }, "resources_after": { "usage_usec": 513616990, "user_usec": 498885536, "system_usec": 14731453, "nice_usec": 0, "nr_periods": 10562, "nr_throttled": 744, "throttled_usec": 2046346, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 409247744, "pids": 11 }, "count_after": 10200, "backend_after": { "version": "1.40.3", "schema": { "class": "Ffb2429e9a001", "invertedIndexConfig": { "bm25": { "b": 0.75, "k1": 1.2 }, "cleanupIntervalSeconds": 60, "stopwords": { "additions": null, "preset": "en", "removals": null }, "usingBlockMaxWAND": true }, "multiTenancyConfig": { "autoTenantActivation": false, "autoTenantCreation": false, "enabled": false }, "properties": [ { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "doc_id", "tokenization": "field" }, { "dataType": [ "int" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": false, "name": "version" }, { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "cohort", "tokenization": "field" }, { "dataType": [ "text" ], "indexFilterable": false, "indexRangeFilters": false, "indexSearchable": false, "name": "text", "tokenization": "word" } ], "shardingConfig": { "actualCount": 1, "actualVirtualCount": 128, "desiredCount": 1, "desiredVirtualCount": 128, "function": "murmur3", "key": "_id", "strategy": "hash", "virtualPerPhysical": 128 }, "vectorIndexConfig": { "bq": { "enabled": false }, "cleanupIntervalSeconds": 300, "distance": "cosine", "dynamicEfFactor": 8, "dynamicEfMax": 500, "dynamicEfMin": 100, "ef": 128, "efConstruction": 128, "filterStrategy": "acorn", "flatSearchCutoff": 0, "maxConnections": 16, "multivector": { "aggregation": "maxSim", "enabled": false, "muvera": { "dprojections": 16, "enabled": false, "ksim": 4, "repetitions": 10 } }, "pq": { "bitCompression": false, "centroids": 256, "enabled": false, "encoder": { "distribution": "log-normal", "type": "kmeans" }, "segments": 0, "trainingLimit": 100000 }, "rq": { "bits": 8, "centering": false, "enabled": false, "rescoreLimit": 20, "trainingLimit": 10000 }, "skip": false, "skipDefaultQuantization": false, "sq": { "enabled": false, "rescoreLimit": 20, "trainingLimit": 100000 }, "trackDefaultQuantization": false, "vectorCacheMaxObjects": 1000000000000 }, "vectorIndexType": "hnsw", "vectorizer": "none", "replicationConfig": { "deletionStrategy": "TimeBasedResolution", "factor": 1, "asyncEnabled": false } }, "shards": [ { "name": "s5fQF41BcNRq", "per_node_status": { "node1": "READY" }, "status": "READY", "vectorQueueSize": 0 } ], "ack": "synchronous indexing, object PUT/POST/DELETE", "transport": "HTTP REST writes + GraphQL search, pooled httpx" } }, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "eligible": true, "protocol": "mixed-mutations-v2" }, "examples": [ { "event": 0, "operation": "insert", "doc_id": "mutation-000000", "scheduled_s": 0.0, "submitted_s": 0.00018037506379187107, "schedule_delay_ms": 0.18037506379187107, "precondition": true, "status": "success", "ack_ms": 4.368041991256177, "write_started_ms": 1.1834579054266214, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 13.644374907016754, "stable_since_ms": 13.644374907016754, "confirmed_ms": 66.42654188908637, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 4.778791917487979, "stable_since_ms": 4.778791917487979, "confirmed_ms": 63.44099994748831, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 13.644374907016754, "confirmed_ms": 66.42654188908637, "search_minus_ack_ms": 0.4107499262318015 }, { "event": 1, "operation": "update", "doc_id": "mutation-000001", "scheduled_s": 0.1, "submitted_s": 0.10054625000339001, "schedule_delay_ms": 0.5462500033900086, "precondition": true, "status": "success", "ack_ms": 5.540583049878478, "write_started_ms": 0.5944169824942946, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 15.516708022914827, "stable_since_ms": 15.516708022914827, "confirmed_ms": 69.74291696678847, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 7.395499967969954, "stable_since_ms": 7.395499967969954, "confirmed_ms": 71.08083297498524, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 15.516708022914827, "confirmed_ms": 71.08083297498524, "search_minus_ack_ms": 1.854916918091476 }, { "event": 2, "operation": "delete", "doc_id": "mutation-000002", "scheduled_s": 0.2, "submitted_s": 0.20005591702647507, "schedule_delay_ms": 0.055917026475060805, "precondition": true, "status": "success", "ack_ms": 6.6555829253047705, "write_started_ms": 0.8542080176994205, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 11.39875000808388, "stable_since_ms": 11.39875000808388, "confirmed_ms": 68.76195792574435, "consecutive": 3, "regressions": 0, "requests": 3 }, "search": { "first_ms": 11.547875008545816, "stable_since_ms": 11.547875008545816, "confirmed_ms": 83.19595793727785, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 11.547875008545816, "confirmed_ms": 83.19595793727785, "search_minus_ack_ms": 4.8922920832410455 } ] }, { "run": "fb2_429e9a_002", "trial": 3, "backend": "weaviate", "scenario": "nq-100k", "status": "complete", "eligible": true, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "summary": { "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "wall_s": 30.000975457951427, "operations": { "insert": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 8.492833003401756, "p50": 18.345354532357305, "p95": 34.56325485603884, "p99": 36.458609092514976, "max": 38.80412504076958 }, "confirmed_ms": { "min": 66.73158297780901, "p50": 86.24691644217819, "p95": 106.84662737767212, "p99": 114.07608713489029, "max": 118.25508298352361 }, "ack_ms": { "min": 4.68591705430299, "p50": 10.671833995729685, "p95": 15.882441640133035, "p99": 17.564155885484084, "max": 19.992707995697856 }, "search_minus_ack_ms": { "min": 0.1645000884309411, "p50": 4.691979032941163, "p95": 20.77137297601439, "p99": 21.52458702912554, "max": 21.665166947059333 }, "regressions": 0, "confirmed_within_100ms": 0.76, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "update": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 7.265583029948175, "p50": 17.847291484940797, "p95": 33.65717075066641, "p99": 34.97613659943457, "max": 40.29437492135912 }, "confirmed_ms": { "min": 66.13799999468029, "p50": 85.44670848641545, "p95": 106.19459378067404, "p99": 117.89867085870368, "max": 142.517082975246 }, "ack_ms": { "min": 4.446958075277507, "p50": 11.06058299774304, "p95": 16.425391368102282, "p99": 17.142829498043312, "max": 22.319290903396904 }, "search_minus_ack_ms": { "min": -0.28312497306615114, "p50": 4.768437531311065, "p95": 19.334055966464803, "p99": 20.2191683894489, "max": 20.95358306542039 }, "regressions": 0, "confirmed_within_100ms": 0.81, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "delete": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 5.193499964661896, "p50": 16.061728994827718, "p95": 20.212427078513425, "p99": 33.9066441636533, "max": 42.58341598324478 }, "confirmed_ms": { "min": 63.08962497860193, "p50": 81.9257500115782, "p95": 87.68574438290669, "p99": 104.83077087556012, "max": 105.48049991484731 }, "ack_ms": { "min": 2.1357920486479998, "p50": 5.420791974756867, "p95": 10.791461099870501, "p99": 12.469373531639585, "max": 14.185291016474366 }, "search_minus_ack_ms": { "min": 0.09491597302258015, "p50": 5.978583532851189, "p95": 7.783193729119375, "p99": 19.738685647025758, "max": 30.131374951452017 }, "regressions": 0, "confirmed_within_100ms": 0.98, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 } }, "total_operations": 300, "statuses": { "success": 300, "timeout": 0, "error": 0, "invalid_precondition": 0, "client_overload": 0 }, "query": { "count": 300, "success": 300, "ann_recall_at_10": 0.991, "ndcg_at_10": 0.7931268675492565, "latency_ms": { "min": 4.893583012744784, "p50": 11.174104001838714, "p95": 16.254522872623056, "p99": 18.45493007684126, "max": 29.08533299341798 }, "dispatch_delay_ms": { "min": 0.08987495675683022, "p50": 1.3525204849429429, "p95": 2.3159103817306463, "p99": 2.936202251585199, "max": 3.4261250402778387 } }, "diagnostics": { "schedule_delay_ms": { "min": 0.004165992141480501, "p50": 0.3222704748608862, "p95": 0.6103979062748621, "p99": 0.6482221328187698, "max": 0.8322499925270677 }, "probe_dispatch_ms": { "min": 0.019000028260052204, "p50": 0.42645796202123165, "p95": 2.2483128588646646, "p99": 2.8677985491231053, "max": 8.893958874978125 }, "probe_service_ms": { "min": 0.8236659923568368, "p50": 5.386374890804291, "p95": 12.175650289282201, "p99": 14.773336069192734, "max": 67.9537079995498 }, "write_service_ms": { "min": 1.9912919960916042, "p50": 8.93922901013866, "p95": 14.94489797623828, "p99": 16.428508190438148, "max": 21.34849992580712 } }, "actual_submission_rate": 9.999878527218874, "metadata": { "backend": "weaviate", "collection": "fb2_429e9a_002", "dataset": { "dataset": "nq", "source": "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/nq.zip", "source_sha256": "2553bf7bfbab47b1436ca00a34bce57320e18e611fd00999a2a3a1b4714be752", "source_md5": "d4d3d2e48787a744b6f6e691ff534307", "records": 124000, "queries": 100, "relevant_documents_retained": 120, "seed": 20261028, "license": "CC-BY-SA-3.0", "license_source": "https://raw.githubusercontent.com/google-research-datasets/natural-questions/master/nq_open/README.md", "attribution": "BEIR and original dataset authors; Wikipedia contributors for NQ, SciFact authors and cited article authors for SciFact.", "transformation": "200-word title+text prefix; seeded reservoir of documents with relevant documents for selected test queries retained at front. Simulated mutations; no historical replay.", "model": "sentence-transformers/all-MiniLM-L6-v2", "model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "dimensions": 384, "normalized": true, "max_tokens": 256, "device": "mps", "files": { "corpus.jsonl": "9dc487d785e6ee493eccaf764daa3bf8720a8f4fc8c406246065e1ad33a91f5d", "queries.json": "5603f52876ada5d36118b65dea490f1d43900bc159e1804e637796eb9115e999", "vectors.npy": "cb7421eb6ca5964231403f0a922843088f6fd0e01cbf67f2a1076cc7101005db", "query_vectors.npy": "5e666f8d966d8db6b0fd93d4963234d2dffa0ee3910b8fad069a8ee201e8b5d4", "UPSTREAM-LICENSE.md": "8b675521da3fa142ac6bda38c9bcfcc8cff0e2d6128be53a062ed55fb53f7b87" } }, "base_count": 100000, "profile": { "dataset": "nq", "base_count": 100000, "rate": 10, "query_rate": 10, "arrival": "steady" }, "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "preloaded_mutable": 200, "started_utc": "2026-10-28T18:20:43Z", "setup_s": 37.115448208060116, "backend_before": { "version": "1.40.3", "schema": { "class": "Ffb2429e9a002", "invertedIndexConfig": { "bm25": { "b": 0.75, "k1": 1.2 }, "cleanupIntervalSeconds": 60, "stopwords": { "additions": null, "preset": "en", "removals": null }, "usingBlockMaxWAND": true }, "multiTenancyConfig": { "autoTenantActivation": false, "autoTenantCreation": false, "enabled": false }, "properties": [ { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "doc_id", "tokenization": "field" }, { "dataType": [ "int" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": false, "name": "version" }, { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "cohort", "tokenization": "field" }, { "dataType": [ "text" ], "indexFilterable": false, "indexRangeFilters": false, "indexSearchable": false, "name": "text", "tokenization": "word" } ], "shardingConfig": { "actualCount": 1, "actualVirtualCount": 128, "desiredCount": 1, "desiredVirtualCount": 128, "function": "murmur3", "key": "_id", "strategy": "hash", "virtualPerPhysical": 128 }, "vectorIndexConfig": { "bq": { "enabled": false }, "cleanupIntervalSeconds": 300, "distance": "cosine", "dynamicEfFactor": 8, "dynamicEfMax": 500, "dynamicEfMin": 100, "ef": 128, "efConstruction": 128, "filterStrategy": "acorn", "flatSearchCutoff": 0, "maxConnections": 16, "multivector": { "aggregation": "maxSim", "enabled": false, "muvera": { "dprojections": 16, "enabled": false, "ksim": 4, "repetitions": 10 } }, "pq": { "bitCompression": false, "centroids": 256, "enabled": false, "encoder": { "distribution": "log-normal", "type": "kmeans" }, "segments": 0, "trainingLimit": 100000 }, "rq": { "bits": 8, "centering": false, "enabled": false, "rescoreLimit": 20, "trainingLimit": 10000 }, "skip": false, "skipDefaultQuantization": false, "sq": { "enabled": false, "rescoreLimit": 20, "trainingLimit": 100000 }, "trackDefaultQuantization": false, "vectorCacheMaxObjects": 1000000000000 }, "vectorIndexType": "hnsw", "vectorizer": "none", "replicationConfig": { "deletionStrategy": "TimeBasedResolution", "factor": 1, "asyncEnabled": false } }, "shards": [ { "name": "jktJ2NmXUKJj", "per_node_status": { "node1": "READY" }, "status": "READY", "vectorQueueSize": 0 } ], "ack": "synchronous indexing, object PUT/POST/DELETE", "transport": "HTTP REST writes + GraphQL search, pooled httpx" }, "count_before": 100200, "idle_duration_s": 10.0, "idle_queries": { "count": 100, "ann_recall": 0.991, "ndcg": 0.7931268675492565, "latency_ms": { "min": 3.96741705480963, "p50": 11.623291531577706, "p95": 13.359854562440887, "p99": 14.217491300078214, "max": 34.66012503486127 } }, "exact_ndcg": 0.7931268675492565, "resources_before": { "usage_usec": 589150354, "user_usec": 573482035, "system_usec": 15668319, "nice_usec": 0, "nr_periods": 11063, "nr_throttled": 893, "throttled_usec": 2355640, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 1104486400, "pids": 11 }, "resources_after": { "usage_usec": 599042392, "user_usec": 582945824, "system_usec": 16096567, "nice_usec": 0, "nr_periods": 11364, "nr_throttled": 893, "throttled_usec": 2355640, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 1035702272, "pids": 11 }, "count_after": 100200, "backend_after": { "version": "1.40.3", "schema": { "class": "Ffb2429e9a002", "invertedIndexConfig": { "bm25": { "b": 0.75, "k1": 1.2 }, "cleanupIntervalSeconds": 60, "stopwords": { "additions": null, "preset": "en", "removals": null }, "usingBlockMaxWAND": true }, "multiTenancyConfig": { "autoTenantActivation": false, "autoTenantCreation": false, "enabled": false }, "properties": [ { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "doc_id", "tokenization": "field" }, { "dataType": [ "int" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": false, "name": "version" }, { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "cohort", "tokenization": "field" }, { "dataType": [ "text" ], "indexFilterable": false, "indexRangeFilters": false, "indexSearchable": false, "name": "text", "tokenization": "word" } ], "shardingConfig": { "actualCount": 1, "actualVirtualCount": 128, "desiredCount": 1, "desiredVirtualCount": 128, "function": "murmur3", "key": "_id", "strategy": "hash", "virtualPerPhysical": 128 }, "vectorIndexConfig": { "bq": { "enabled": false }, "cleanupIntervalSeconds": 300, "distance": "cosine", "dynamicEfFactor": 8, "dynamicEfMax": 500, "dynamicEfMin": 100, "ef": 128, "efConstruction": 128, "filterStrategy": "acorn", "flatSearchCutoff": 0, "maxConnections": 16, "multivector": { "aggregation": "maxSim", "enabled": false, "muvera": { "dprojections": 16, "enabled": false, "ksim": 4, "repetitions": 10 } }, "pq": { "bitCompression": false, "centroids": 256, "enabled": false, "encoder": { "distribution": "log-normal", "type": "kmeans" }, "segments": 0, "trainingLimit": 100000 }, "rq": { "bits": 8, "centering": false, "enabled": false, "rescoreLimit": 20, "trainingLimit": 10000 }, "skip": false, "skipDefaultQuantization": false, "sq": { "enabled": false, "rescoreLimit": 20, "trainingLimit": 100000 }, "trackDefaultQuantization": false, "vectorCacheMaxObjects": 1000000000000 }, "vectorIndexType": "hnsw", "vectorizer": "none", "replicationConfig": { "deletionStrategy": "TimeBasedResolution", "factor": 1, "asyncEnabled": false } }, "shards": [ { "name": "jktJ2NmXUKJj", "per_node_status": { "node1": "READY" }, "status": "READY", "vectorQueueSize": 0 } ], "ack": "synchronous indexing, object PUT/POST/DELETE", "transport": "HTTP REST writes + GraphQL search, pooled httpx" } }, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "eligible": true, "protocol": "mixed-mutations-v2" }, "examples": [ { "event": 0, "operation": "insert", "doc_id": "mutation-000000", "scheduled_s": 0.0, "submitted_s": 0.0002371249720454216, "schedule_delay_ms": 0.2371249720454216, "precondition": true, "status": "success", "ack_ms": 5.751332966610789, "write_started_ms": 1.5489159850403666, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 16.94812497589737, "stable_since_ms": 16.94812497589737, "confirmed_ms": 71.9735830789432, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 10.447166045196354, "stable_since_ms": 10.447166045196354, "confirmed_ms": 73.62212496809661, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 16.94812497589737, "confirmed_ms": 73.62212496809661, "search_minus_ack_ms": 4.695833078585565 }, { "event": 1, "operation": "update", "doc_id": "mutation-000001", "scheduled_s": 0.1, "submitted_s": 0.10022212495096028, "schedule_delay_ms": 0.22212495096027296, "precondition": true, "status": "success", "ack_ms": 4.775125067681074, "write_started_ms": 1.1647500796243548, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 14.90479102358222, "stable_since_ms": 14.90479102358222, "confirmed_ms": 69.17650008108467, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 6.083458079956472, "stable_since_ms": 6.083458079956472, "confirmed_ms": 66.10662501771003, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 14.90479102358222, "confirmed_ms": 69.17650008108467, "search_minus_ack_ms": 1.3083330122753978 }, { "event": 2, "operation": "delete", "doc_id": "mutation-000002", "scheduled_s": 0.2, "submitted_s": 0.20005025004502386, "schedule_delay_ms": 0.050250045023847445, "precondition": true, "status": "success", "ack_ms": 4.399291006848216, "write_started_ms": 1.338415895588696, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 8.420625003054738, "stable_since_ms": 8.420625003054738, "confirmed_ms": 64.85191592946649, "consecutive": 3, "regressions": 0, "requests": 3 }, "search": { "first_ms": 8.313208003528416, "stable_since_ms": 8.313208003528416, "confirmed_ms": 73.32529092673212, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 8.420625003054738, "confirmed_ms": 73.32529092673212, "search_minus_ack_ms": 3.9139169966802 } ] }, { "run": "fb2_429e9a_003", "trial": 3, "backend": "qdrant", "scenario": "scifact-small", "status": "complete", "eligible": true, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "summary": { "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "wall_s": 30.05059004109353, "operations": { "insert": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 17.854291945695877, "p50": 65.51350007066503, "p95": 104.76627564639784, "p99": 107.3272997350432, "max": 108.14075008966029 }, "confirmed_ms": { "min": 109.38783304300159, "p50": 207.64668745687231, "p95": 211.82654576841742, "p99": 240.05298715201207, "max": 240.1672500418499 }, "ack_ms": { "min": 4.982916987501085, "p50": 9.380104544106871, "p95": 11.502031929558143, "p99": 12.361073660431476, "max": 21.41033299267292 }, "search_minus_ack_ms": { "min": 0.6435830146074295, "p50": 55.62347895465791, "p95": 95.88559834519401, "p99": 97.7360464935191, "max": 98.31395803485066 }, "regressions": 0, "confirmed_within_100ms": 0.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "update": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 19.168916973285377, "p50": 63.3388955029659, "p95": 65.92168157803826, "p99": 67.02312196721323, "max": 67.88908294402063 }, "confirmed_ms": { "min": 108.46579202916473, "p50": 206.18400000967085, "p95": 211.2761639582459, "p99": 211.98235465912148, "max": 213.34158291574568 }, "ack_ms": { "min": 4.913915996439755, "p50": 8.266583550721407, "p95": 10.68010843009688, "p99": 12.461375320563095, "max": 12.65525002963841 }, "search_minus_ack_ms": { "min": 0.8481249678879976, "p50": 54.93447941262275, "p95": 56.82391244918108, "p99": 57.46084766113199, "max": 57.65208404045552 }, "regressions": 0, "confirmed_within_100ms": 0.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "delete": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 15.299083082936704, "p50": 18.788499932270497, "p95": 63.427677791332826, "p99": 65.11727933073436, "max": 102.979457937181 }, "confirmed_ms": { "min": 105.79575004521757, "p50": 110.96664593787864, "p95": 204.8482791869901, "p99": 210.46507315128127, "max": 244.41279203165323 }, "ack_ms": { "min": 4.341125022619963, "p50": 7.110333535820246, "p95": 9.343185723992065, "p99": 10.81863048137166, "max": 11.677208007313311 }, "search_minus_ack_ms": { "min": -2.1587919909507036, "p50": 0.3107499796897173, "p95": 54.48561818921007, "p99": 58.14462426002157, "max": 97.21658297348768 }, "regressions": 0, "confirmed_within_100ms": 0.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 } }, "total_operations": 300, "statuses": { "success": 300, "timeout": 0, "error": 0, "invalid_precondition": 0, "client_overload": 0 }, "query": { "count": 300, "success": 300, "ann_recall_at_10": 0.9970000000000001, "ndcg_at_10": 0.6567157762068453, "latency_ms": { "min": 3.2287919893860817, "p50": 7.187771028839052, "p95": 11.283929855562839, "p99": 48.034340424928814, "max": 50.055791973136365 }, "dispatch_delay_ms": { "min": 0.04950002767145634, "p50": 1.3182915281504393, "p95": 1.9972538284491752, "p99": 2.2482484288048, "max": 2.288290997967124 } }, "diagnostics": { "schedule_delay_ms": { "min": 0.0030410941675995673, "p50": 0.32062452519276974, "p95": 0.6145722989451485, "p99": 0.6379460096362788, "max": 0.6424580235027122 }, "probe_dispatch_ms": { "min": 0.027999980375170708, "p50": 0.37662492832168937, "p95": 1.5825330629013474, "p99": 2.059504747157912, "max": 2.6174590457230806 }, "probe_service_ms": { "min": 1.1634159600362182, "p50": 5.708187993150204, "p95": 46.75015605753288, "p99": 47.986247275257476, "max": 55.62370806001127 }, "write_service_ms": { "min": 3.9517920231446624, "p50": 7.663687516469508, "p95": 10.367122589377686, "p99": 11.44238881766796, "max": 21.05487498920411 } }, "actual_submission_rate": 9.999938712723488, "metadata": { "backend": "qdrant", "collection": "fb2_429e9a_003", "dataset": { "dataset": "scifact", "source": "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/scifact.zip", "source_sha256": "536e14446a0ba56ed1398ab1055f39fe852686ecad24a6306c80c490fa8e0165", "source_md5": "5f7d1de60b170fc8027bb7898e2efca1", "records": 5183, "queries": 100, "relevant_documents_retained": 107, "seed": 20261028, "license": "Corpus: ODC-By-1.0; claims/evidence: CC-BY-4.0", "license_source": "https://raw.githubusercontent.com/allenai/scifact/master/LICENSE.md", "attribution": "BEIR and original dataset authors; Wikipedia contributors for NQ, SciFact authors and cited article authors for SciFact.", "transformation": "200-word title+text prefix; seeded reservoir of documents with relevant documents for selected test queries retained at front. Simulated mutations; no historical replay.", "model": "sentence-transformers/all-MiniLM-L6-v2", "model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "dimensions": 384, "normalized": true, "max_tokens": 256, "device": "mps", "files": { "corpus.jsonl": "189d58f273468352c9f73c6333c5120dcfb7ba1695984f073f0ebb6c100ed0cd", "queries.json": "c8b333c73353a8c4aeb7d6b63067c9a241aa5d39a0314f0c315ec63019ee9558", "vectors.npy": "b6b21404977151ce565e7d6d57d9931a4679e1ecf4aa44fcd9b29eb54955d09e", "query_vectors.npy": "02324eb7c3b3d2e2f0ec3208453e2ec0efd5d61d9fc845b1f8b7950e913f2730", "UPSTREAM-LICENSE.md": "56ee69bc639cbab8f0b2e4b67091ec5e9390aa99a1d65f0a35e4274686c9a806" } }, "base_count": 4000, "profile": { "dataset": "scifact", "base_count": 4000, "rate": 10, "query_rate": 10, "arrival": "steady" }, "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "preloaded_mutable": 200, "started_utc": "2026-10-28T18:22:05Z", "setup_s": 1.6772901670774445, "backend_before": { "version": "1.20.0", "collection": { "status": "green", "optimizer_status": "ok", "indexed_vectors_count": 4200, "points_count": 4200, "segments_count": 2, "config": { "params": { "vectors": { "size": 384, "distance": "Cosine" }, "shard_number": 1, "replication_factor": 1, "write_consistency_factor": 1, "on_disk_payload": true }, "hnsw_config": { "m": 16, "ef_construct": 128, "full_scan_threshold": 10, "max_indexing_threads": 0, "on_disk": false }, "optimizer_config": { "deleted_threshold": 0.2, "vacuum_min_vector_number": 1000, "default_segment_number": 0, "max_segment_size": null, "memmap_threshold": null, "indexing_threshold": 1024, "flush_interval_sec": 5, "max_optimization_threads": null, "prevent_unoptimized": null }, "wal_config": { "wal_capacity_mb": 32, "wal_segments_ahead": 0, "wal_retain_closed": 1 }, "quantization_config": null }, "payload_schema": { "cohort": { "data_type": "keyword", "points": 4200 } }, "update_queue": { "length": 0 }, "created_at": "2026-10-28T18:22:05.234Z" }, "ack": "wait=true", "transport": "HTTP REST, pooled httpx" }, "count_before": 4200, "idle_duration_s": 10.0, "idle_queries": { "count": 100, "ann_recall": 0.997, "ndcg": 0.6567157762068453, "latency_ms": { "min": 3.2669579377397895, "p50": 7.062145974487066, "p95": 8.48379306262359, "p99": 20.886226582806692, "max": 47.01158299576491 } }, "exact_ndcg": 0.6567157762068453, "resources_before": { "usage_usec": 274029574, "user_usec": 261490021, "system_usec": 12539553, "nice_usec": 230678000, "nr_periods": 4120, "nr_throttled": 433, "throttled_usec": 1274020, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 331124736, "pids": 31 }, "resources_after": { "usage_usec": 278972015, "user_usec": 265715226, "system_usec": 13256788, "nice_usec": 230678000, "nr_periods": 4421, "nr_throttled": 433, "throttled_usec": 1274020, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 328400896, "pids": 32 }, "count_after": 4200, "backend_after": { "version": "1.20.0", "collection": { "status": "green", "optimizer_status": "ok", "indexed_vectors_count": 4200, "points_count": 4200, "segments_count": 2, "config": { "params": { "vectors": { "size": 384, "distance": "Cosine" }, "shard_number": 1, "replication_factor": 1, "write_consistency_factor": 1, "on_disk_payload": true }, "hnsw_config": { "m": 16, "ef_construct": 128, "full_scan_threshold": 10, "max_indexing_threads": 0, "on_disk": false }, "optimizer_config": { "deleted_threshold": 0.2, "vacuum_min_vector_number": 1000, "default_segment_number": 0, "max_segment_size": null, "memmap_threshold": null, "indexing_threshold": 1024, "flush_interval_sec": 5, "max_optimization_threads": null, "prevent_unoptimized": null }, "wal_config": { "wal_capacity_mb": 32, "wal_segments_ahead": 0, "wal_retain_closed": 1 }, "quantization_config": null }, "payload_schema": { "cohort": { "data_type": "keyword", "points": 4200 } }, "update_queue": { "length": 0 }, "created_at": "2026-10-28T18:22:05.234Z" }, "ack": "wait=true", "transport": "HTTP REST, pooled httpx" } }, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "eligible": true, "protocol": "mixed-mutations-v2" }, "examples": [ { "event": 0, "operation": "insert", "doc_id": "mutation-000000", "scheduled_s": 0.0, "submitted_s": 0.00027658301405608654, "schedule_delay_ms": 0.27658301405608654, "precondition": true, "status": "success", "ack_ms": 8.654874982312322, "write_started_ms": 1.5055000549182296, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 14.597958070226014, "stable_since_ms": 14.597958070226014, "confirmed_ms": 69.25458298064768, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 63.64341697189957, "stable_since_ms": 63.64341697189957, "confirmed_ms": 204.10454203374684, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 63.64341697189957, "confirmed_ms": 204.10454203374684, "search_minus_ack_ms": 54.98854198958725 }, { "event": 1, "operation": "update", "doc_id": "mutation-000001", "scheduled_s": 0.1, "submitted_s": 0.1000629160553217, "schedule_delay_ms": 0.06291605532168787, "precondition": true, "status": "success", "ack_ms": 10.204499936662614, "write_started_ms": 0.8986250031739473, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 18.566875020042062, "stable_since_ms": 18.566875020042062, "confirmed_ms": 75.31824999023229, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 61.186666949652135, "stable_since_ms": 61.186666949652135, "confirmed_ms": 208.15291698090732, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 61.186666949652135, "confirmed_ms": 208.15291698090732, "search_minus_ack_ms": 50.98216701298952 }, { "event": 2, "operation": "delete", "doc_id": "mutation-000002", "scheduled_s": 0.2, "submitted_s": 0.2001042499905452, "schedule_delay_ms": 0.10424999054520212, "precondition": true, "status": "success", "ack_ms": 6.984875071793795, "write_started_ms": 1.1946660233661532, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 18.274250091053545, "stable_since_ms": 18.274250091053545, "confirmed_ms": 74.50075005181134, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 64.7348330821842, "stable_since_ms": 64.7348330821842, "confirmed_ms": 204.81662498787045, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 64.7348330821842, "confirmed_ms": 204.81662498787045, "search_minus_ack_ms": 57.7499580103904 } ] }, { "run": "fb2_429e9a_004", "trial": 4, "backend": "pgvector", "scenario": "nq-100k-burst", "status": "complete", "eligible": true, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "summary": { "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "burst", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "wall_s": 30.00107770797331, "operations": { "insert": { "count": 130, "success": 130, "success_rate": 1.0, "first_observed_ms": { "min": 16.602540970779955, "p50": 27.720896527171135, "p95": 35.7725894311443, "p99": 38.04479265236295, "max": 43.072041939012706 }, "confirmed_ms": { "min": 71.86000002548099, "p50": 88.18041702033952, "p95": 97.59422272909433, "p99": 102.41002966533415, "max": 112.62020899448544 }, "ack_ms": { "min": 5.269124987535179, "p50": 16.99568744516, "p95": 24.650112516246736, "p99": 28.60970610636286, "max": 32.74354198947549 }, "search_minus_ack_ms": { "min": 2.711583045311272, "p50": 11.642812984064221, "p95": 13.484658295055851, "p99": 14.394092137226835, "max": 19.981457968242466 }, "regressions": 0, "confirmed_within_100ms": 0.9692307692307692, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "update": { "count": 130, "success": 130, "success_rate": 1.0, "first_observed_ms": { "min": 11.557167046703398, "p50": 25.761833472643048, "p95": 30.797529354458675, "p99": 33.19587192265316, "max": 35.617167013697326 }, "confirmed_ms": { "min": 65.53437502589077, "p50": 85.82508348627016, "p95": 92.33727288083173, "p99": 96.7302526824642, "max": 97.33866597525775 }, "ack_ms": { "min": 3.924334072507918, "p50": 14.001396019011736, "p95": 21.49006004910916, "p99": 24.97117939637974, "max": 26.195792015641928 }, "search_minus_ack_ms": { "min": 2.9906659619882703, "p50": 11.704187490977347, "p95": 13.131270842859522, "p99": 13.866586447693408, "max": 14.512291993014514 }, "regressions": 0, "confirmed_within_100ms": 1.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "delete": { "count": 130, "success": 130, "success_rate": 1.0, "first_observed_ms": { "min": 3.8351250113919377, "p50": 12.67633301904425, "p95": 30.597235780442134, "p99": 34.955536393681534, "max": 79.67637502588332 }, "confirmed_ms": { "min": 59.90454100538045, "p50": 73.2299794908613, "p95": 93.40618106070906, "p99": 105.19608348491599, "max": 135.66566701047122 }, "ack_ms": { "min": 0.623915926553309, "p50": 1.7928539891727269, "p95": 5.8859312906861305, "p99": 10.457325925817711, "max": 71.95854198653251 }, "search_minus_ack_ms": { "min": 2.1915000397711992, "p50": 8.75658355653286, "p95": 25.413324870169163, "p99": 30.73270321590825, "max": 31.926749972626567 }, "regressions": 0, "confirmed_within_100ms": 0.9769230769230769, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 } }, "total_operations": 390, "statuses": { "success": 390, "timeout": 0, "error": 0, "invalid_precondition": 0, "client_overload": 0 }, "query": { "count": 300, "success": 300, "ann_recall_at_10": 0.9970000000000001, "ndcg_at_10": 0.7931268675492565, "latency_ms": { "min": 3.2520409440621734, "p50": 10.055103979539126, "p95": 15.092180948704483, "p99": 17.144330148585134, "max": 19.68545897398144 }, "dispatch_delay_ms": { "min": 0.09879190474748611, "p50": 0.9899789001792669, "p95": 1.5590519993565979, "p99": 1.788962974678725, "max": 2.4451250210404396 } }, "diagnostics": { "schedule_delay_ms": { "min": 0.003582914359867573, "p50": 0.34322892315774567, "p95": 0.6277644168582341, "p99": 0.6715429676689945, "max": 1.4272499829530716 }, "probe_dispatch_ms": { "min": 0.021583051420748234, "p50": 0.3936874563805759, "p95": 1.2739895028062165, "p99": 1.6126040718518169, "max": 8.492041961289942 }, "probe_service_ms": { "min": 0.18045806791633368, "p50": 1.8745629349723458, "p95": 10.582228802377358, "p99": 15.399865951621877, "max": 21.511874976567924 }, "write_service_ms": { "min": 0.5169999785721302, "p50": 8.48193751880899, "p95": 21.930558024905615, "p99": 26.419405223568926, "max": 71.79958291817456 } }, "actual_submission_rate": 13.009918592894216, "metadata": { "backend": "pgvector", "collection": "fb2_429e9a_004", "dataset": { "dataset": "nq", "source": "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/nq.zip", "source_sha256": "2553bf7bfbab47b1436ca00a34bce57320e18e611fd00999a2a3a1b4714be752", "source_md5": "d4d3d2e48787a744b6f6e691ff534307", "records": 124000, "queries": 100, "relevant_documents_retained": 120, "seed": 20261028, "license": "CC-BY-SA-3.0", "license_source": "https://raw.githubusercontent.com/google-research-datasets/natural-questions/master/nq_open/README.md", "attribution": "BEIR and original dataset authors; Wikipedia contributors for NQ, SciFact authors and cited article authors for SciFact.", "transformation": "200-word title+text prefix; seeded reservoir of documents with relevant documents for selected test queries retained at front. Simulated mutations; no historical replay.", "model": "sentence-transformers/all-MiniLM-L6-v2", "model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "dimensions": 384, "normalized": true, "max_tokens": 256, "device": "mps", "files": { "corpus.jsonl": "9dc487d785e6ee493eccaf764daa3bf8720a8f4fc8c406246065e1ad33a91f5d", "queries.json": "5603f52876ada5d36118b65dea490f1d43900bc159e1804e637796eb9115e999", "vectors.npy": "cb7421eb6ca5964231403f0a922843088f6fd0e01cbf67f2a1076cc7101005db", "query_vectors.npy": "5e666f8d966d8db6b0fd93d4963234d2dffa0ee3910b8fad069a8ee201e8b5d4", "UPSTREAM-LICENSE.md": "8b675521da3fa142ac6bda38c9bcfcc8cff0e2d6128be53a062ed55fb53f7b87" } }, "base_count": 100000, "profile": { "dataset": "nq", "base_count": 100000, "rate": 10, "query_rate": 10, "arrival": "burst" }, "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "burst", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "preloaded_mutable": 260, "started_utc": "2026-10-28T18:22:52Z", "setup_s": 59.57857800007332, "backend_before": { "version": "0.8.7", "postgres": "PostgreSQL 16.15 (Debian 16.15-1.pgdg12+2) on aarch64-unknown-linux-gnu, compiled by gcc (Debian 12.2.0-14+deb12u1) 12.2.0, 64-bit", "ef_search": 128, "ack": "autocommit, READ COMMITTED", "transport": "psycopg, connection per worker", "query_plan": [ { "Plan": { "Node Type": "Limit", "Parallel Aware": false, "Async Capable": false, "Startup Cost": 1674.28, "Total Cost": 1691.42, "Plan Rows": 10, "Plan Width": 18, "Plans": [ { "Node Type": "Index Scan", "Parent Relationship": "Outer", "Parallel Aware": false, "Async Capable": false, "Scan Direction": "Forward", "Index Name": "fb2_429e9a_004_hnsw", "Relation Name": "fb2_429e9a_004", "Alias": "fb2_429e9a_004", "Startup Cost": 1674.28, "Total Cost": 173051.22, "Plan Rows": 100009, "Plan Width": 18, "Order By": "(embedding <=> '[-0.08946895,-0.094362676,0.092023626,-0.0040844884,0.043233823,0.00062519487,0.1075197,-0.035732996,0.013746996,-0.024103122,0.0021706754,-0.04720602,0.04787615,-0.02043222,0.058315646,0.081252135,-0.05375015,-0.030799072,-0.0077205584,0.035735283,-0.04474784,0.01614922,-0.06905586,0.04194401,-0.08832908,0.007873556,0.0044992934,0.021968655,0.010953364,0.07212316,0.023185596,0.001176476,0.04725166,-0.03580423,-0.032344446,0.004752835,-0.056179125,-0.0353235,-0.037251893,-0.005143068,-0.027599342,0.024403147,0.031338032,0.0056410627,0.008425455,-0.0087741045,-0.039870497,-0.011831836,0.042621233,0.048142947,0.01603842,0.06745542,-0.038018145,0.064212486,-0.009636566,0.01607475,-0.0027868534,-0.03761097,-0.011889431,-0.01084577,0.083403684,0.028946918,0.061870236,-0.045382615,0.03812586,0.060426526,-0.017003024,0.02328925,-0.093020774,-0.024072766,0.042720392,-0.022444837,-0.011597358,0.01209231,-0.005953629,0.029691145,-0.011711495,-0.053345133,-0.081757374,-0.049303155,0.032193717,-0.038507868,-0.0060101775,-0.031470332,-0.07362443,0.03575578,0.0245484,0.01985519,-0.15116754,0.050386053,-0.0389634,0.07548389,-0.09264168,-0.012602816,-0.0035539444,-0.046416666,0.015100493,0.100691915,-0.037529,0.04600518,-0.021859422,0.014675989,-0.097722545,-0.053200893,-0.017040461,-0.01629616,-0.012339222,0.06543302,0.07252074,-0.044301156,0.03054835,-0.038570553,0.019310065,-0.005527747,-0.029386865,-0.058248505,0.065443166,0.028506057,0.0878254,0.058387812,0.081721246,-0.0005962058,0.0569388,0.00960514,0.011667701,0.014456492,-0.07218306,-3.0444767e-33,0.06687367,-0.019213954,0.058202963,0.025171444,-0.028259544,-0.04171671,0.03127722,0.04865431,-0.0721459,-0.050143186,-0.029907746,0.06248994,-0.05096076,-0.065192886,0.010210447,-0.077256374,-0.043370593,-0.0901922,-0.066522665,0.026072454,-0.022315364,0.075375885,-0.029968103,-0.0050771856,-0.0046955296,0.0065650316,-0.06791897,0.07368851,0.056069646,-0.021026025,-0.011352883,0.038811427,-0.090847075,-0.056490287,-0.039881036,0.032885592,-0.029586185,0.04753608,0.006341778,-0.0034275919,0.03681623,0.09326265,-0.0532089,0.032396164,0.012605483,0.045778517,-0.0006472902,0.029721813,0.0734947,-0.037117425,-0.09356724,-0.0071634455,-0.032400057,0.010453378,0.06286513,0.006053158,0.060524322,0.091866635,0.063640274,0.055919155,0.053222507,-0.019174587,0.089296214,-0.058922265,0.013745364,-0.022907477,0.028347451,-0.040759467,-0.06820364,-0.061398294,-0.046857864,0.0060627847,0.010442015,-0.07662317,0.031950947,-0.107813306,-0.0016125626,-0.033413693,-0.09989722,-0.03580293,-0.036118146,-0.08276185,0.1280713,0.07596603,0.049225595,-0.026312606,0.0031930343,-0.0012469466,0.000872793,-0.040353082,-0.034516506,-0.036047976,0.004999689,-0.01570079,-0.09783203,1.4186912e-33,-0.0130903,0.04742377,0.02889633,0.09809088,0.043639127,-0.0747531,0.03402747,0.029147558,0.019848268,0.111865185,0.04845458,-0.016222177,0.025501896,0.09813583,-0.002833195,0.0038929228,0.048537854,0.014111116,0.036570966,-0.017512968,-0.079755045,0.043452248,0.10843892,0.061968718,-0.015678233,0.0060376576,0.010718569,-0.03907552,0.040307663,0.04553227,0.041731495,0.083179526,-0.0844024,-0.023447752,0.014485363,-0.00788756,-0.11410207,0.039309476,0.011220049,0.12889338,-0.07970701,0.010658111,-0.06657531,-0.022037188,-0.043459363,-0.030291978,0.05803613,0.06109021,-0.09415837,0.030727232,0.011858702,0.013526454,-0.049580045,-0.052486703,0.013218615,0.0050231596,0.07660412,-0.0046343356,-0.0692584,-0.03353194,0.022288544,0.022836983,0.01361394,-0.036331616,0.008194689,0.09226991,0.08229875,-0.047639843,-0.005033471,0.0066081914,-0.106666416,-0.06385503,0.006385333,0.06318358,0.03146985,-0.060764216,-0.043287244,-0.042385958,-0.07963737,-0.043478705,-0.031840656,0.060027804,0.027829666,0.03728562,-0.07014622,0.0649568,-0.052579004,-0.0034938955,-0.15390792,-0.04426162,0.028217599,0.05593145,0.023251446,0.029115746,-0.040321343,-1.8832651e-08,-0.037536032,0.0005056563,0.013919978,-0.038129315,0.061486673,0.005534046,-0.010768211,-0.16527015,0.061414365,0.019963786,-0.019666849,-0.059065994,0.00083181454,-0.016695915,0.01090875,0.02685501,-0.0008743374,-0.060967337,-0.049258173,0.012731965,-0.03973743,-0.0018210232,-0.03797001,0.00035104176,0.04477184,0.04185603,-0.03215796,-0.09103283,0.0467481,0.017810425,0.037437033,0.054782324,-0.0059442874,0.022557084,0.053393807,-0.03746794,-0.07080435,0.10548834,-0.04215435,0.042547524,0.05064807,0.07560324,0.04072184,-0.0011278372,0.00702018,-0.070807114,0.040035766,0.051417697,0.011316928,-0.028955372,0.04197099,-0.0026867494,-0.056857754,0.0761941,0.030621601,-0.052413527,-0.021492323,0.0957355,0.1013111,-0.082977295,-0.036080915,-0.03327806,0.0029404552,-0.03586191]'::vector)", "Filter": "(cohort = 'static'::text)" } ] } } ] }, "count_before": 100260, "idle_duration_s": 10.0, "idle_queries": { "count": 100, "ann_recall": 0.997, "ndcg": 0.7931268675492565, "latency_ms": { "min": 6.140333949588239, "p50": 17.155895475298166, "p95": 20.95703929080628, "p99": 22.814381247153516, "max": 25.11675003916025 } }, "exact_ndcg": 0.7931268675492565, "resources_before": { "usage_usec": 744227895, "user_usec": 686871395, "system_usec": 57356500, "nice_usec": 0, "nr_periods": 7287, "nr_throttled": 3219, "throttled_usec": 430945132, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 1627922432, "pids": 23 }, "resources_after": { "usage_usec": 755597226, "user_usec": 694581243, "system_usec": 61015982, "nice_usec": 0, "nr_periods": 7588, "nr_throttled": 3219, "throttled_usec": 430945132, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 1647042560, "pids": 30 }, "count_after": 100260, "backend_after": { "version": "0.8.7", "postgres": "PostgreSQL 16.15 (Debian 16.15-1.pgdg12+2) on aarch64-unknown-linux-gnu, compiled by gcc (Debian 12.2.0-14+deb12u1) 12.2.0, 64-bit", "ef_search": 128, "ack": "autocommit, READ COMMITTED", "transport": "psycopg, connection per worker", "query_plan": [ { "Plan": { "Node Type": "Limit", "Parallel Aware": false, "Async Capable": false, "Startup Cost": 1676.86, "Total Cost": 1694.01, "Plan Rows": 10, "Plan Width": 18, "Plans": [ { "Node Type": "Index Scan", "Parent Relationship": "Outer", "Parallel Aware": false, "Async Capable": false, "Scan Direction": "Forward", "Index Name": "fb2_429e9a_004_hnsw", "Relation Name": "fb2_429e9a_004", "Alias": "fb2_429e9a_004", "Startup Cost": 1676.86, "Total Cost": 173393.79, "Plan Rows": 100123, "Plan Width": 18, "Order By": "(embedding <=> '[-0.08946895,-0.094362676,0.092023626,-0.0040844884,0.043233823,0.00062519487,0.1075197,-0.035732996,0.013746996,-0.024103122,0.0021706754,-0.04720602,0.04787615,-0.02043222,0.058315646,0.081252135,-0.05375015,-0.030799072,-0.0077205584,0.035735283,-0.04474784,0.01614922,-0.06905586,0.04194401,-0.08832908,0.007873556,0.0044992934,0.021968655,0.010953364,0.07212316,0.023185596,0.001176476,0.04725166,-0.03580423,-0.032344446,0.004752835,-0.056179125,-0.0353235,-0.037251893,-0.005143068,-0.027599342,0.024403147,0.031338032,0.0056410627,0.008425455,-0.0087741045,-0.039870497,-0.011831836,0.042621233,0.048142947,0.01603842,0.06745542,-0.038018145,0.064212486,-0.009636566,0.01607475,-0.0027868534,-0.03761097,-0.011889431,-0.01084577,0.083403684,0.028946918,0.061870236,-0.045382615,0.03812586,0.060426526,-0.017003024,0.02328925,-0.093020774,-0.024072766,0.042720392,-0.022444837,-0.011597358,0.01209231,-0.005953629,0.029691145,-0.011711495,-0.053345133,-0.081757374,-0.049303155,0.032193717,-0.038507868,-0.0060101775,-0.031470332,-0.07362443,0.03575578,0.0245484,0.01985519,-0.15116754,0.050386053,-0.0389634,0.07548389,-0.09264168,-0.012602816,-0.0035539444,-0.046416666,0.015100493,0.100691915,-0.037529,0.04600518,-0.021859422,0.014675989,-0.097722545,-0.053200893,-0.017040461,-0.01629616,-0.012339222,0.06543302,0.07252074,-0.044301156,0.03054835,-0.038570553,0.019310065,-0.005527747,-0.029386865,-0.058248505,0.065443166,0.028506057,0.0878254,0.058387812,0.081721246,-0.0005962058,0.0569388,0.00960514,0.011667701,0.014456492,-0.07218306,-3.0444767e-33,0.06687367,-0.019213954,0.058202963,0.025171444,-0.028259544,-0.04171671,0.03127722,0.04865431,-0.0721459,-0.050143186,-0.029907746,0.06248994,-0.05096076,-0.065192886,0.010210447,-0.077256374,-0.043370593,-0.0901922,-0.066522665,0.026072454,-0.022315364,0.075375885,-0.029968103,-0.0050771856,-0.0046955296,0.0065650316,-0.06791897,0.07368851,0.056069646,-0.021026025,-0.011352883,0.038811427,-0.090847075,-0.056490287,-0.039881036,0.032885592,-0.029586185,0.04753608,0.006341778,-0.0034275919,0.03681623,0.09326265,-0.0532089,0.032396164,0.012605483,0.045778517,-0.0006472902,0.029721813,0.0734947,-0.037117425,-0.09356724,-0.0071634455,-0.032400057,0.010453378,0.06286513,0.006053158,0.060524322,0.091866635,0.063640274,0.055919155,0.053222507,-0.019174587,0.089296214,-0.058922265,0.013745364,-0.022907477,0.028347451,-0.040759467,-0.06820364,-0.061398294,-0.046857864,0.0060627847,0.010442015,-0.07662317,0.031950947,-0.107813306,-0.0016125626,-0.033413693,-0.09989722,-0.03580293,-0.036118146,-0.08276185,0.1280713,0.07596603,0.049225595,-0.026312606,0.0031930343,-0.0012469466,0.000872793,-0.040353082,-0.034516506,-0.036047976,0.004999689,-0.01570079,-0.09783203,1.4186912e-33,-0.0130903,0.04742377,0.02889633,0.09809088,0.043639127,-0.0747531,0.03402747,0.029147558,0.019848268,0.111865185,0.04845458,-0.016222177,0.025501896,0.09813583,-0.002833195,0.0038929228,0.048537854,0.014111116,0.036570966,-0.017512968,-0.079755045,0.043452248,0.10843892,0.061968718,-0.015678233,0.0060376576,0.010718569,-0.03907552,0.040307663,0.04553227,0.041731495,0.083179526,-0.0844024,-0.023447752,0.014485363,-0.00788756,-0.11410207,0.039309476,0.011220049,0.12889338,-0.07970701,0.010658111,-0.06657531,-0.022037188,-0.043459363,-0.030291978,0.05803613,0.06109021,-0.09415837,0.030727232,0.011858702,0.013526454,-0.049580045,-0.052486703,0.013218615,0.0050231596,0.07660412,-0.0046343356,-0.0692584,-0.03353194,0.022288544,0.022836983,0.01361394,-0.036331616,0.008194689,0.09226991,0.08229875,-0.047639843,-0.005033471,0.0066081914,-0.106666416,-0.06385503,0.006385333,0.06318358,0.03146985,-0.060764216,-0.043287244,-0.042385958,-0.07963737,-0.043478705,-0.031840656,0.060027804,0.027829666,0.03728562,-0.07014622,0.0649568,-0.052579004,-0.0034938955,-0.15390792,-0.04426162,0.028217599,0.05593145,0.023251446,0.029115746,-0.040321343,-1.8832651e-08,-0.037536032,0.0005056563,0.013919978,-0.038129315,0.061486673,0.005534046,-0.010768211,-0.16527015,0.061414365,0.019963786,-0.019666849,-0.059065994,0.00083181454,-0.016695915,0.01090875,0.02685501,-0.0008743374,-0.060967337,-0.049258173,0.012731965,-0.03973743,-0.0018210232,-0.03797001,0.00035104176,0.04477184,0.04185603,-0.03215796,-0.09103283,0.0467481,0.017810425,0.037437033,0.054782324,-0.0059442874,0.022557084,0.053393807,-0.03746794,-0.07080435,0.10548834,-0.04215435,0.042547524,0.05064807,0.07560324,0.04072184,-0.0011278372,0.00702018,-0.070807114,0.040035766,0.051417697,0.011316928,-0.028955372,0.04197099,-0.0026867494,-0.056857754,0.0761941,0.030621601,-0.052413527,-0.021492323,0.0957355,0.1013111,-0.082977295,-0.036080915,-0.03327806,0.0029404552,-0.03586191]'::vector)", "Filter": "(cohort = 'static'::text)" } ] } } ] } }, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "eligible": true, "protocol": "mixed-mutations-v2" }, "examples": [ { "event": 0, "operation": "insert", "doc_id": "mutation-000000", "scheduled_s": 0.0, "submitted_s": 0.00019816693384200335, "schedule_delay_ms": 0.19816693384200335, "precondition": true, "status": "success", "ack_ms": 15.24837501347065, "write_started_ms": 0.6813330110162497, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 19.492291030474007, "stable_since_ms": 19.492291030474007, "confirmed_ms": 70.68800006527454, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 27.35049999319017, "stable_since_ms": 27.35049999319017, "confirmed_ms": 83.59579101670533, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 27.35049999319017, "confirmed_ms": 83.59579101670533, "search_minus_ack_ms": 12.10212497971952 }, { "event": 1, "operation": "update", "doc_id": "mutation-000001", "scheduled_s": 0.025, "submitted_s": 0.02564124995842576, "schedule_delay_ms": 0.6412499584257589, "precondition": true, "status": "success", "ack_ms": 6.044417037628591, "write_started_ms": 0.34666701685637236, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 11.557167046703398, "stable_since_ms": 11.557167046703398, "confirmed_ms": 62.99354205839336, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 9.035082999616861, "stable_since_ms": 9.035082999616861, "confirmed_ms": 65.53437502589077, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 11.557167046703398, "confirmed_ms": 65.53437502589077, "search_minus_ack_ms": 2.9906659619882703 }, { "event": 2, "operation": "delete", "doc_id": "mutation-000002", "scheduled_s": 0.05, "submitted_s": 0.050160666927695274, "schedule_delay_ms": 0.16066692769527158, "precondition": true, "status": "success", "ack_ms": 0.6637499900534749, "write_started_ms": 0.05825003609061241, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 11.032166075892746, "stable_since_ms": 11.032166075892746, "confirmed_ms": 62.21850006841123, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 5.771916010417044, "stable_since_ms": 5.771916010417044, "confirmed_ms": 62.69608298316598, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 11.032166075892746, "confirmed_ms": 62.69608298316598, "search_minus_ack_ms": 5.108166020363569 } ] }, { "run": "fb2_429e9a_005", "trial": 1, "backend": "pgvector", "scenario": "scifact-small", "status": "complete", "eligible": true, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "summary": { "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "wall_s": 30.000954082934186, "operations": { "insert": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 14.594833017326891, "p50": 20.6444165087305, "p95": 25.18198542529717, "p99": 28.54723228956575, "max": 32.34866703860462 }, "confirmed_ms": { "min": 68.83483310230076, "p50": 80.8775415061973, "p95": 86.06329791946337, "p99": 87.12270441232249, "max": 87.419250048697 }, "ack_ms": { "min": 3.619750030338764, "p50": 8.165437495335937, "p95": 11.046298575820401, "p99": 19.254931015893845, "max": 24.82091705314815 }, "search_minus_ack_ms": { "min": 7.527749985456467, "p50": 12.476853909902275, "p95": 15.008454484632239, "p99": 15.939137048553677, "max": 16.422957996837795 }, "regressions": 0, "confirmed_within_100ms": 1.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "update": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 13.935334049165249, "p50": 20.161166437901556, "p95": 23.853521165437993, "p99": 26.18102040025413, "max": 40.73604196310043 }, "confirmed_ms": { "min": 68.13091703224927, "p50": 80.43170854216442, "p95": 85.30728297191672, "p99": 90.30389623017987, "max": 100.92374996747822 }, "ack_ms": { "min": 3.909124992787838, "p50": 7.745854498352855, "p95": 10.958806052803991, "p99": 13.703829522710384, "max": 31.370833981782198 }, "search_minus_ack_ms": { "min": 8.831000071950257, "p50": 11.8085210560821, "p95": 14.934141252888365, "p99": 16.061397847952325, "max": 16.604000004008412 }, "regressions": 0, "confirmed_within_100ms": 0.99, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "delete": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 11.71879202593118, "p50": 17.92891655350104, "p95": 21.294864383526146, "p99": 23.056024026591345, "max": 26.06137504335493 }, "confirmed_ms": { "min": 63.39958298485726, "p50": 77.37904152600095, "p95": 82.22172509995289, "p99": 84.83067673747428, "max": 85.21120809018612 }, "ack_ms": { "min": 1.3331669615581632, "p50": 3.039750561583787, "p95": 4.62031252682209, "p99": 10.831634684000178, "max": 13.604583102278411 }, "search_minus_ack_ms": { "min": 2.103333012200892, "p50": 14.936791500076652, "p95": 17.743541713571176, "p99": 18.179639138979837, "max": 18.354292027652264 }, "regressions": 0, "confirmed_within_100ms": 1.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 } }, "total_operations": 300, "statuses": { "success": 300, "timeout": 0, "error": 0, "invalid_precondition": 0, "client_overload": 0 }, "query": { "count": 300, "success": 300, "ann_recall_at_10": 0.9993333333333334, "ndcg_at_10": 0.6567157762068453, "latency_ms": { "min": 1.8299170769751072, "p50": 4.701916535850614, "p95": 6.378347240388393, "p99": 13.800348166842012, "max": 18.523999955505133 }, "dispatch_delay_ms": { "min": 0.05716690793633461, "p50": 0.7342294557020068, "p95": 1.7580791201908141, "p99": 1.9598724972456691, "max": 2.171416999772191 } }, "diagnostics": { "schedule_delay_ms": { "min": 0.003874953836913164, "p50": 0.28591647278508603, "p95": 0.5982833064626547, "p99": 0.6214574344928891, "max": 0.6355419754981995 }, "probe_dispatch_ms": { "min": 0.014499993994832039, "p50": 0.40749995969235897, "p95": 1.361804758198559, "p99": 1.9512277795001869, "max": 3.0619159806519747 }, "probe_service_ms": { "min": 0.36929198540747166, "p50": 2.517790999263525, "p95": 5.917124997358767, "p99": 7.372764239553362, "max": 26.812458992935717 }, "write_service_ms": { "min": 1.114542013965547, "p50": 6.8205209681764245, "p95": 10.480652406113222, "p99": 13.204048071056555, "max": 31.040709000080824 } }, "actual_submission_rate": 10.00005613176518, "metadata": { "backend": "pgvector", "collection": "fb2_429e9a_005", "dataset": { "dataset": "scifact", "source": "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/scifact.zip", "source_sha256": "536e14446a0ba56ed1398ab1055f39fe852686ecad24a6306c80c490fa8e0165", "source_md5": "5f7d1de60b170fc8027bb7898e2efca1", "records": 5183, "queries": 100, "relevant_documents_retained": 107, "seed": 20261028, "license": "Corpus: ODC-By-1.0; claims/evidence: CC-BY-4.0", "license_source": "https://raw.githubusercontent.com/allenai/scifact/master/LICENSE.md", "attribution": "BEIR and original dataset authors; Wikipedia contributors for NQ, SciFact authors and cited article authors for SciFact.", "transformation": "200-word title+text prefix; seeded reservoir of documents with relevant documents for selected test queries retained at front. Simulated mutations; no historical replay.", "model": "sentence-transformers/all-MiniLM-L6-v2", "model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "dimensions": 384, "normalized": true, "max_tokens": 256, "device": "mps", "files": { "corpus.jsonl": "189d58f273468352c9f73c6333c5120dcfb7ba1695984f073f0ebb6c100ed0cd", "queries.json": "c8b333c73353a8c4aeb7d6b63067c9a241aa5d39a0314f0c315ec63019ee9558", "vectors.npy": "b6b21404977151ce565e7d6d57d9931a4679e1ecf4aa44fcd9b29eb54955d09e", "query_vectors.npy": "02324eb7c3b3d2e2f0ec3208453e2ec0efd5d61d9fc845b1f8b7950e913f2730", "UPSTREAM-LICENSE.md": "56ee69bc639cbab8f0b2e4b67091ec5e9390aa99a1d65f0a35e4274686c9a806" } }, "base_count": 4000, "profile": { "dataset": "scifact", "base_count": 4000, "rate": 10, "query_rate": 10, "arrival": "steady" }, "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "preloaded_mutable": 200, "started_utc": "2026-10-28T18:24:37Z", "setup_s": 1.745013624895364, "backend_before": { "version": "0.8.7", "postgres": "PostgreSQL 16.15 (Debian 16.15-1.pgdg12+2) on aarch64-unknown-linux-gnu, compiled by gcc (Debian 12.2.0-14+deb12u1) 12.2.0, 64-bit", "ef_search": 128, "ack": "autocommit, READ COMMITTED", "transport": "psycopg, connection per worker", "query_plan": [ { "Plan": { "Node Type": "Limit", "Parallel Aware": false, "Async Capable": false, "Startup Cost": 1216.56, "Total Cost": 1229.82, "Plan Rows": 10, "Plan Width": 16, "Plans": [ { "Node Type": "Index Scan", "Parent Relationship": "Outer", "Parallel Aware": false, "Async Capable": false, "Scan Direction": "Forward", "Index Name": "fb2_429e9a_005_hnsw", "Relation Name": "fb2_429e9a_005", "Alias": "fb2_429e9a_005", "Startup Cost": 1216.56, "Total Cost": 6522.0, "Plan Rows": 4000, "Plan Width": 16, "Order By": "(embedding <=> '[-0.025241137,-0.0037172192,-0.046351995,0.046476826,0.01966622,0.015340689,0.01234306,0.07297605,-0.018329926,-0.013840419,-0.07423897,0.0226357,-0.060784698,-0.028650705,0.05173048,-0.03129233,0.07885052,-0.024390804,0.034739245,0.002474597,0.06695868,-0.03807076,0.016879125,-0.007840027,0.026065528,0.014293832,-0.07275669,0.053708147,-0.020156316,0.07862689,-0.122224644,0.023270043,-0.03987285,-0.029884664,0.008407885,0.06157951,-0.045617852,-0.0013952592,-0.1272149,0.012878671,0.03199118,-0.054489113,-0.07601716,0.016042225,-0.02170239,0.0031456149,-0.03146653,0.036349196,0.030850349,0.101148084,-0.04483407,-0.02210456,0.015648471,0.008023243,0.02329231,-0.047311362,-0.00033133395,-0.060376305,-0.031292018,-0.052068412,-0.055494312,0.03995384,-0.05426452,-0.027163653,0.060765423,-0.029587762,0.062952094,-0.032716926,0.03284741,0.03383774,0.025922673,0.09215419,-0.045252554,0.033018086,-0.0063568754,0.003192685,0.036198895,0.071328856,0.061275735,-0.03834579,0.04380842,0.10884386,-0.06443366,0.027753524,0.016000912,0.031735342,-0.048004173,0.04857545,-0.058097336,0.043568186,-0.014455384,0.008579895,0.023373613,0.0023358837,0.070720226,-0.008269884,-0.011370413,-0.016605737,-0.002399004,-0.03909848,0.017561054,0.042421922,0.011341604,0.025210828,0.09824794,0.0038353435,0.015525245,0.082866095,0.0066154706,0.027564062,0.051245574,-0.009089501,0.13282458,0.04902157,-0.013380719,0.042171925,0.033162206,0.048019677,0.029081784,0.116638325,0.00094674755,0.07029037,0.004596228,-0.02894104,-0.0037263758,0.0041222423,-0.04681563,2.8784053e-33,0.09736768,0.02274426,-0.05078067,0.03864741,0.052899208,0.0496459,-0.060387712,-0.08358956,0.07020227,0.042317823,-0.037229892,-0.1324623,0.08240096,0.039485864,-0.09648351,-0.04383094,-0.06106865,0.025340972,0.031598456,0.04274742,-0.0010978275,0.0007333723,0.041061625,0.011878232,-0.12491255,0.06554766,-0.06920273,0.05781889,-0.05118846,0.043497268,-0.029946037,-0.08991292,0.095012516,-0.12172844,-0.048556324,-0.022020115,-0.014565633,-0.045546,-0.05527385,0.06144455,-0.040042106,0.011806558,-0.13014492,-0.024195487,0.008652814,-0.05887302,0.074748434,-0.03390347,-0.026171094,-0.046370335,-0.02871423,0.01613992,0.050530907,-0.072944105,0.0142246755,-0.0029443551,-0.028645331,0.024415584,-0.021900453,-0.035877362,-0.012359783,0.0469807,0.062022846,0.09110963,-0.011605537,-0.02338493,0.0334725,-0.03523439,-0.039391812,0.020923749,0.045133665,-0.0050133714,0.05868334,0.05490907,0.002556749,0.04097332,0.008711031,-0.06200574,0.03028164,-0.11109333,0.0780934,0.012912109,0.07097299,-0.04068633,0.008197527,0.030907633,0.05238045,0.0011536852,-0.01918761,-0.15324447,0.016286323,-0.05679703,-0.04369561,-0.06417541,0.02090238,-3.0734922e-33,-0.005520113,0.007917555,0.051423088,-0.08971186,0.009558501,0.04202714,-0.032392696,0.007947915,-0.033026196,0.057292204,0.049505785,-0.017359108,-0.087639086,-0.047633022,0.012320442,0.036888227,-0.032912884,-0.05555789,-0.030620035,0.040593192,0.056201898,-0.0435655,0.031078862,0.0809475,0.0595156,-0.008109408,-0.10603491,0.0726334,-0.06864153,-0.034877256,0.023989158,-0.04435082,-0.019972347,-0.12077245,0.044683173,0.014618874,-0.020796062,0.114251524,0.01766482,0.0160371,0.012701173,-0.0053563626,0.074527174,-0.021048209,-0.022659509,0.08406541,-0.06423724,-0.071950026,0.05369397,0.01140933,-0.088458955,-0.01212312,0.0024443534,0.12023613,-0.027182862,-0.022696352,0.020270811,0.08858118,-0.10382351,0.086645365,-0.06383489,-0.04042209,0.04494627,0.016389402,0.13325493,0.08580276,0.060353823,0.03661967,0.041859947,0.021768665,0.0073611615,-0.009300088,-0.033082195,-0.040072612,0.06993921,0.02415639,0.012099922,0.034952287,-0.10813522,-0.024580343,0.05526578,-0.043421194,-0.00020226264,0.013380121,-0.013031641,-0.0023380888,0.020561703,-0.026072076,-0.058541298,-0.027999775,-0.024194289,-0.07036205,-0.05425848,-0.07206627,-0.026656583,-2.3484924e-08,-0.037731905,0.015433777,-0.059194244,-0.062783524,-0.047979638,-0.024383545,-0.1138203,0.040749338,-0.038158197,0.008040415,0.024223538,-0.018507883,0.108268395,0.025551548,0.068713784,-0.043465756,0.017785417,0.07973369,-0.0075122416,-0.04605511,-0.047828607,-0.017772881,0.05118608,0.0022625374,0.03168905,0.0143064195,0.017061999,-0.05608364,0.0064040497,0.0072717075,-0.013301113,0.018809363,-0.017954899,0.08202268,-0.06270375,0.029983893,0.0048537604,-0.035181895,-0.06037397,-0.0063187,0.0034529127,0.011674293,0.013009845,0.012574133,0.031151973,-0.087101504,0.030415354,0.036086276,-0.06438181,-0.08931628,0.0122331055,-0.0099123195,0.008941321,-0.024656717,-0.025116868,-0.0044285995,-0.065338664,0.0398255,0.039999187,0.011154338,0.010968719,-0.048901215,0.096052974,-0.08210191]'::vector)", "Filter": "(cohort = 'static'::text)" } ] } } ] }, "count_before": 4200, "idle_duration_s": 10.0, "idle_queries": { "count": 100, "ann_recall": 1.0, "ndcg": 0.6567157762068453, "latency_ms": { "min": 2.477666945196688, "p50": 5.900854070205241, "p95": 7.387130975257604, "p99": 9.282100615091684, "max": 21.31224994082004 } }, "exact_ndcg": 0.6567157762068453, "resources_before": { "usage_usec": 757663087, "user_usec": 696384667, "system_usec": 61278419, "nice_usec": 0, "nr_periods": 7728, "nr_throttled": 3219, "throttled_usec": 430945132, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 1205153792, "pids": 23 }, "resources_after": { "usage_usec": 763229267, "user_usec": 701406219, "system_usec": 61823047, "nice_usec": 0, "nr_periods": 8029, "nr_throttled": 3219, "throttled_usec": 430945132, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 1220390912, "pids": 29 }, "count_after": 4200, "backend_after": { "version": "0.8.7", "postgres": "PostgreSQL 16.15 (Debian 16.15-1.pgdg12+2) on aarch64-unknown-linux-gnu, compiled by gcc (Debian 12.2.0-14+deb12u1) 12.2.0, 64-bit", "ef_search": 128, "ack": "autocommit, READ COMMITTED", "transport": "psycopg, connection per worker", "query_plan": [ { "Plan": { "Node Type": "Limit", "Parallel Aware": false, "Async Capable": false, "Startup Cost": 1240.87, "Total Cost": 1254.34, "Plan Rows": 10, "Plan Width": 16, "Plans": [ { "Node Type": "Index Scan", "Parent Relationship": "Outer", "Parallel Aware": false, "Async Capable": false, "Scan Direction": "Forward", "Index Name": "fb2_429e9a_005_hnsw", "Relation Name": "fb2_429e9a_005", "Alias": "fb2_429e9a_005", "Startup Cost": 1240.87, "Total Cost": 6792.87, "Plan Rows": 4122, "Plan Width": 16, "Order By": "(embedding <=> '[-0.025241137,-0.0037172192,-0.046351995,0.046476826,0.01966622,0.015340689,0.01234306,0.07297605,-0.018329926,-0.013840419,-0.07423897,0.0226357,-0.060784698,-0.028650705,0.05173048,-0.03129233,0.07885052,-0.024390804,0.034739245,0.002474597,0.06695868,-0.03807076,0.016879125,-0.007840027,0.026065528,0.014293832,-0.07275669,0.053708147,-0.020156316,0.07862689,-0.122224644,0.023270043,-0.03987285,-0.029884664,0.008407885,0.06157951,-0.045617852,-0.0013952592,-0.1272149,0.012878671,0.03199118,-0.054489113,-0.07601716,0.016042225,-0.02170239,0.0031456149,-0.03146653,0.036349196,0.030850349,0.101148084,-0.04483407,-0.02210456,0.015648471,0.008023243,0.02329231,-0.047311362,-0.00033133395,-0.060376305,-0.031292018,-0.052068412,-0.055494312,0.03995384,-0.05426452,-0.027163653,0.060765423,-0.029587762,0.062952094,-0.032716926,0.03284741,0.03383774,0.025922673,0.09215419,-0.045252554,0.033018086,-0.0063568754,0.003192685,0.036198895,0.071328856,0.061275735,-0.03834579,0.04380842,0.10884386,-0.06443366,0.027753524,0.016000912,0.031735342,-0.048004173,0.04857545,-0.058097336,0.043568186,-0.014455384,0.008579895,0.023373613,0.0023358837,0.070720226,-0.008269884,-0.011370413,-0.016605737,-0.002399004,-0.03909848,0.017561054,0.042421922,0.011341604,0.025210828,0.09824794,0.0038353435,0.015525245,0.082866095,0.0066154706,0.027564062,0.051245574,-0.009089501,0.13282458,0.04902157,-0.013380719,0.042171925,0.033162206,0.048019677,0.029081784,0.116638325,0.00094674755,0.07029037,0.004596228,-0.02894104,-0.0037263758,0.0041222423,-0.04681563,2.8784053e-33,0.09736768,0.02274426,-0.05078067,0.03864741,0.052899208,0.0496459,-0.060387712,-0.08358956,0.07020227,0.042317823,-0.037229892,-0.1324623,0.08240096,0.039485864,-0.09648351,-0.04383094,-0.06106865,0.025340972,0.031598456,0.04274742,-0.0010978275,0.0007333723,0.041061625,0.011878232,-0.12491255,0.06554766,-0.06920273,0.05781889,-0.05118846,0.043497268,-0.029946037,-0.08991292,0.095012516,-0.12172844,-0.048556324,-0.022020115,-0.014565633,-0.045546,-0.05527385,0.06144455,-0.040042106,0.011806558,-0.13014492,-0.024195487,0.008652814,-0.05887302,0.074748434,-0.03390347,-0.026171094,-0.046370335,-0.02871423,0.01613992,0.050530907,-0.072944105,0.0142246755,-0.0029443551,-0.028645331,0.024415584,-0.021900453,-0.035877362,-0.012359783,0.0469807,0.062022846,0.09110963,-0.011605537,-0.02338493,0.0334725,-0.03523439,-0.039391812,0.020923749,0.045133665,-0.0050133714,0.05868334,0.05490907,0.002556749,0.04097332,0.008711031,-0.06200574,0.03028164,-0.11109333,0.0780934,0.012912109,0.07097299,-0.04068633,0.008197527,0.030907633,0.05238045,0.0011536852,-0.01918761,-0.15324447,0.016286323,-0.05679703,-0.04369561,-0.06417541,0.02090238,-3.0734922e-33,-0.005520113,0.007917555,0.051423088,-0.08971186,0.009558501,0.04202714,-0.032392696,0.007947915,-0.033026196,0.057292204,0.049505785,-0.017359108,-0.087639086,-0.047633022,0.012320442,0.036888227,-0.032912884,-0.05555789,-0.030620035,0.040593192,0.056201898,-0.0435655,0.031078862,0.0809475,0.0595156,-0.008109408,-0.10603491,0.0726334,-0.06864153,-0.034877256,0.023989158,-0.04435082,-0.019972347,-0.12077245,0.044683173,0.014618874,-0.020796062,0.114251524,0.01766482,0.0160371,0.012701173,-0.0053563626,0.074527174,-0.021048209,-0.022659509,0.08406541,-0.06423724,-0.071950026,0.05369397,0.01140933,-0.088458955,-0.01212312,0.0024443534,0.12023613,-0.027182862,-0.022696352,0.020270811,0.08858118,-0.10382351,0.086645365,-0.06383489,-0.04042209,0.04494627,0.016389402,0.13325493,0.08580276,0.060353823,0.03661967,0.041859947,0.021768665,0.0073611615,-0.009300088,-0.033082195,-0.040072612,0.06993921,0.02415639,0.012099922,0.034952287,-0.10813522,-0.024580343,0.05526578,-0.043421194,-0.00020226264,0.013380121,-0.013031641,-0.0023380888,0.020561703,-0.026072076,-0.058541298,-0.027999775,-0.024194289,-0.07036205,-0.05425848,-0.07206627,-0.026656583,-2.3484924e-08,-0.037731905,0.015433777,-0.059194244,-0.062783524,-0.047979638,-0.024383545,-0.1138203,0.040749338,-0.038158197,0.008040415,0.024223538,-0.018507883,0.108268395,0.025551548,0.068713784,-0.043465756,0.017785417,0.07973369,-0.0075122416,-0.04605511,-0.047828607,-0.017772881,0.05118608,0.0022625374,0.03168905,0.0143064195,0.017061999,-0.05608364,0.0064040497,0.0072717075,-0.013301113,0.018809363,-0.017954899,0.08202268,-0.06270375,0.029983893,0.0048537604,-0.035181895,-0.06037397,-0.0063187,0.0034529127,0.011674293,0.013009845,0.012574133,0.031151973,-0.087101504,0.030415354,0.036086276,-0.06438181,-0.08931628,0.0122331055,-0.0099123195,0.008941321,-0.024656717,-0.025116868,-0.0044285995,-0.065338664,0.0398255,0.039999187,0.011154338,0.010968719,-0.048901215,0.096052974,-0.08210191]'::vector)", "Filter": "(cohort = 'static'::text)" } ] } } ] } }, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "eligible": true, "protocol": "mixed-mutations-v2" }, "examples": [ { "event": 0, "operation": "insert", "doc_id": "mutation-000000", "scheduled_s": 0.0, "submitted_s": 0.00027774996124207973, "schedule_delay_ms": 0.27774996124207973, "precondition": true, "status": "success", "ack_ms": 24.82091705314815, "write_started_ms": 1.3206669827923179, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 28.477499959990382, "stable_since_ms": 28.477499959990382, "confirmed_ms": 79.7230830648914, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 32.34866703860462, "stable_since_ms": 32.34866703860462, "confirmed_ms": 86.68554201722145, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 32.34866703860462, "confirmed_ms": 86.68554201722145, "search_minus_ack_ms": 7.527749985456467 }, { "event": 1, "operation": "update", "doc_id": "mutation-000001", "scheduled_s": 0.1, "submitted_s": 0.10003245796542615, "schedule_delay_ms": 0.03245796542614143, "precondition": true, "status": "success", "ack_ms": 7.575292023830116, "write_started_ms": 0.5527919856831431, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 15.732625033706427, "stable_since_ms": 15.732625033706427, "confirmed_ms": 68.84883402381092, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 18.220667028799653, "stable_since_ms": 18.220667028799653, "confirmed_ms": 76.83787506539375, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 18.220667028799653, "confirmed_ms": 76.83787506539375, "search_minus_ack_ms": 10.645375004969537 }, { "event": 2, "operation": "delete", "doc_id": "mutation-000002", "scheduled_s": 0.2, "submitted_s": 0.2004965420346707, "schedule_delay_ms": 0.49654203467069946, "precondition": true, "status": "success", "ack_ms": 3.583707963116467, "write_started_ms": 0.30954089015722275, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 12.834332999773324, "stable_since_ms": 12.834332999773324, "confirmed_ms": 66.36399996932596, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 17.380749923177063, "stable_since_ms": 17.380749923177063, "confirmed_ms": 77.72149995435029, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 17.380749923177063, "confirmed_ms": 77.72149995435029, "search_minus_ack_ms": 13.797041960060596 } ] }, { "run": "fb2_429e9a_006", "trial": 2, "backend": "pgvector", "scenario": "scifact-small", "status": "complete", "eligible": true, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "summary": { "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "wall_s": 30.001358333043754, "operations": { "insert": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 15.531374956481159, "p50": 20.340625022072345, "p95": 22.485160746145993, "p99": 27.292993839364524, "max": 32.419791095890105 }, "confirmed_ms": { "min": 70.54983300622553, "p50": 81.31995797157288, "p95": 85.13979614363052, "p99": 86.87967123230922, "max": 93.74000004027039 }, "ack_ms": { "min": 4.068917012773454, "p50": 8.21025000186637, "p95": 9.528789552859962, "p99": 10.209359205327972, "max": 18.824958009645343 }, "search_minus_ack_ms": { "min": 8.41624999884516, "p50": 12.002729054074734, "p95": 14.199955988442525, "p99": 16.293417583219746, "max": 23.858750006183982 }, "regressions": 0, "confirmed_within_100ms": 1.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "update": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 16.272707958705723, "p50": 19.99241701560095, "p95": 22.332787478808314, "p99": 25.99047338473611, "max": 26.396208093501627 }, "confirmed_ms": { "min": 72.25370896048844, "p50": 80.97664552042261, "p95": 84.26738956477493, "p99": 85.54086913703941, "max": 86.3075410015881 }, "ack_ms": { "min": 4.056834033690393, "p50": 8.076458529103547, "p95": 9.658262453740463, "p99": 11.320334167685361, "max": 11.514291050843894 }, "search_minus_ack_ms": { "min": 9.618417010642588, "p50": 12.13614596053958, "p95": 14.53261844580993, "p99": 15.031085808295765, "max": 16.685458016581833 }, "regressions": 0, "confirmed_within_100ms": 1.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "delete": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 12.257375055924058, "p50": 14.185916515998542, "p95": 21.1818712297827, "p99": 22.737326580099772, "max": 24.89400003105402 }, "confirmed_ms": { "min": 64.45545901078731, "p50": 68.04910401115194, "p95": 82.7590441913344, "p99": 83.10153619735503, "max": 86.45875006914139 }, "ack_ms": { "min": 1.4863340184092522, "p50": 2.8297085082158446, "p95": 4.201605659909546, "p99": 4.820543769747036, "max": 8.042374975048006 }, "search_minus_ack_ms": { "min": 1.229582936502993, "p50": 2.946354041341692, "p95": 17.56098992191255, "p99": 19.034709827974446, "max": 20.486833062022924 }, "regressions": 0, "confirmed_within_100ms": 1.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 } }, "total_operations": 300, "statuses": { "success": 300, "timeout": 0, "error": 0, "invalid_precondition": 0, "client_overload": 0 }, "query": { "count": 300, "success": 300, "ann_recall_at_10": 0.9993333333333334, "ndcg_at_10": 0.6567157762068453, "latency_ms": { "min": 1.7192090163007379, "p50": 4.635146469809115, "p95": 6.216997542651371, "p99": 7.320366336498405, "max": 14.619166031479836 }, "dispatch_delay_ms": { "min": 0.08775002788752317, "p50": 1.1922494741156697, "p95": 2.0358704612590377, "p99": 2.1655392646789546, "max": 2.265791059471667 } }, "diagnostics": { "schedule_delay_ms": { "min": 0.004833005368354293, "p50": 0.4546870128265823, "p95": 0.7244368188548123, "p99": 0.7557071344912812, "max": 0.7741659879680896 }, "probe_dispatch_ms": { "min": 0.03674998879432678, "p50": 0.4724169848486781, "p95": 2.1823046263307333, "p99": 2.4365045502781864, "max": 2.579250023700297 }, "probe_service_ms": { "min": 0.298874918371439, "p50": 2.8757910476997495, "p95": 5.781983211636543, "p99": 6.800846224650741, "max": 15.11100004427135 }, "write_service_ms": { "min": 1.1241249740123749, "p50": 7.285437546670437, "p95": 8.886945125414059, "p99": 9.44626155192963, "max": 17.950374982319772 } }, "actual_submission_rate": 9.999854740231754, "metadata": { "backend": "pgvector", "collection": "fb2_429e9a_006", "dataset": { "dataset": "scifact", "source": "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/scifact.zip", "source_sha256": "536e14446a0ba56ed1398ab1055f39fe852686ecad24a6306c80c490fa8e0165", "source_md5": "5f7d1de60b170fc8027bb7898e2efca1", "records": 5183, "queries": 100, "relevant_documents_retained": 107, "seed": 20261028, "license": "Corpus: ODC-By-1.0; claims/evidence: CC-BY-4.0", "license_source": "https://raw.githubusercontent.com/allenai/scifact/master/LICENSE.md", "attribution": "BEIR and original dataset authors; Wikipedia contributors for NQ, SciFact authors and cited article authors for SciFact.", "transformation": "200-word title+text prefix; seeded reservoir of documents with relevant documents for selected test queries retained at front. Simulated mutations; no historical replay.", "model": "sentence-transformers/all-MiniLM-L6-v2", "model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "dimensions": 384, "normalized": true, "max_tokens": 256, "device": "mps", "files": { "corpus.jsonl": "189d58f273468352c9f73c6333c5120dcfb7ba1695984f073f0ebb6c100ed0cd", "queries.json": "c8b333c73353a8c4aeb7d6b63067c9a241aa5d39a0314f0c315ec63019ee9558", "vectors.npy": "b6b21404977151ce565e7d6d57d9931a4679e1ecf4aa44fcd9b29eb54955d09e", "query_vectors.npy": "02324eb7c3b3d2e2f0ec3208453e2ec0efd5d61d9fc845b1f8b7950e913f2730", "UPSTREAM-LICENSE.md": "56ee69bc639cbab8f0b2e4b67091ec5e9390aa99a1d65f0a35e4274686c9a806" } }, "base_count": 4000, "profile": { "dataset": "scifact", "base_count": 4000, "rate": 10, "query_rate": 10, "arrival": "steady" }, "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "preloaded_mutable": 200, "started_utc": "2026-10-29T02:47:54Z", "setup_s": 1.6031887079589069, "backend_before": { "version": "0.8.7", "postgres": "PostgreSQL 16.15 (Debian 16.15-1.pgdg12+2) on aarch64-unknown-linux-gnu, compiled by gcc (Debian 12.2.0-14+deb12u1) 12.2.0, 64-bit", "ef_search": 128, "ack": "autocommit, READ COMMITTED", "transport": "psycopg, connection per worker", "query_plan": [ { "Plan": { "Node Type": "Limit", "Parallel Aware": false, "Async Capable": false, "Startup Cost": 1216.56, "Total Cost": 1229.85, "Plan Rows": 10, "Plan Width": 16, "Plans": [ { "Node Type": "Index Scan", "Parent Relationship": "Outer", "Parallel Aware": false, "Async Capable": false, "Scan Direction": "Forward", "Index Name": "fb2_429e9a_006_hnsw", "Relation Name": "fb2_429e9a_006", "Alias": "fb2_429e9a_006", "Startup Cost": 1216.56, "Total Cost": 6534.0, "Plan Rows": 4000, "Plan Width": 16, "Order By": "(embedding <=> '[-0.025241137,-0.0037172192,-0.046351995,0.046476826,0.01966622,0.015340689,0.01234306,0.07297605,-0.018329926,-0.013840419,-0.07423897,0.0226357,-0.060784698,-0.028650705,0.05173048,-0.03129233,0.07885052,-0.024390804,0.034739245,0.002474597,0.06695868,-0.03807076,0.016879125,-0.007840027,0.026065528,0.014293832,-0.07275669,0.053708147,-0.020156316,0.07862689,-0.122224644,0.023270043,-0.03987285,-0.029884664,0.008407885,0.06157951,-0.045617852,-0.0013952592,-0.1272149,0.012878671,0.03199118,-0.054489113,-0.07601716,0.016042225,-0.02170239,0.0031456149,-0.03146653,0.036349196,0.030850349,0.101148084,-0.04483407,-0.02210456,0.015648471,0.008023243,0.02329231,-0.047311362,-0.00033133395,-0.060376305,-0.031292018,-0.052068412,-0.055494312,0.03995384,-0.05426452,-0.027163653,0.060765423,-0.029587762,0.062952094,-0.032716926,0.03284741,0.03383774,0.025922673,0.09215419,-0.045252554,0.033018086,-0.0063568754,0.003192685,0.036198895,0.071328856,0.061275735,-0.03834579,0.04380842,0.10884386,-0.06443366,0.027753524,0.016000912,0.031735342,-0.048004173,0.04857545,-0.058097336,0.043568186,-0.014455384,0.008579895,0.023373613,0.0023358837,0.070720226,-0.008269884,-0.011370413,-0.016605737,-0.002399004,-0.03909848,0.017561054,0.042421922,0.011341604,0.025210828,0.09824794,0.0038353435,0.015525245,0.082866095,0.0066154706,0.027564062,0.051245574,-0.009089501,0.13282458,0.04902157,-0.013380719,0.042171925,0.033162206,0.048019677,0.029081784,0.116638325,0.00094674755,0.07029037,0.004596228,-0.02894104,-0.0037263758,0.0041222423,-0.04681563,2.8784053e-33,0.09736768,0.02274426,-0.05078067,0.03864741,0.052899208,0.0496459,-0.060387712,-0.08358956,0.07020227,0.042317823,-0.037229892,-0.1324623,0.08240096,0.039485864,-0.09648351,-0.04383094,-0.06106865,0.025340972,0.031598456,0.04274742,-0.0010978275,0.0007333723,0.041061625,0.011878232,-0.12491255,0.06554766,-0.06920273,0.05781889,-0.05118846,0.043497268,-0.029946037,-0.08991292,0.095012516,-0.12172844,-0.048556324,-0.022020115,-0.014565633,-0.045546,-0.05527385,0.06144455,-0.040042106,0.011806558,-0.13014492,-0.024195487,0.008652814,-0.05887302,0.074748434,-0.03390347,-0.026171094,-0.046370335,-0.02871423,0.01613992,0.050530907,-0.072944105,0.0142246755,-0.0029443551,-0.028645331,0.024415584,-0.021900453,-0.035877362,-0.012359783,0.0469807,0.062022846,0.09110963,-0.011605537,-0.02338493,0.0334725,-0.03523439,-0.039391812,0.020923749,0.045133665,-0.0050133714,0.05868334,0.05490907,0.002556749,0.04097332,0.008711031,-0.06200574,0.03028164,-0.11109333,0.0780934,0.012912109,0.07097299,-0.04068633,0.008197527,0.030907633,0.05238045,0.0011536852,-0.01918761,-0.15324447,0.016286323,-0.05679703,-0.04369561,-0.06417541,0.02090238,-3.0734922e-33,-0.005520113,0.007917555,0.051423088,-0.08971186,0.009558501,0.04202714,-0.032392696,0.007947915,-0.033026196,0.057292204,0.049505785,-0.017359108,-0.087639086,-0.047633022,0.012320442,0.036888227,-0.032912884,-0.05555789,-0.030620035,0.040593192,0.056201898,-0.0435655,0.031078862,0.0809475,0.0595156,-0.008109408,-0.10603491,0.0726334,-0.06864153,-0.034877256,0.023989158,-0.04435082,-0.019972347,-0.12077245,0.044683173,0.014618874,-0.020796062,0.114251524,0.01766482,0.0160371,0.012701173,-0.0053563626,0.074527174,-0.021048209,-0.022659509,0.08406541,-0.06423724,-0.071950026,0.05369397,0.01140933,-0.088458955,-0.01212312,0.0024443534,0.12023613,-0.027182862,-0.022696352,0.020270811,0.08858118,-0.10382351,0.086645365,-0.06383489,-0.04042209,0.04494627,0.016389402,0.13325493,0.08580276,0.060353823,0.03661967,0.041859947,0.021768665,0.0073611615,-0.009300088,-0.033082195,-0.040072612,0.06993921,0.02415639,0.012099922,0.034952287,-0.10813522,-0.024580343,0.05526578,-0.043421194,-0.00020226264,0.013380121,-0.013031641,-0.0023380888,0.020561703,-0.026072076,-0.058541298,-0.027999775,-0.024194289,-0.07036205,-0.05425848,-0.07206627,-0.026656583,-2.3484924e-08,-0.037731905,0.015433777,-0.059194244,-0.062783524,-0.047979638,-0.024383545,-0.1138203,0.040749338,-0.038158197,0.008040415,0.024223538,-0.018507883,0.108268395,0.025551548,0.068713784,-0.043465756,0.017785417,0.07973369,-0.0075122416,-0.04605511,-0.047828607,-0.017772881,0.05118608,0.0022625374,0.03168905,0.0143064195,0.017061999,-0.05608364,0.0064040497,0.0072717075,-0.013301113,0.018809363,-0.017954899,0.08202268,-0.06270375,0.029983893,0.0048537604,-0.035181895,-0.06037397,-0.0063187,0.0034529127,0.011674293,0.013009845,0.012574133,0.031151973,-0.087101504,0.030415354,0.036086276,-0.06438181,-0.08931628,0.0122331055,-0.0099123195,0.008941321,-0.024656717,-0.025116868,-0.0044285995,-0.065338664,0.0398255,0.039999187,0.011154338,0.010968719,-0.048901215,0.096052974,-0.08210191]'::vector)", "Filter": "(cohort = 'static'::text)" } ] } } ] }, "count_before": 4200, "idle_duration_s": 10.0, "idle_queries": { "count": 100, "ann_recall": 1.0, "ndcg": 0.6567157762068453, "latency_ms": { "min": 3.0180829344317317, "p50": 6.0320000047795475, "p95": 7.213316380511969, "p99": 16.97483637253756, "max": 20.766082918271422 } }, "exact_ndcg": 0.6567157762068453, "resources_before": { "usage_usec": 1971521, "user_usec": 1723047, "system_usec": 248474, "nice_usec": 0, "nr_periods": 291, "nr_throttled": 0, "throttled_usec": 0, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 159703040, "pids": 23 }, "resources_after": { "usage_usec": 7046446, "user_usec": 6380057, "system_usec": 666388, "nice_usec": 0, "nr_periods": 592, "nr_throttled": 0, "throttled_usec": 0, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 170598400, "pids": 27 }, "count_after": 4200, "backend_after": { "version": "0.8.7", "postgres": "PostgreSQL 16.15 (Debian 16.15-1.pgdg12+2) on aarch64-unknown-linux-gnu, compiled by gcc (Debian 12.2.0-14+deb12u1) 12.2.0, 64-bit", "ef_search": 128, "ack": "autocommit, READ COMMITTED", "transport": "psycopg, connection per worker", "query_plan": [ { "Plan": { "Node Type": "Limit", "Parallel Aware": false, "Async Capable": false, "Startup Cost": 1240.87, "Total Cost": 1254.37, "Plan Rows": 10, "Plan Width": 16, "Plans": [ { "Node Type": "Index Scan", "Parent Relationship": "Outer", "Parallel Aware": false, "Async Capable": false, "Scan Direction": "Forward", "Index Name": "fb2_429e9a_006_hnsw", "Relation Name": "fb2_429e9a_006", "Alias": "fb2_429e9a_006", "Startup Cost": 1240.87, "Total Cost": 6804.87, "Plan Rows": 4122, "Plan Width": 16, "Order By": "(embedding <=> '[-0.025241137,-0.0037172192,-0.046351995,0.046476826,0.01966622,0.015340689,0.01234306,0.07297605,-0.018329926,-0.013840419,-0.07423897,0.0226357,-0.060784698,-0.028650705,0.05173048,-0.03129233,0.07885052,-0.024390804,0.034739245,0.002474597,0.06695868,-0.03807076,0.016879125,-0.007840027,0.026065528,0.014293832,-0.07275669,0.053708147,-0.020156316,0.07862689,-0.122224644,0.023270043,-0.03987285,-0.029884664,0.008407885,0.06157951,-0.045617852,-0.0013952592,-0.1272149,0.012878671,0.03199118,-0.054489113,-0.07601716,0.016042225,-0.02170239,0.0031456149,-0.03146653,0.036349196,0.030850349,0.101148084,-0.04483407,-0.02210456,0.015648471,0.008023243,0.02329231,-0.047311362,-0.00033133395,-0.060376305,-0.031292018,-0.052068412,-0.055494312,0.03995384,-0.05426452,-0.027163653,0.060765423,-0.029587762,0.062952094,-0.032716926,0.03284741,0.03383774,0.025922673,0.09215419,-0.045252554,0.033018086,-0.0063568754,0.003192685,0.036198895,0.071328856,0.061275735,-0.03834579,0.04380842,0.10884386,-0.06443366,0.027753524,0.016000912,0.031735342,-0.048004173,0.04857545,-0.058097336,0.043568186,-0.014455384,0.008579895,0.023373613,0.0023358837,0.070720226,-0.008269884,-0.011370413,-0.016605737,-0.002399004,-0.03909848,0.017561054,0.042421922,0.011341604,0.025210828,0.09824794,0.0038353435,0.015525245,0.082866095,0.0066154706,0.027564062,0.051245574,-0.009089501,0.13282458,0.04902157,-0.013380719,0.042171925,0.033162206,0.048019677,0.029081784,0.116638325,0.00094674755,0.07029037,0.004596228,-0.02894104,-0.0037263758,0.0041222423,-0.04681563,2.8784053e-33,0.09736768,0.02274426,-0.05078067,0.03864741,0.052899208,0.0496459,-0.060387712,-0.08358956,0.07020227,0.042317823,-0.037229892,-0.1324623,0.08240096,0.039485864,-0.09648351,-0.04383094,-0.06106865,0.025340972,0.031598456,0.04274742,-0.0010978275,0.0007333723,0.041061625,0.011878232,-0.12491255,0.06554766,-0.06920273,0.05781889,-0.05118846,0.043497268,-0.029946037,-0.08991292,0.095012516,-0.12172844,-0.048556324,-0.022020115,-0.014565633,-0.045546,-0.05527385,0.06144455,-0.040042106,0.011806558,-0.13014492,-0.024195487,0.008652814,-0.05887302,0.074748434,-0.03390347,-0.026171094,-0.046370335,-0.02871423,0.01613992,0.050530907,-0.072944105,0.0142246755,-0.0029443551,-0.028645331,0.024415584,-0.021900453,-0.035877362,-0.012359783,0.0469807,0.062022846,0.09110963,-0.011605537,-0.02338493,0.0334725,-0.03523439,-0.039391812,0.020923749,0.045133665,-0.0050133714,0.05868334,0.05490907,0.002556749,0.04097332,0.008711031,-0.06200574,0.03028164,-0.11109333,0.0780934,0.012912109,0.07097299,-0.04068633,0.008197527,0.030907633,0.05238045,0.0011536852,-0.01918761,-0.15324447,0.016286323,-0.05679703,-0.04369561,-0.06417541,0.02090238,-3.0734922e-33,-0.005520113,0.007917555,0.051423088,-0.08971186,0.009558501,0.04202714,-0.032392696,0.007947915,-0.033026196,0.057292204,0.049505785,-0.017359108,-0.087639086,-0.047633022,0.012320442,0.036888227,-0.032912884,-0.05555789,-0.030620035,0.040593192,0.056201898,-0.0435655,0.031078862,0.0809475,0.0595156,-0.008109408,-0.10603491,0.0726334,-0.06864153,-0.034877256,0.023989158,-0.04435082,-0.019972347,-0.12077245,0.044683173,0.014618874,-0.020796062,0.114251524,0.01766482,0.0160371,0.012701173,-0.0053563626,0.074527174,-0.021048209,-0.022659509,0.08406541,-0.06423724,-0.071950026,0.05369397,0.01140933,-0.088458955,-0.01212312,0.0024443534,0.12023613,-0.027182862,-0.022696352,0.020270811,0.08858118,-0.10382351,0.086645365,-0.06383489,-0.04042209,0.04494627,0.016389402,0.13325493,0.08580276,0.060353823,0.03661967,0.041859947,0.021768665,0.0073611615,-0.009300088,-0.033082195,-0.040072612,0.06993921,0.02415639,0.012099922,0.034952287,-0.10813522,-0.024580343,0.05526578,-0.043421194,-0.00020226264,0.013380121,-0.013031641,-0.0023380888,0.020561703,-0.026072076,-0.058541298,-0.027999775,-0.024194289,-0.07036205,-0.05425848,-0.07206627,-0.026656583,-2.3484924e-08,-0.037731905,0.015433777,-0.059194244,-0.062783524,-0.047979638,-0.024383545,-0.1138203,0.040749338,-0.038158197,0.008040415,0.024223538,-0.018507883,0.108268395,0.025551548,0.068713784,-0.043465756,0.017785417,0.07973369,-0.0075122416,-0.04605511,-0.047828607,-0.017772881,0.05118608,0.0022625374,0.03168905,0.0143064195,0.017061999,-0.05608364,0.0064040497,0.0072717075,-0.013301113,0.018809363,-0.017954899,0.08202268,-0.06270375,0.029983893,0.0048537604,-0.035181895,-0.06037397,-0.0063187,0.0034529127,0.011674293,0.013009845,0.012574133,0.031151973,-0.087101504,0.030415354,0.036086276,-0.06438181,-0.08931628,0.0122331055,-0.0099123195,0.008941321,-0.024656717,-0.025116868,-0.0044285995,-0.065338664,0.0398255,0.039999187,0.011154338,0.010968719,-0.048901215,0.096052974,-0.08210191]'::vector)", "Filter": "(cohort = 'static'::text)" } ] } } ] } }, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "eligible": true, "protocol": "mixed-mutations-v2" }, "examples": [ { "event": 0, "operation": "insert", "doc_id": "mutation-000000", "scheduled_s": 0.0, "submitted_s": 0.00023258302826434374, "schedule_delay_ms": 0.23258302826434374, "precondition": true, "status": "success", "ack_ms": 18.824958009645343, "write_started_ms": 0.8745830273255706, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 25.440583005547523, "stable_since_ms": 25.440583005547523, "confirmed_ms": 77.36700004898012, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 27.241208008490503, "stable_since_ms": 27.241208008490503, "confirmed_ms": 81.69029199052602, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 27.241208008490503, "confirmed_ms": 81.69029199052602, "search_minus_ack_ms": 8.41624999884516 }, { "event": 1, "operation": "update", "doc_id": "mutation-000001", "scheduled_s": 0.1, "submitted_s": 0.10069383296649903, "schedule_delay_ms": 0.693832966499025, "precondition": true, "status": "success", "ack_ms": 6.570750032551587, "write_started_ms": 0.5021250108256936, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 13.315875083208084, "stable_since_ms": 13.315875083208084, "confirmed_ms": 65.87104208301753, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 17.836667015217245, "stable_since_ms": 17.836667015217245, "confirmed_ms": 75.76383301056921, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 17.836667015217245, "confirmed_ms": 75.76383301056921, "search_minus_ack_ms": 11.265916982665658 }, { "event": 2, "operation": "delete", "doc_id": "mutation-000002", "scheduled_s": 0.2, "submitted_s": 0.20001579099334776, "schedule_delay_ms": 0.015790993347752913, "precondition": true, "status": "success", "ack_ms": 3.025209065526724, "write_started_ms": 0.322542036883533, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 12.904584058560431, "stable_since_ms": 12.904584058560431, "confirmed_ms": 64.99841704498976, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 18.24750006198883, "stable_since_ms": 18.24750006198883, "confirmed_ms": 76.8546670442447, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 18.24750006198883, "confirmed_ms": 76.8546670442447, "search_minus_ack_ms": 15.222290996462107 } ] }, { "run": "fb2_429e9a_007", "trial": 1, "backend": "pgvector", "scenario": "nq-100k", "status": "complete", "eligible": true, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "summary": { "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "wall_s": 30.001236875075847, "operations": { "insert": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 19.448166014626622, "p50": 30.19420796772465, "p95": 35.72623304789886, "p99": 36.117575496900834, "max": 37.14395791757852 }, "confirmed_ms": { "min": 76.95495791267604, "p50": 91.10949950991198, "p95": 97.7326333464589, "p99": 98.87498793075792, "max": 98.92329096328467 }, "ack_ms": { "min": 7.823584019206464, "p50": 18.787666980642825, "p95": 24.396904540481046, "p99": 26.090821288526072, "max": 29.05549993738532 }, "search_minus_ack_ms": { "min": 7.77570903301239, "p50": 11.453603976406157, "p95": 12.817478878423572, "p99": 13.412334752501927, "max": 14.641791000030935 }, "regressions": 0, "confirmed_within_100ms": 1.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "update": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 18.71041697449982, "p50": 27.666936977766454, "p95": 33.07623571599834, "p99": 34.904768927954144, "max": 35.73170804884285 }, "confirmed_ms": { "min": 76.08237501699477, "p50": 87.50222896924242, "p95": 94.34119205106981, "p99": 97.06071372958831, "max": 97.40362502634525 }, "ack_ms": { "min": 7.138499990105629, "p50": 16.337145993020386, "p95": 21.727368386927992, "p99": 23.03705423022621, "max": 24.185082991607487 }, "search_minus_ack_ms": { "min": 7.677084067836404, "p50": 11.473187478259206, "p95": 12.707337562460452, "p99": 13.021047378424555, "max": 13.190665980800986 }, "regressions": 0, "confirmed_within_100ms": 1.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "delete": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 11.705167009495199, "p50": 20.65629151184112, "p95": 30.97562462207861, "p99": 37.9678531026002, "max": 38.105875020846725 }, "confirmed_ms": { "min": 63.71737504377961, "p50": 80.51031245850027, "p95": 95.1784104516264, "p99": 103.64990767790006, "max": 106.44158395007253 }, "ack_ms": { "min": 1.0814169654622674, "p50": 2.2734375088475645, "p95": 5.035485385451465, "p99": 6.081508393399419, "max": 6.428833003155887 }, "search_minus_ack_ms": { "min": 3.693290986120701, "p50": 15.638249926269054, "p95": 27.21568190609105, "p99": 34.45791751029901, "max": 34.82100006658584 }, "regressions": 0, "confirmed_within_100ms": 0.98, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 } }, "total_operations": 300, "statuses": { "success": 300, "timeout": 0, "error": 0, "invalid_precondition": 0, "client_overload": 0 }, "query": { "count": 300, "success": 300, "ann_recall_at_10": 0.996, "ndcg_at_10": 0.7931268675492565, "latency_ms": { "min": 3.824833082035184, "p50": 11.205165996216238, "p95": 16.133387497393414, "p99": 18.065255408873774, "max": 18.80962506402284 }, "dispatch_delay_ms": { "min": 0.08504209108650684, "p50": 0.8365005487576127, "p95": 1.7243278271052989, "p99": 1.856710313586517, "max": 2.165084006264806 } }, "diagnostics": { "schedule_delay_ms": { "min": 0.005416967905347292, "p50": 0.3940835129458975, "p95": 0.7172797794916086, "p99": 0.7515085970503188, "max": 0.7639999967068434 }, "probe_dispatch_ms": { "min": 0.03145798109471798, "p50": 0.48906245501711965, "p95": 1.387731177965179, "p99": 1.7614902695640922, "max": 2.5122499791905284 }, "probe_service_ms": { "min": 0.3237089840695262, "p50": 1.9050419796258211, "p95": 12.469670485006644, "p99": 16.784815720748153, "max": 20.605500089004636 }, "write_service_ms": { "min": 0.9535420686006546, "p50": 14.5632085041143, "p95": 22.329833306139335, "p99": 24.248267384245988, "max": 28.749958029948175 } }, "actual_submission_rate": 10.000026184669968, "metadata": { "backend": "pgvector", "collection": "fb2_429e9a_007", "dataset": { "dataset": "nq", "source": "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/nq.zip", "source_sha256": "2553bf7bfbab47b1436ca00a34bce57320e18e611fd00999a2a3a1b4714be752", "source_md5": "d4d3d2e48787a744b6f6e691ff534307", "records": 124000, "queries": 100, "relevant_documents_retained": 120, "seed": 20261028, "license": "CC-BY-SA-3.0", "license_source": "https://raw.githubusercontent.com/google-research-datasets/natural-questions/master/nq_open/README.md", "attribution": "BEIR and original dataset authors; Wikipedia contributors for NQ, SciFact authors and cited article authors for SciFact.", "transformation": "200-word title+text prefix; seeded reservoir of documents with relevant documents for selected test queries retained at front. Simulated mutations; no historical replay.", "model": "sentence-transformers/all-MiniLM-L6-v2", "model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "dimensions": 384, "normalized": true, "max_tokens": 256, "device": "mps", "files": { "corpus.jsonl": "9dc487d785e6ee493eccaf764daa3bf8720a8f4fc8c406246065e1ad33a91f5d", "queries.json": "5603f52876ada5d36118b65dea490f1d43900bc159e1804e637796eb9115e999", "vectors.npy": "cb7421eb6ca5964231403f0a922843088f6fd0e01cbf67f2a1076cc7101005db", "query_vectors.npy": "5e666f8d966d8db6b0fd93d4963234d2dffa0ee3910b8fad069a8ee201e8b5d4", "UPSTREAM-LICENSE.md": "8b675521da3fa142ac6bda38c9bcfcc8cff0e2d6128be53a062ed55fb53f7b87" } }, "base_count": 100000, "profile": { "dataset": "nq", "base_count": 100000, "rate": 10, "query_rate": 10, "arrival": "steady" }, "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "preloaded_mutable": 200, "started_utc": "2026-10-29T02:48:40Z", "setup_s": 57.35902483400423, "backend_before": { "version": "0.8.7", "postgres": "PostgreSQL 16.15 (Debian 16.15-1.pgdg12+2) on aarch64-unknown-linux-gnu, compiled by gcc (Debian 12.2.0-14+deb12u1) 12.2.0, 64-bit", "ef_search": 128, "ack": "autocommit, READ COMMITTED", "transport": "psycopg, connection per worker", "query_plan": [ { "Plan": { "Node Type": "Limit", "Parallel Aware": false, "Async Capable": false, "Startup Cost": 1674.2, "Total Cost": 1691.32, "Plan Rows": 10, "Plan Width": 18, "Plans": [ { "Node Type": "Index Scan", "Parent Relationship": "Outer", "Parallel Aware": false, "Async Capable": false, "Scan Direction": "Forward", "Index Name": "fb2_429e9a_007_hnsw", "Relation Name": "fb2_429e9a_007", "Alias": "fb2_429e9a_007", "Startup Cost": 1674.2, "Total Cost": 172897.98, "Plan Rows": 99993, "Plan Width": 18, "Order By": "(embedding <=> '[-0.08946895,-0.094362676,0.092023626,-0.0040844884,0.043233823,0.00062519487,0.1075197,-0.035732996,0.013746996,-0.024103122,0.0021706754,-0.04720602,0.04787615,-0.02043222,0.058315646,0.081252135,-0.05375015,-0.030799072,-0.0077205584,0.035735283,-0.04474784,0.01614922,-0.06905586,0.04194401,-0.08832908,0.007873556,0.0044992934,0.021968655,0.010953364,0.07212316,0.023185596,0.001176476,0.04725166,-0.03580423,-0.032344446,0.004752835,-0.056179125,-0.0353235,-0.037251893,-0.005143068,-0.027599342,0.024403147,0.031338032,0.0056410627,0.008425455,-0.0087741045,-0.039870497,-0.011831836,0.042621233,0.048142947,0.01603842,0.06745542,-0.038018145,0.064212486,-0.009636566,0.01607475,-0.0027868534,-0.03761097,-0.011889431,-0.01084577,0.083403684,0.028946918,0.061870236,-0.045382615,0.03812586,0.060426526,-0.017003024,0.02328925,-0.093020774,-0.024072766,0.042720392,-0.022444837,-0.011597358,0.01209231,-0.005953629,0.029691145,-0.011711495,-0.053345133,-0.081757374,-0.049303155,0.032193717,-0.038507868,-0.0060101775,-0.031470332,-0.07362443,0.03575578,0.0245484,0.01985519,-0.15116754,0.050386053,-0.0389634,0.07548389,-0.09264168,-0.012602816,-0.0035539444,-0.046416666,0.015100493,0.100691915,-0.037529,0.04600518,-0.021859422,0.014675989,-0.097722545,-0.053200893,-0.017040461,-0.01629616,-0.012339222,0.06543302,0.07252074,-0.044301156,0.03054835,-0.038570553,0.019310065,-0.005527747,-0.029386865,-0.058248505,0.065443166,0.028506057,0.0878254,0.058387812,0.081721246,-0.0005962058,0.0569388,0.00960514,0.011667701,0.014456492,-0.07218306,-3.0444767e-33,0.06687367,-0.019213954,0.058202963,0.025171444,-0.028259544,-0.04171671,0.03127722,0.04865431,-0.0721459,-0.050143186,-0.029907746,0.06248994,-0.05096076,-0.065192886,0.010210447,-0.077256374,-0.043370593,-0.0901922,-0.066522665,0.026072454,-0.022315364,0.075375885,-0.029968103,-0.0050771856,-0.0046955296,0.0065650316,-0.06791897,0.07368851,0.056069646,-0.021026025,-0.011352883,0.038811427,-0.090847075,-0.056490287,-0.039881036,0.032885592,-0.029586185,0.04753608,0.006341778,-0.0034275919,0.03681623,0.09326265,-0.0532089,0.032396164,0.012605483,0.045778517,-0.0006472902,0.029721813,0.0734947,-0.037117425,-0.09356724,-0.0071634455,-0.032400057,0.010453378,0.06286513,0.006053158,0.060524322,0.091866635,0.063640274,0.055919155,0.053222507,-0.019174587,0.089296214,-0.058922265,0.013745364,-0.022907477,0.028347451,-0.040759467,-0.06820364,-0.061398294,-0.046857864,0.0060627847,0.010442015,-0.07662317,0.031950947,-0.107813306,-0.0016125626,-0.033413693,-0.09989722,-0.03580293,-0.036118146,-0.08276185,0.1280713,0.07596603,0.049225595,-0.026312606,0.0031930343,-0.0012469466,0.000872793,-0.040353082,-0.034516506,-0.036047976,0.004999689,-0.01570079,-0.09783203,1.4186912e-33,-0.0130903,0.04742377,0.02889633,0.09809088,0.043639127,-0.0747531,0.03402747,0.029147558,0.019848268,0.111865185,0.04845458,-0.016222177,0.025501896,0.09813583,-0.002833195,0.0038929228,0.048537854,0.014111116,0.036570966,-0.017512968,-0.079755045,0.043452248,0.10843892,0.061968718,-0.015678233,0.0060376576,0.010718569,-0.03907552,0.040307663,0.04553227,0.041731495,0.083179526,-0.0844024,-0.023447752,0.014485363,-0.00788756,-0.11410207,0.039309476,0.011220049,0.12889338,-0.07970701,0.010658111,-0.06657531,-0.022037188,-0.043459363,-0.030291978,0.05803613,0.06109021,-0.09415837,0.030727232,0.011858702,0.013526454,-0.049580045,-0.052486703,0.013218615,0.0050231596,0.07660412,-0.0046343356,-0.0692584,-0.03353194,0.022288544,0.022836983,0.01361394,-0.036331616,0.008194689,0.09226991,0.08229875,-0.047639843,-0.005033471,0.0066081914,-0.106666416,-0.06385503,0.006385333,0.06318358,0.03146985,-0.060764216,-0.043287244,-0.042385958,-0.07963737,-0.043478705,-0.031840656,0.060027804,0.027829666,0.03728562,-0.07014622,0.0649568,-0.052579004,-0.0034938955,-0.15390792,-0.04426162,0.028217599,0.05593145,0.023251446,0.029115746,-0.040321343,-1.8832651e-08,-0.037536032,0.0005056563,0.013919978,-0.038129315,0.061486673,0.005534046,-0.010768211,-0.16527015,0.061414365,0.019963786,-0.019666849,-0.059065994,0.00083181454,-0.016695915,0.01090875,0.02685501,-0.0008743374,-0.060967337,-0.049258173,0.012731965,-0.03973743,-0.0018210232,-0.03797001,0.00035104176,0.04477184,0.04185603,-0.03215796,-0.09103283,0.0467481,0.017810425,0.037437033,0.054782324,-0.0059442874,0.022557084,0.053393807,-0.03746794,-0.07080435,0.10548834,-0.04215435,0.042547524,0.05064807,0.07560324,0.04072184,-0.0011278372,0.00702018,-0.070807114,0.040035766,0.051417697,0.011316928,-0.028955372,0.04197099,-0.0026867494,-0.056857754,0.0761941,0.030621601,-0.052413527,-0.021492323,0.0957355,0.1013111,-0.082977295,-0.036080915,-0.03327806,0.0029404552,-0.03586191]'::vector)", "Filter": "(cohort = 'static'::text)" } ] } } ] }, "count_before": 100200, "idle_duration_s": 10.0, "idle_queries": { "count": 100, "ann_recall": 0.996, "ndcg": 0.7931268675492565, "latency_ms": { "min": 3.904833924025297, "p50": 18.11020798049867, "p95": 22.587614512303844, "p99": 23.227023277431726, "max": 23.50158290937543 } }, "exact_ndcg": 0.7931268675492565, "resources_before": { "usage_usec": 97343954, "user_usec": 93594468, "system_usec": 3749486, "nice_usec": 0, "nr_periods": 1295, "nr_throttled": 414, "throttled_usec": 44017280, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 1126670336, "pids": 23 }, "resources_after": { "usage_usec": 108585850, "user_usec": 101105201, "system_usec": 7480649, "nice_usec": 0, "nr_periods": 1596, "nr_throttled": 414, "throttled_usec": 44017280, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 1142185984, "pids": 28 }, "count_after": 100200, "backend_after": { "version": "0.8.7", "postgres": "PostgreSQL 16.15 (Debian 16.15-1.pgdg12+2) on aarch64-unknown-linux-gnu, compiled by gcc (Debian 12.2.0-14+deb12u1) 12.2.0, 64-bit", "ef_search": 128, "ack": "autocommit, READ COMMITTED", "transport": "psycopg, connection per worker", "query_plan": [ { "Plan": { "Node Type": "Limit", "Parallel Aware": false, "Async Capable": false, "Startup Cost": 1676.73, "Total Cost": 1693.87, "Plan Rows": 10, "Plan Width": 18, "Plans": [ { "Node Type": "Index Scan", "Parent Relationship": "Outer", "Parallel Aware": false, "Async Capable": false, "Scan Direction": "Forward", "Index Name": "fb2_429e9a_007_hnsw", "Relation Name": "fb2_429e9a_007", "Alias": "fb2_429e9a_007", "Startup Cost": 1676.73, "Total Cost": 173167.15, "Plan Rows": 100045, "Plan Width": 18, "Order By": "(embedding <=> '[-0.08946895,-0.094362676,0.092023626,-0.0040844884,0.043233823,0.00062519487,0.1075197,-0.035732996,0.013746996,-0.024103122,0.0021706754,-0.04720602,0.04787615,-0.02043222,0.058315646,0.081252135,-0.05375015,-0.030799072,-0.0077205584,0.035735283,-0.04474784,0.01614922,-0.06905586,0.04194401,-0.08832908,0.007873556,0.0044992934,0.021968655,0.010953364,0.07212316,0.023185596,0.001176476,0.04725166,-0.03580423,-0.032344446,0.004752835,-0.056179125,-0.0353235,-0.037251893,-0.005143068,-0.027599342,0.024403147,0.031338032,0.0056410627,0.008425455,-0.0087741045,-0.039870497,-0.011831836,0.042621233,0.048142947,0.01603842,0.06745542,-0.038018145,0.064212486,-0.009636566,0.01607475,-0.0027868534,-0.03761097,-0.011889431,-0.01084577,0.083403684,0.028946918,0.061870236,-0.045382615,0.03812586,0.060426526,-0.017003024,0.02328925,-0.093020774,-0.024072766,0.042720392,-0.022444837,-0.011597358,0.01209231,-0.005953629,0.029691145,-0.011711495,-0.053345133,-0.081757374,-0.049303155,0.032193717,-0.038507868,-0.0060101775,-0.031470332,-0.07362443,0.03575578,0.0245484,0.01985519,-0.15116754,0.050386053,-0.0389634,0.07548389,-0.09264168,-0.012602816,-0.0035539444,-0.046416666,0.015100493,0.100691915,-0.037529,0.04600518,-0.021859422,0.014675989,-0.097722545,-0.053200893,-0.017040461,-0.01629616,-0.012339222,0.06543302,0.07252074,-0.044301156,0.03054835,-0.038570553,0.019310065,-0.005527747,-0.029386865,-0.058248505,0.065443166,0.028506057,0.0878254,0.058387812,0.081721246,-0.0005962058,0.0569388,0.00960514,0.011667701,0.014456492,-0.07218306,-3.0444767e-33,0.06687367,-0.019213954,0.058202963,0.025171444,-0.028259544,-0.04171671,0.03127722,0.04865431,-0.0721459,-0.050143186,-0.029907746,0.06248994,-0.05096076,-0.065192886,0.010210447,-0.077256374,-0.043370593,-0.0901922,-0.066522665,0.026072454,-0.022315364,0.075375885,-0.029968103,-0.0050771856,-0.0046955296,0.0065650316,-0.06791897,0.07368851,0.056069646,-0.021026025,-0.011352883,0.038811427,-0.090847075,-0.056490287,-0.039881036,0.032885592,-0.029586185,0.04753608,0.006341778,-0.0034275919,0.03681623,0.09326265,-0.0532089,0.032396164,0.012605483,0.045778517,-0.0006472902,0.029721813,0.0734947,-0.037117425,-0.09356724,-0.0071634455,-0.032400057,0.010453378,0.06286513,0.006053158,0.060524322,0.091866635,0.063640274,0.055919155,0.053222507,-0.019174587,0.089296214,-0.058922265,0.013745364,-0.022907477,0.028347451,-0.040759467,-0.06820364,-0.061398294,-0.046857864,0.0060627847,0.010442015,-0.07662317,0.031950947,-0.107813306,-0.0016125626,-0.033413693,-0.09989722,-0.03580293,-0.036118146,-0.08276185,0.1280713,0.07596603,0.049225595,-0.026312606,0.0031930343,-0.0012469466,0.000872793,-0.040353082,-0.034516506,-0.036047976,0.004999689,-0.01570079,-0.09783203,1.4186912e-33,-0.0130903,0.04742377,0.02889633,0.09809088,0.043639127,-0.0747531,0.03402747,0.029147558,0.019848268,0.111865185,0.04845458,-0.016222177,0.025501896,0.09813583,-0.002833195,0.0038929228,0.048537854,0.014111116,0.036570966,-0.017512968,-0.079755045,0.043452248,0.10843892,0.061968718,-0.015678233,0.0060376576,0.010718569,-0.03907552,0.040307663,0.04553227,0.041731495,0.083179526,-0.0844024,-0.023447752,0.014485363,-0.00788756,-0.11410207,0.039309476,0.011220049,0.12889338,-0.07970701,0.010658111,-0.06657531,-0.022037188,-0.043459363,-0.030291978,0.05803613,0.06109021,-0.09415837,0.030727232,0.011858702,0.013526454,-0.049580045,-0.052486703,0.013218615,0.0050231596,0.07660412,-0.0046343356,-0.0692584,-0.03353194,0.022288544,0.022836983,0.01361394,-0.036331616,0.008194689,0.09226991,0.08229875,-0.047639843,-0.005033471,0.0066081914,-0.106666416,-0.06385503,0.006385333,0.06318358,0.03146985,-0.060764216,-0.043287244,-0.042385958,-0.07963737,-0.043478705,-0.031840656,0.060027804,0.027829666,0.03728562,-0.07014622,0.0649568,-0.052579004,-0.0034938955,-0.15390792,-0.04426162,0.028217599,0.05593145,0.023251446,0.029115746,-0.040321343,-1.8832651e-08,-0.037536032,0.0005056563,0.013919978,-0.038129315,0.061486673,0.005534046,-0.010768211,-0.16527015,0.061414365,0.019963786,-0.019666849,-0.059065994,0.00083181454,-0.016695915,0.01090875,0.02685501,-0.0008743374,-0.060967337,-0.049258173,0.012731965,-0.03973743,-0.0018210232,-0.03797001,0.00035104176,0.04477184,0.04185603,-0.03215796,-0.09103283,0.0467481,0.017810425,0.037437033,0.054782324,-0.0059442874,0.022557084,0.053393807,-0.03746794,-0.07080435,0.10548834,-0.04215435,0.042547524,0.05064807,0.07560324,0.04072184,-0.0011278372,0.00702018,-0.070807114,0.040035766,0.051417697,0.011316928,-0.028955372,0.04197099,-0.0026867494,-0.056857754,0.0761941,0.030621601,-0.052413527,-0.021492323,0.0957355,0.1013111,-0.082977295,-0.036080915,-0.03327806,0.0029404552,-0.03586191]'::vector)", "Filter": "(cohort = 'static'::text)" } ] } } ] } }, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "eligible": true, "protocol": "mixed-mutations-v2" }, "examples": [ { "event": 0, "operation": "insert", "doc_id": "mutation-000000", "scheduled_s": 0.0, "submitted_s": 0.00022791698575019836, "schedule_delay_ms": 0.22791698575019836, "precondition": true, "status": "success", "ack_ms": 21.24362508766353, "write_started_ms": 1.0857080342248082, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 24.72804207354784, "stable_since_ms": 24.72804207354784, "confirmed_ms": 76.79179206024855, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 32.23037498537451, "stable_since_ms": 32.23037498537451, "confirmed_ms": 87.98204199410975, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 32.23037498537451, "confirmed_ms": 87.98204199410975, "search_minus_ack_ms": 10.986749897710979 }, { "event": 1, "operation": "update", "doc_id": "mutation-000001", "scheduled_s": 0.1, "submitted_s": 0.10033679206389934, "schedule_delay_ms": 0.3367920638993327, "precondition": true, "status": "success", "ack_ms": 9.559792000800371, "write_started_ms": 0.648624962195754, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 13.215582934208214, "stable_since_ms": 13.215582934208214, "confirmed_ms": 65.59354194905609, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 19.78412491735071, "stable_since_ms": 19.78412491735071, "confirmed_ms": 76.80099993012846, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 19.78412491735071, "confirmed_ms": 76.80099993012846, "search_minus_ack_ms": 10.224332916550338 }, { "event": 2, "operation": "delete", "doc_id": "mutation-000002", "scheduled_s": 0.2, "submitted_s": 0.2002550420584157, "schedule_delay_ms": 0.2550420584156998, "precondition": true, "status": "success", "ack_ms": 3.553875023499131, "write_started_ms": 0.8842080133035779, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 13.373291934840381, "stable_since_ms": 13.373291934840381, "confirmed_ms": 67.40008294582367, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 12.98437500372529, "stable_since_ms": 12.98437500372529, "confirmed_ms": 77.86004198715091, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 13.373291934840381, "confirmed_ms": 77.86004198715091, "search_minus_ack_ms": 9.430499980226159 } ] }, { "run": "fb2_429e9a_008", "trial": 1, "backend": "weaviate", "scenario": "nq-100k-burst", "status": "complete", "eligible": true, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "summary": { "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "burst", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "wall_s": 30.001332249958068, "operations": { "insert": { "count": 130, "success": 130, "success_rate": 1.0, "first_observed_ms": { "min": 5.230374983511865, "p50": 17.30947953183204, "p95": 32.20703732222319, "p99": 36.123908901354284, "max": 36.836290964856744 }, "confirmed_ms": { "min": 64.03537490405142, "p50": 82.32022944139317, "p95": 104.35666070552543, "p99": 108.04903669399208, "max": 115.25137501303107 }, "ack_ms": { "min": 2.3805409437045455, "p50": 9.543500025756657, "p95": 15.11642120894976, "p99": 18.17390624433755, "max": 20.98550007212907 }, "search_minus_ack_ms": { "min": 0.063000014051795, "p50": 1.479270518757403, "p95": 18.649406218901277, "p99": 20.46131410519592, "max": 20.80674993339926 }, "regressions": 0, "confirmed_within_100ms": 0.8846153846153846, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "update": { "count": 130, "success": 130, "success_rate": 1.0, "first_observed_ms": { "min": 8.128167013637722, "p50": 16.828833031468093, "p95": 32.38528749789111, "p99": 34.14640799164772, "max": 39.661459042690694 }, "confirmed_ms": { "min": 64.12058300338686, "p50": 81.87858294695616, "p95": 104.62289359420538, "p99": 106.74594372045247, "max": 109.30033400654793 }, "ack_ms": { "min": 2.5302499998360872, "p50": 9.51141701079905, "p95": 15.797200112137944, "p99": 16.93354236311279, "max": 20.711333956569433 }, "search_minus_ack_ms": { "min": 0.0806670868769288, "p50": 1.813228998798877, "p95": 18.96038567647338, "p99": 21.140725207515057, "max": 24.99299996998161 }, "regressions": 0, "confirmed_within_100ms": 0.8615384615384616, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "delete": { "count": 130, "success": 130, "success_rate": 1.0, "first_observed_ms": { "min": 3.4917080774903297, "p50": 15.663000522181392, "p95": 20.620960614178326, "p99": 31.616147226886817, "max": 32.35145891085267 }, "confirmed_ms": { "min": 58.432291029021144, "p50": 80.34833351848647, "p95": 86.93205261370167, "p99": 103.30055375001392, "max": 104.44466699846089 }, "ack_ms": { "min": 1.445583999156952, "p50": 4.931500006932765, "p95": 11.506537476088852, "p99": 13.562658933224153, "max": 16.054082894697785 }, "search_minus_ack_ms": { "min": -2.772207953967154, "p50": 5.105791555251926, "p95": 7.761510054115204, "p99": 18.892248070333146, "max": 19.373917020857334 }, "regressions": 0, "confirmed_within_100ms": 0.9769230769230769, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 } }, "total_operations": 390, "statuses": { "success": 390, "timeout": 0, "error": 0, "invalid_precondition": 0, "client_overload": 0 }, "query": { "count": 300, "success": 300, "ann_recall_at_10": 0.991, "ndcg_at_10": 0.7931268675492565, "latency_ms": { "min": 4.174749949015677, "p50": 11.290666472632438, "p95": 16.028337145689875, "p99": 20.181778760161244, "max": 22.380374954082072 }, "dispatch_delay_ms": { "min": 0.0813750084489584, "p50": 1.4669160009361804, "p95": 2.5273603911045943, "p99": 3.254050926771015, "max": 3.5286659840494394 } }, "diagnostics": { "schedule_delay_ms": { "min": 0.003583007490703949, "p50": 0.41708300123044495, "p95": 0.746422582305814, "p99": 0.7809327039871589, "max": 1.2135409517206597 }, "probe_dispatch_ms": { "min": 0.019124941900372505, "p50": 0.4516670014709234, "p95": 2.0829336484894156, "p99": 2.812047014012933, "max": 3.479541977867484 }, "probe_service_ms": { "min": 0.5482080159708858, "p50": 4.087624954991043, "p95": 11.566408444195986, "p99": 14.475405020639306, "max": 22.831209003925323 }, "write_service_ms": { "min": 1.2791251065209508, "p50": 6.6196254920214415, "p95": 14.115677465451876, "p99": 16.712904574815187, "max": 20.595209090970457 } }, "actual_submission_rate": 13.009851567984631, "metadata": { "backend": "weaviate", "collection": "fb2_429e9a_008", "dataset": { "dataset": "nq", "source": "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/nq.zip", "source_sha256": "2553bf7bfbab47b1436ca00a34bce57320e18e611fd00999a2a3a1b4714be752", "source_md5": "d4d3d2e48787a744b6f6e691ff534307", "records": 124000, "queries": 100, "relevant_documents_retained": 120, "seed": 20261028, "license": "CC-BY-SA-3.0", "license_source": "https://raw.githubusercontent.com/google-research-datasets/natural-questions/master/nq_open/README.md", "attribution": "BEIR and original dataset authors; Wikipedia contributors for NQ, SciFact authors and cited article authors for SciFact.", "transformation": "200-word title+text prefix; seeded reservoir of documents with relevant documents for selected test queries retained at front. Simulated mutations; no historical replay.", "model": "sentence-transformers/all-MiniLM-L6-v2", "model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "dimensions": 384, "normalized": true, "max_tokens": 256, "device": "mps", "files": { "corpus.jsonl": "9dc487d785e6ee493eccaf764daa3bf8720a8f4fc8c406246065e1ad33a91f5d", "queries.json": "5603f52876ada5d36118b65dea490f1d43900bc159e1804e637796eb9115e999", "vectors.npy": "cb7421eb6ca5964231403f0a922843088f6fd0e01cbf67f2a1076cc7101005db", "query_vectors.npy": "5e666f8d966d8db6b0fd93d4963234d2dffa0ee3910b8fad069a8ee201e8b5d4", "UPSTREAM-LICENSE.md": "8b675521da3fa142ac6bda38c9bcfcc8cff0e2d6128be53a062ed55fb53f7b87" } }, "base_count": 100000, "profile": { "dataset": "nq", "base_count": 100000, "rate": 10, "query_rate": 10, "arrival": "burst" }, "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "burst", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "preloaded_mutable": 260, "started_utc": "2026-10-29T02:50:23Z", "setup_s": 35.84677649999503, "backend_before": { "version": "1.40.3", "schema": { "class": "Ffb2429e9a008", "invertedIndexConfig": { "bm25": { "b": 0.75, "k1": 1.2 }, "cleanupIntervalSeconds": 60, "stopwords": { "additions": null, "preset": "en", "removals": null }, "usingBlockMaxWAND": true }, "multiTenancyConfig": { "autoTenantActivation": false, "autoTenantCreation": false, "enabled": false }, "properties": [ { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "doc_id", "tokenization": "field" }, { "dataType": [ "int" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": false, "name": "version" }, { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "cohort", "tokenization": "field" }, { "dataType": [ "text" ], "indexFilterable": false, "indexRangeFilters": false, "indexSearchable": false, "name": "text", "tokenization": "word" } ], "shardingConfig": { "actualCount": 1, "actualVirtualCount": 128, "desiredCount": 1, "desiredVirtualCount": 128, "function": "murmur3", "key": "_id", "strategy": "hash", "virtualPerPhysical": 128 }, "vectorIndexConfig": { "bq": { "enabled": false }, "cleanupIntervalSeconds": 300, "distance": "cosine", "dynamicEfFactor": 8, "dynamicEfMax": 500, "dynamicEfMin": 100, "ef": 128, "efConstruction": 128, "filterStrategy": "acorn", "flatSearchCutoff": 0, "maxConnections": 16, "multivector": { "aggregation": "maxSim", "enabled": false, "muvera": { "dprojections": 16, "enabled": false, "ksim": 4, "repetitions": 10 } }, "pq": { "bitCompression": false, "centroids": 256, "enabled": false, "encoder": { "distribution": "log-normal", "type": "kmeans" }, "segments": 0, "trainingLimit": 100000 }, "rq": { "bits": 8, "centering": false, "enabled": false, "rescoreLimit": 20, "trainingLimit": 10000 }, "skip": false, "skipDefaultQuantization": false, "sq": { "enabled": false, "rescoreLimit": 20, "trainingLimit": 100000 }, "trackDefaultQuantization": false, "vectorCacheMaxObjects": 1000000000000 }, "vectorIndexType": "hnsw", "vectorizer": "none", "replicationConfig": { "deletionStrategy": "TimeBasedResolution", "factor": 1, "asyncEnabled": false } }, "shards": [ { "name": "WTLacvyJVr9h", "per_node_status": { "node1": "READY" }, "status": "READY", "vectorQueueSize": 0 } ], "ack": "synchronous indexing, object PUT/POST/DELETE", "transport": "HTTP REST writes + GraphQL search, pooled httpx" }, "count_before": 100260, "idle_duration_s": 10.0, "idle_queries": { "count": 100, "ann_recall": 0.991, "ndcg": 0.7931268675492565, "latency_ms": { "min": 3.3992499811574817, "p50": 11.229187482967973, "p95": 13.320814981125295, "p99": 13.813596346881242, "max": 14.095375081524253 } }, "exact_ndcg": 0.7931268675492565, "resources_before": { "usage_usec": 74167107, "user_usec": 72808916, "system_usec": 1358190, "nice_usec": 0, "nr_periods": 2251, "nr_throttled": 123, "throttled_usec": 544633, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 1249009664, "pids": 10 }, "resources_after": { "usage_usec": 84262026, "user_usec": 82429935, "system_usec": 1832090, "nice_usec": 0, "nr_periods": 2552, "nr_throttled": 123, "throttled_usec": 544633, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 1219600384, "pids": 10 }, "count_after": 100260, "backend_after": { "version": "1.40.3", "schema": { "class": "Ffb2429e9a008", "invertedIndexConfig": { "bm25": { "b": 0.75, "k1": 1.2 }, "cleanupIntervalSeconds": 60, "stopwords": { "additions": null, "preset": "en", "removals": null }, "usingBlockMaxWAND": true }, "multiTenancyConfig": { "autoTenantActivation": false, "autoTenantCreation": false, "enabled": false }, "properties": [ { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "doc_id", "tokenization": "field" }, { "dataType": [ "int" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": false, "name": "version" }, { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "cohort", "tokenization": "field" }, { "dataType": [ "text" ], "indexFilterable": false, "indexRangeFilters": false, "indexSearchable": false, "name": "text", "tokenization": "word" } ], "shardingConfig": { "actualCount": 1, "actualVirtualCount": 128, "desiredCount": 1, "desiredVirtualCount": 128, "function": "murmur3", "key": "_id", "strategy": "hash", "virtualPerPhysical": 128 }, "vectorIndexConfig": { "bq": { "enabled": false }, "cleanupIntervalSeconds": 300, "distance": "cosine", "dynamicEfFactor": 8, "dynamicEfMax": 500, "dynamicEfMin": 100, "ef": 128, "efConstruction": 128, "filterStrategy": "acorn", "flatSearchCutoff": 0, "maxConnections": 16, "multivector": { "aggregation": "maxSim", "enabled": false, "muvera": { "dprojections": 16, "enabled": false, "ksim": 4, "repetitions": 10 } }, "pq": { "bitCompression": false, "centroids": 256, "enabled": false, "encoder": { "distribution": "log-normal", "type": "kmeans" }, "segments": 0, "trainingLimit": 100000 }, "rq": { "bits": 8, "centering": false, "enabled": false, "rescoreLimit": 20, "trainingLimit": 10000 }, "skip": false, "skipDefaultQuantization": false, "sq": { "enabled": false, "rescoreLimit": 20, "trainingLimit": 100000 }, "trackDefaultQuantization": false, "vectorCacheMaxObjects": 1000000000000 }, "vectorIndexType": "hnsw", "vectorizer": "none", "replicationConfig": { "deletionStrategy": "TimeBasedResolution", "factor": 1, "asyncEnabled": false } }, "shards": [ { "name": "WTLacvyJVr9h", "per_node_status": { "node1": "READY" }, "status": "READY", "vectorQueueSize": 0 } ], "ack": "synchronous indexing, object PUT/POST/DELETE", "transport": "HTTP REST writes + GraphQL search, pooled httpx" } }, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "eligible": true, "protocol": "mixed-mutations-v2" }, "examples": [ { "event": 0, "operation": "insert", "doc_id": "mutation-000000", "scheduled_s": 0.0, "submitted_s": 0.00015075004193931818, "schedule_delay_ms": 0.15075004193931818, "precondition": true, "status": "success", "ack_ms": 6.405749940313399, "write_started_ms": 1.2495409464463592, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 13.675750000402331, "stable_since_ms": 13.675750000402331, "confirmed_ms": 66.4208329981193, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 8.33604100625962, "stable_since_ms": 8.33604100625962, "confirmed_ms": 66.34749996010214, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 13.675750000402331, "confirmed_ms": 66.4208329981193, "search_minus_ack_ms": 1.9302910659462214 }, { "event": 1, "operation": "update", "doc_id": "mutation-000001", "scheduled_s": 0.025, "submitted_s": 0.025072374963201582, "schedule_delay_ms": 0.07237496320158104, "precondition": true, "status": "success", "ack_ms": 3.3393329940736294, "write_started_ms": 0.15429104678332806, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 12.84762506838888, "stable_since_ms": 12.84762506838888, "confirmed_ms": 66.44058309029788, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 3.791125025600195, "stable_since_ms": 3.791125025600195, "confirmed_ms": 60.4742910945788, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 12.84762506838888, "confirmed_ms": 66.44058309029788, "search_minus_ack_ms": 0.45179203152656555 }, { "event": 2, "operation": "delete", "doc_id": "mutation-000002", "scheduled_s": 0.05, "submitted_s": 0.05018604104407132, "schedule_delay_ms": 0.18604104407131394, "precondition": true, "status": "success", "ack_ms": 2.082417020574212, "write_started_ms": 0.1664169831201434, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 14.01074999012053, "stable_since_ms": 14.01074999012053, "confirmed_ms": 65.93470892403275, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 4.764958983287215, "stable_since_ms": 4.764958983287215, "confirmed_ms": 61.29933393094689, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 14.01074999012053, "confirmed_ms": 65.93470892403275, "search_minus_ack_ms": 2.682541962713003 } ] }, { "run": "fb2_429e9a_009", "trial": 1, "backend": "qdrant", "scenario": "nq-100k-burst", "status": "complete", "eligible": true, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "summary": { "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "burst", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "wall_s": 30.064481833018363, "operations": { "insert": { "count": 130, "success": 130, "success_rate": 1.0, "first_observed_ms": { "min": 15.119959018193185, "p50": 69.1551459603943, "p95": 103.72933360631578, "p99": 108.61954802297986, "max": 114.02158299461007 }, "confirmed_ms": { "min": 148.10608397237957, "p50": 216.98093746090308, "p95": 245.9421022911556, "p99": 252.29500113404356, "max": 261.55824994202703 }, "ack_ms": { "min": 3.0774999177083373, "p50": 8.224707969930023, "p95": 12.566612754017115, "p99": 17.826335130957855, "max": 26.824917062185705 }, "search_minus_ack_ms": { "min": 2.121250028721988, "p50": 58.97485447349027, "p95": 98.62390900379978, "p99": 99.74153068847954, "max": 100.15337506774813 }, "regressions": 0, "confirmed_within_100ms": 0.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "update": { "count": 130, "success": 130, "success_rate": 1.0, "first_observed_ms": { "min": 17.745082965120673, "p50": 68.68595798732713, "p95": 103.45862485119142, "p99": 104.86222242936492, "max": 105.59854097664356 }, "confirmed_ms": { "min": 153.48937502130866, "p50": 215.80883348360658, "p95": 243.98482901160605, "p99": 248.77249860670418, "max": 251.03474990464747 }, "ack_ms": { "min": 2.799791982397437, "p50": 9.545895969495177, "p95": 12.696655985200778, "p99": 13.298515347996725, "max": 13.63904110621661 }, "search_minus_ack_ms": { "min": 2.034334000200033, "p50": 58.02535399561748, "p95": 97.3900291195605, "p99": 98.07028652401641, "max": 98.24416600167751 }, "regressions": 0, "confirmed_within_100ms": 0.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "delete": { "count": 130, "success": 130, "success_rate": 1.0, "first_observed_ms": { "min": 10.852333973161876, "p50": 18.94195849308744, "p95": 47.999368741875514, "p99": 60.65341420471659, "max": 99.2084169993177 }, "confirmed_ms": { "min": 109.40191592089832, "p50": 157.67056250479072, "p95": 191.35867680306546, "p99": 209.03700796188775, "max": 241.7753339977935 }, "ack_ms": { "min": 2.258082968182862, "p50": 8.336792001500726, "p95": 11.11932328203693, "p99": 12.878650883212684, "max": 18.68008403107524 }, "search_minus_ack_ms": { "min": -2.829583012498915, "p50": 0.3781875129789114, "p95": 43.00684812478721, "p99": 50.516656148247364, "max": 94.81754200533032 }, "regressions": 0, "confirmed_within_100ms": 0.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 } }, "total_operations": 390, "statuses": { "success": 390, "timeout": 0, "error": 0, "invalid_precondition": 0, "client_overload": 0 }, "query": { "count": 300, "success": 300, "ann_recall_at_10": 0.991, "ndcg_at_10": 0.7931268675492565, "latency_ms": { "min": 4.1133330669254065, "p50": 9.431167040020227, "p95": 48.03217915468849, "p99": 50.38923615124076, "max": 52.21054097637534 }, "dispatch_delay_ms": { "min": 0.2909171162173152, "p50": 1.4619790017604828, "p95": 2.327129570767284, "p99": 2.6462320075370336, "max": 7.509417016990483 } }, "diagnostics": { "schedule_delay_ms": { "min": 0.005083065481414906, "p50": 0.38856248138507965, "p95": 0.76676683966026, "p99": 0.9808239922859471, "max": 6.682917056604509 }, "probe_dispatch_ms": { "min": 0.01916696783155203, "p50": 0.46262494288384914, "p95": 1.6546586528420448, "p99": 2.5826695747673516, "max": 13.74512491747737 }, "probe_service_ms": { "min": 0.8032920304685831, "p50": 5.910875042900443, "p95": 49.32021624408662, "p99": 50.9727343916893, "max": 57.35616700258106 }, "write_service_ms": { "min": 2.148207975551486, "p50": 8.003020950127393, "p95": 11.844860599376256, "p99": 13.05718951276504, "max": 26.15579206030816 } }, "actual_submission_rate": 13.009991075305543, "metadata": { "backend": "qdrant", "collection": "fb2_429e9a_009", "dataset": { "dataset": "nq", "source": "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/nq.zip", "source_sha256": "2553bf7bfbab47b1436ca00a34bce57320e18e611fd00999a2a3a1b4714be752", "source_md5": "d4d3d2e48787a744b6f6e691ff534307", "records": 124000, "queries": 100, "relevant_documents_retained": 120, "seed": 20261028, "license": "CC-BY-SA-3.0", "license_source": "https://raw.githubusercontent.com/google-research-datasets/natural-questions/master/nq_open/README.md", "attribution": "BEIR and original dataset authors; Wikipedia contributors for NQ, SciFact authors and cited article authors for SciFact.", "transformation": "200-word title+text prefix; seeded reservoir of documents with relevant documents for selected test queries retained at front. Simulated mutations; no historical replay.", "model": "sentence-transformers/all-MiniLM-L6-v2", "model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "dimensions": 384, "normalized": true, "max_tokens": 256, "device": "mps", "files": { "corpus.jsonl": "9dc487d785e6ee493eccaf764daa3bf8720a8f4fc8c406246065e1ad33a91f5d", "queries.json": "5603f52876ada5d36118b65dea490f1d43900bc159e1804e637796eb9115e999", "vectors.npy": "cb7421eb6ca5964231403f0a922843088f6fd0e01cbf67f2a1076cc7101005db", "query_vectors.npy": "5e666f8d966d8db6b0fd93d4963234d2dffa0ee3910b8fad069a8ee201e8b5d4", "UPSTREAM-LICENSE.md": "8b675521da3fa142ac6bda38c9bcfcc8cff0e2d6128be53a062ed55fb53f7b87" } }, "base_count": 100000, "profile": { "dataset": "nq", "base_count": 100000, "rate": 10, "query_rate": 10, "arrival": "burst" }, "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "burst", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "preloaded_mutable": 260, "started_utc": "2026-10-29T02:51:43Z", "setup_s": 42.15754475002177, "backend_before": { "version": "1.20.0", "collection": { "status": "green", "optimizer_status": "ok", "indexed_vectors_count": 100260, "points_count": 100260, "segments_count": 2, "config": { "params": { "vectors": { "size": 384, "distance": "Cosine" }, "shard_number": 1, "replication_factor": 1, "write_consistency_factor": 1, "on_disk_payload": true }, "hnsw_config": { "m": 16, "ef_construct": 128, "full_scan_threshold": 10, "max_indexing_threads": 0, "on_disk": false }, "optimizer_config": { "deleted_threshold": 0.2, "vacuum_min_vector_number": 1000, "default_segment_number": 0, "max_segment_size": null, "memmap_threshold": null, "indexing_threshold": 1024, "flush_interval_sec": 5, "max_optimization_threads": null, "prevent_unoptimized": null }, "wal_config": { "wal_capacity_mb": 32, "wal_segments_ahead": 0, "wal_retain_closed": 1 }, "quantization_config": null }, "payload_schema": { "cohort": { "data_type": "keyword", "points": 100260 } }, "update_queue": { "length": 0 }, "created_at": "2026-10-29T02:51:43.538Z" }, "ack": "wait=true", "transport": "HTTP REST, pooled httpx" }, "count_before": 100260, "idle_duration_s": 10.0, "idle_queries": { "count": 100, "ann_recall": 0.991, "ndcg": 0.7931268675492565, "latency_ms": { "min": 4.7958329087123275, "p50": 9.539729042444378, "p95": 11.009451729478315, "p99": 12.776280452963162, "max": 50.83241709508002 } }, "exact_ndcg": 0.7931268675492565, "resources_before": { "usage_usec": 62308121, "user_usec": 59976420, "system_usec": 2331701, "nice_usec": 55206000, "nr_periods": 892, "nr_throttled": 79, "throttled_usec": 295707, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 586047488, "pids": 31 }, "resources_after": { "usage_usec": 69598959, "user_usec": 66472469, "system_usec": 3126490, "nice_usec": 55206000, "nr_periods": 1193, "nr_throttled": 79, "throttled_usec": 295707, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 566857728, "pids": 33 }, "count_after": 100260, "backend_after": { "version": "1.20.0", "collection": { "status": "green", "optimizer_status": "ok", "indexed_vectors_count": 100260, "points_count": 100260, "segments_count": 2, "config": { "params": { "vectors": { "size": 384, "distance": "Cosine" }, "shard_number": 1, "replication_factor": 1, "write_consistency_factor": 1, "on_disk_payload": true }, "hnsw_config": { "m": 16, "ef_construct": 128, "full_scan_threshold": 10, "max_indexing_threads": 0, "on_disk": false }, "optimizer_config": { "deleted_threshold": 0.2, "vacuum_min_vector_number": 1000, "default_segment_number": 0, "max_segment_size": null, "memmap_threshold": null, "indexing_threshold": 1024, "flush_interval_sec": 5, "max_optimization_threads": null, "prevent_unoptimized": null }, "wal_config": { "wal_capacity_mb": 32, "wal_segments_ahead": 0, "wal_retain_closed": 1 }, "quantization_config": null }, "payload_schema": { "cohort": { "data_type": "keyword", "points": 100260 } }, "update_queue": { "length": 0 }, "created_at": "2026-10-29T02:51:43.538Z" }, "ack": "wait=true", "transport": "HTTP REST, pooled httpx" } }, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "eligible": true, "protocol": "mixed-mutations-v2" }, "examples": [ { "event": 0, "operation": "insert", "doc_id": "mutation-000000", "scheduled_s": 0.0, "submitted_s": 0.00019329204224050045, "schedule_delay_ms": 0.19329204224050045, "precondition": true, "status": "success", "ack_ms": 6.5833330154418945, "write_started_ms": 1.6379579901695251, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 14.609540929086506, "stable_since_ms": 14.609540929086506, "confirmed_ms": 67.99341598525643, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 60.451166005805135, "stable_since_ms": 60.451166005805135, "confirmed_ms": 198.24437494389713, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 60.451166005805135, "confirmed_ms": 198.24437494389713, "search_minus_ack_ms": 53.86783299036324 }, { "event": 1, "operation": "update", "doc_id": "mutation-000001", "scheduled_s": 0.025, "submitted_s": 0.025360875064507127, "schedule_delay_ms": 0.3608750645071254, "precondition": true, "status": "success", "ack_ms": 4.396833013743162, "write_started_ms": 0.8031249744817615, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 14.335707994177938, "stable_since_ms": 14.335707994177938, "confirmed_ms": 67.78666691388935, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 97.95074991416186, "stable_since_ms": 97.95074991416186, "confirmed_ms": 238.39074990246445, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 97.95074991416186, "confirmed_ms": 238.39074990246445, "search_minus_ack_ms": 93.5539169004187 }, { "event": 2, "operation": "delete", "doc_id": "mutation-000002", "scheduled_s": 0.05, "submitted_s": 0.050257875001989305, "schedule_delay_ms": 0.25787500198930224, "precondition": true, "status": "success", "ack_ms": 3.9828330045565963, "write_started_ms": 0.6767919985577464, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 14.326333068311214, "stable_since_ms": 14.326333068311214, "confirmed_ms": 69.4234580732882, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 44.26866699941456, "stable_since_ms": 44.26866699941456, "confirmed_ms": 185.09845796506852, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 44.26866699941456, "confirmed_ms": 185.09845796506852, "search_minus_ack_ms": 40.28583399485797 } ] }, { "run": "fb2_429e9a_010", "trial": 2, "backend": "pgvector", "scenario": "nq-100k-burst", "status": "complete", "eligible": true, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "summary": { "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "burst", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "wall_s": 30.001297708018683, "operations": { "insert": { "count": 130, "success": 130, "success_rate": 1.0, "first_observed_ms": { "min": 13.498207903467119, "p50": 28.602082980796695, "p95": 34.159068594453856, "p99": 37.116854357300326, "max": 42.316957958973944 }, "confirmed_ms": { "min": 72.11112498771399, "p50": 88.74406246468425, "p95": 97.11937916581519, "p99": 101.53049270506015, "max": 109.22554100397974 }, "ack_ms": { "min": 4.680458921939135, "p50": 16.674395476002246, "p95": 22.356064926134422, "p99": 24.556323195574805, "max": 28.08412502054125 }, "search_minus_ack_ms": { "min": 5.539707955904305, "p50": 11.930249980650842, "p95": 13.55524375103414, "p99": 13.98791816434823, "max": 14.232832938432693 }, "regressions": 0, "confirmed_within_100ms": 0.9846153846153847, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "update": { "count": 130, "success": 130, "success_rate": 1.0, "first_observed_ms": { "min": 16.676916042342782, "p50": 26.32120851194486, "p95": 31.7727543937508, "p99": 32.81097156577743, "max": 34.63258303236216 }, "confirmed_ms": { "min": 72.64895795378834, "p50": 86.6471664630808, "p95": 93.06218500132672, "p99": 93.65598365198821, "max": 97.93345804791898 }, "ack_ms": { "min": 5.210583098232746, "p50": 14.850978972390294, "p95": 21.104285452747717, "p99": 22.061234493739903, "max": 22.47370802797377 }, "search_minus_ack_ms": { "min": 8.232458960264921, "p50": 11.860541999340057, "p95": 13.670677284244446, "p99": 14.252661406062542, "max": 14.453875017352402 }, "regressions": 0, "confirmed_within_100ms": 1.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "delete": { "count": 130, "success": 130, "success_rate": 1.0, "first_observed_ms": { "min": 5.399915971793234, "p50": 12.862250034231693, "p95": 28.914977057138458, "p99": 30.147843370214105, "max": 31.472417060285807 }, "confirmed_ms": { "min": 62.37908301409334, "p50": 74.50629153754562, "p95": 93.75949619570747, "p99": 95.47603425453417, "max": 96.26120899338275 }, "ack_ms": { "min": 0.5900840042158961, "p50": 1.8913540407083929, "p95": 5.161808739649132, "p99": 5.692638971377163, "max": 6.114750052802265 }, "search_minus_ack_ms": { "min": 2.7849170146510005, "p50": 9.671375039033592, "p95": 25.431973213562742, "p99": 26.46444346057251, "max": 27.85191696602851 }, "regressions": 0, "confirmed_within_100ms": 1.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 } }, "total_operations": 390, "statuses": { "success": 390, "timeout": 0, "error": 0, "invalid_precondition": 0, "client_overload": 0 }, "query": { "count": 300, "success": 300, "ann_recall_at_10": 0.994, "ndcg_at_10": 0.7931268675492565, "latency_ms": { "min": 3.517167060635984, "p50": 10.165291489101946, "p95": 14.714598224963993, "p99": 16.809578399406735, "max": 27.54795807413757 }, "dispatch_delay_ms": { "min": 0.0831669894978404, "p50": 1.0720000718720257, "p95": 1.7645976680796596, "p99": 1.9484527281019832, "max": 2.099083038046956 } }, "diagnostics": { "schedule_delay_ms": { "min": 0.004333024843816702, "p50": 0.40308353491136373, "p95": 0.7513983198440322, "p99": 0.8121587124175635, "max": 1.1605420848361803 }, "probe_dispatch_ms": { "min": 0.02745899837464094, "p50": 0.485624885186553, "p95": 1.3585247332230208, "p99": 1.85043451376259, "max": 3.1998750055208802 }, "probe_service_ms": { "min": 0.20949996542185545, "p50": 1.9899581093341112, "p95": 11.238533351570368, "p99": 14.89116176497188, "max": 27.27808302734047 }, "write_service_ms": { "min": 0.5145840113982558, "p50": 8.686396002303809, "p95": 20.93720454722643, "p99": 23.217451308155432, "max": 27.96237508300692 } }, "actual_submission_rate": 13.009889277433512, "metadata": { "backend": "pgvector", "collection": "fb2_429e9a_010", "dataset": { "dataset": "nq", "source": "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/nq.zip", "source_sha256": "2553bf7bfbab47b1436ca00a34bce57320e18e611fd00999a2a3a1b4714be752", "source_md5": "d4d3d2e48787a744b6f6e691ff534307", "records": 124000, "queries": 100, "relevant_documents_retained": 120, "seed": 20261028, "license": "CC-BY-SA-3.0", "license_source": "https://raw.githubusercontent.com/google-research-datasets/natural-questions/master/nq_open/README.md", "attribution": "BEIR and original dataset authors; Wikipedia contributors for NQ, SciFact authors and cited article authors for SciFact.", "transformation": "200-word title+text prefix; seeded reservoir of documents with relevant documents for selected test queries retained at front. Simulated mutations; no historical replay.", "model": "sentence-transformers/all-MiniLM-L6-v2", "model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "dimensions": 384, "normalized": true, "max_tokens": 256, "device": "mps", "files": { "corpus.jsonl": "9dc487d785e6ee493eccaf764daa3bf8720a8f4fc8c406246065e1ad33a91f5d", "queries.json": "5603f52876ada5d36118b65dea490f1d43900bc159e1804e637796eb9115e999", "vectors.npy": "cb7421eb6ca5964231403f0a922843088f6fd0e01cbf67f2a1076cc7101005db", "query_vectors.npy": "5e666f8d966d8db6b0fd93d4963234d2dffa0ee3910b8fad069a8ee201e8b5d4", "UPSTREAM-LICENSE.md": "8b675521da3fa142ac6bda38c9bcfcc8cff0e2d6128be53a062ed55fb53f7b87" } }, "base_count": 100000, "profile": { "dataset": "nq", "base_count": 100000, "rate": 10, "query_rate": 10, "arrival": "burst" }, "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "burst", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "preloaded_mutable": 260, "started_utc": "2026-10-29T02:53:12Z", "setup_s": 58.30551987502258, "backend_before": { "version": "0.8.7", "postgres": "PostgreSQL 16.15 (Debian 16.15-1.pgdg12+2) on aarch64-unknown-linux-gnu, compiled by gcc (Debian 12.2.0-14+deb12u1) 12.2.0, 64-bit", "ef_search": 128, "ack": "autocommit, READ COMMITTED", "transport": "psycopg, connection per worker", "query_plan": [ { "Plan": { "Node Type": "Limit", "Parallel Aware": false, "Async Capable": false, "Startup Cost": 1674.28, "Total Cost": 1691.41, "Plan Rows": 10, "Plan Width": 18, "Plans": [ { "Node Type": "Index Scan", "Parent Relationship": "Outer", "Parallel Aware": false, "Async Capable": false, "Scan Direction": "Forward", "Index Name": "fb2_429e9a_010_hnsw", "Relation Name": "fb2_429e9a_010", "Alias": "fb2_429e9a_010", "Startup Cost": 1674.28, "Total Cost": 173019.27, "Plan Rows": 100029, "Plan Width": 18, "Order By": "(embedding <=> '[-0.08946895,-0.094362676,0.092023626,-0.0040844884,0.043233823,0.00062519487,0.1075197,-0.035732996,0.013746996,-0.024103122,0.0021706754,-0.04720602,0.04787615,-0.02043222,0.058315646,0.081252135,-0.05375015,-0.030799072,-0.0077205584,0.035735283,-0.04474784,0.01614922,-0.06905586,0.04194401,-0.08832908,0.007873556,0.0044992934,0.021968655,0.010953364,0.07212316,0.023185596,0.001176476,0.04725166,-0.03580423,-0.032344446,0.004752835,-0.056179125,-0.0353235,-0.037251893,-0.005143068,-0.027599342,0.024403147,0.031338032,0.0056410627,0.008425455,-0.0087741045,-0.039870497,-0.011831836,0.042621233,0.048142947,0.01603842,0.06745542,-0.038018145,0.064212486,-0.009636566,0.01607475,-0.0027868534,-0.03761097,-0.011889431,-0.01084577,0.083403684,0.028946918,0.061870236,-0.045382615,0.03812586,0.060426526,-0.017003024,0.02328925,-0.093020774,-0.024072766,0.042720392,-0.022444837,-0.011597358,0.01209231,-0.005953629,0.029691145,-0.011711495,-0.053345133,-0.081757374,-0.049303155,0.032193717,-0.038507868,-0.0060101775,-0.031470332,-0.07362443,0.03575578,0.0245484,0.01985519,-0.15116754,0.050386053,-0.0389634,0.07548389,-0.09264168,-0.012602816,-0.0035539444,-0.046416666,0.015100493,0.100691915,-0.037529,0.04600518,-0.021859422,0.014675989,-0.097722545,-0.053200893,-0.017040461,-0.01629616,-0.012339222,0.06543302,0.07252074,-0.044301156,0.03054835,-0.038570553,0.019310065,-0.005527747,-0.029386865,-0.058248505,0.065443166,0.028506057,0.0878254,0.058387812,0.081721246,-0.0005962058,0.0569388,0.00960514,0.011667701,0.014456492,-0.07218306,-3.0444767e-33,0.06687367,-0.019213954,0.058202963,0.025171444,-0.028259544,-0.04171671,0.03127722,0.04865431,-0.0721459,-0.050143186,-0.029907746,0.06248994,-0.05096076,-0.065192886,0.010210447,-0.077256374,-0.043370593,-0.0901922,-0.066522665,0.026072454,-0.022315364,0.075375885,-0.029968103,-0.0050771856,-0.0046955296,0.0065650316,-0.06791897,0.07368851,0.056069646,-0.021026025,-0.011352883,0.038811427,-0.090847075,-0.056490287,-0.039881036,0.032885592,-0.029586185,0.04753608,0.006341778,-0.0034275919,0.03681623,0.09326265,-0.0532089,0.032396164,0.012605483,0.045778517,-0.0006472902,0.029721813,0.0734947,-0.037117425,-0.09356724,-0.0071634455,-0.032400057,0.010453378,0.06286513,0.006053158,0.060524322,0.091866635,0.063640274,0.055919155,0.053222507,-0.019174587,0.089296214,-0.058922265,0.013745364,-0.022907477,0.028347451,-0.040759467,-0.06820364,-0.061398294,-0.046857864,0.0060627847,0.010442015,-0.07662317,0.031950947,-0.107813306,-0.0016125626,-0.033413693,-0.09989722,-0.03580293,-0.036118146,-0.08276185,0.1280713,0.07596603,0.049225595,-0.026312606,0.0031930343,-0.0012469466,0.000872793,-0.040353082,-0.034516506,-0.036047976,0.004999689,-0.01570079,-0.09783203,1.4186912e-33,-0.0130903,0.04742377,0.02889633,0.09809088,0.043639127,-0.0747531,0.03402747,0.029147558,0.019848268,0.111865185,0.04845458,-0.016222177,0.025501896,0.09813583,-0.002833195,0.0038929228,0.048537854,0.014111116,0.036570966,-0.017512968,-0.079755045,0.043452248,0.10843892,0.061968718,-0.015678233,0.0060376576,0.010718569,-0.03907552,0.040307663,0.04553227,0.041731495,0.083179526,-0.0844024,-0.023447752,0.014485363,-0.00788756,-0.11410207,0.039309476,0.011220049,0.12889338,-0.07970701,0.010658111,-0.06657531,-0.022037188,-0.043459363,-0.030291978,0.05803613,0.06109021,-0.09415837,0.030727232,0.011858702,0.013526454,-0.049580045,-0.052486703,0.013218615,0.0050231596,0.07660412,-0.0046343356,-0.0692584,-0.03353194,0.022288544,0.022836983,0.01361394,-0.036331616,0.008194689,0.09226991,0.08229875,-0.047639843,-0.005033471,0.0066081914,-0.106666416,-0.06385503,0.006385333,0.06318358,0.03146985,-0.060764216,-0.043287244,-0.042385958,-0.07963737,-0.043478705,-0.031840656,0.060027804,0.027829666,0.03728562,-0.07014622,0.0649568,-0.052579004,-0.0034938955,-0.15390792,-0.04426162,0.028217599,0.05593145,0.023251446,0.029115746,-0.040321343,-1.8832651e-08,-0.037536032,0.0005056563,0.013919978,-0.038129315,0.061486673,0.005534046,-0.010768211,-0.16527015,0.061414365,0.019963786,-0.019666849,-0.059065994,0.00083181454,-0.016695915,0.01090875,0.02685501,-0.0008743374,-0.060967337,-0.049258173,0.012731965,-0.03973743,-0.0018210232,-0.03797001,0.00035104176,0.04477184,0.04185603,-0.03215796,-0.09103283,0.0467481,0.017810425,0.037437033,0.054782324,-0.0059442874,0.022557084,0.053393807,-0.03746794,-0.07080435,0.10548834,-0.04215435,0.042547524,0.05064807,0.07560324,0.04072184,-0.0011278372,0.00702018,-0.070807114,0.040035766,0.051417697,0.011316928,-0.028955372,0.04197099,-0.0026867494,-0.056857754,0.0761941,0.030621601,-0.052413527,-0.021492323,0.0957355,0.1013111,-0.082977295,-0.036080915,-0.03327806,0.0029404552,-0.03586191]'::vector)", "Filter": "(cohort = 'static'::text)" } ] } } ] }, "count_before": 100260, "idle_duration_s": 10.0, "idle_queries": { "count": 100, "ann_recall": 0.9940000000000001, "ndcg": 0.7931268675492565, "latency_ms": { "min": 3.96891706623137, "p50": 17.751478997524828, "p95": 21.705120289698243, "p99": 22.992438721703365, "max": 23.10595801100135 } }, "exact_ndcg": 0.7931268675492565, "resources_before": { "usage_usec": 200470230, "user_usec": 189615995, "system_usec": 10854234, "nice_usec": 0, "nr_periods": 2342, "nr_throttled": 835, "throttled_usec": 88603928, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 1422323712, "pids": 23 }, "resources_after": { "usage_usec": 211974760, "user_usec": 197640777, "system_usec": 14333982, "nice_usec": 0, "nr_periods": 2643, "nr_throttled": 835, "throttled_usec": 88603928, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 1450696704, "pids": 34 }, "count_after": 100260, "backend_after": { "version": "0.8.7", "postgres": "PostgreSQL 16.15 (Debian 16.15-1.pgdg12+2) on aarch64-unknown-linux-gnu, compiled by gcc (Debian 12.2.0-14+deb12u1) 12.2.0, 64-bit", "ef_search": 128, "ack": "autocommit, READ COMMITTED", "transport": "psycopg, connection per worker", "query_plan": [ { "Plan": { "Node Type": "Limit", "Parallel Aware": false, "Async Capable": false, "Startup Cost": 1676.79, "Total Cost": 1693.93, "Plan Rows": 10, "Plan Width": 18, "Plans": [ { "Node Type": "Index Scan", "Parent Relationship": "Outer", "Parallel Aware": false, "Async Capable": false, "Scan Direction": "Forward", "Index Name": "fb2_429e9a_010_hnsw", "Relation Name": "fb2_429e9a_010", "Alias": "fb2_429e9a_010", "Startup Cost": 1676.79, "Total Cost": 173365.95, "Plan Rows": 100148, "Plan Width": 18, "Order By": "(embedding <=> '[-0.08946895,-0.094362676,0.092023626,-0.0040844884,0.043233823,0.00062519487,0.1075197,-0.035732996,0.013746996,-0.024103122,0.0021706754,-0.04720602,0.04787615,-0.02043222,0.058315646,0.081252135,-0.05375015,-0.030799072,-0.0077205584,0.035735283,-0.04474784,0.01614922,-0.06905586,0.04194401,-0.08832908,0.007873556,0.0044992934,0.021968655,0.010953364,0.07212316,0.023185596,0.001176476,0.04725166,-0.03580423,-0.032344446,0.004752835,-0.056179125,-0.0353235,-0.037251893,-0.005143068,-0.027599342,0.024403147,0.031338032,0.0056410627,0.008425455,-0.0087741045,-0.039870497,-0.011831836,0.042621233,0.048142947,0.01603842,0.06745542,-0.038018145,0.064212486,-0.009636566,0.01607475,-0.0027868534,-0.03761097,-0.011889431,-0.01084577,0.083403684,0.028946918,0.061870236,-0.045382615,0.03812586,0.060426526,-0.017003024,0.02328925,-0.093020774,-0.024072766,0.042720392,-0.022444837,-0.011597358,0.01209231,-0.005953629,0.029691145,-0.011711495,-0.053345133,-0.081757374,-0.049303155,0.032193717,-0.038507868,-0.0060101775,-0.031470332,-0.07362443,0.03575578,0.0245484,0.01985519,-0.15116754,0.050386053,-0.0389634,0.07548389,-0.09264168,-0.012602816,-0.0035539444,-0.046416666,0.015100493,0.100691915,-0.037529,0.04600518,-0.021859422,0.014675989,-0.097722545,-0.053200893,-0.017040461,-0.01629616,-0.012339222,0.06543302,0.07252074,-0.044301156,0.03054835,-0.038570553,0.019310065,-0.005527747,-0.029386865,-0.058248505,0.065443166,0.028506057,0.0878254,0.058387812,0.081721246,-0.0005962058,0.0569388,0.00960514,0.011667701,0.014456492,-0.07218306,-3.0444767e-33,0.06687367,-0.019213954,0.058202963,0.025171444,-0.028259544,-0.04171671,0.03127722,0.04865431,-0.0721459,-0.050143186,-0.029907746,0.06248994,-0.05096076,-0.065192886,0.010210447,-0.077256374,-0.043370593,-0.0901922,-0.066522665,0.026072454,-0.022315364,0.075375885,-0.029968103,-0.0050771856,-0.0046955296,0.0065650316,-0.06791897,0.07368851,0.056069646,-0.021026025,-0.011352883,0.038811427,-0.090847075,-0.056490287,-0.039881036,0.032885592,-0.029586185,0.04753608,0.006341778,-0.0034275919,0.03681623,0.09326265,-0.0532089,0.032396164,0.012605483,0.045778517,-0.0006472902,0.029721813,0.0734947,-0.037117425,-0.09356724,-0.0071634455,-0.032400057,0.010453378,0.06286513,0.006053158,0.060524322,0.091866635,0.063640274,0.055919155,0.053222507,-0.019174587,0.089296214,-0.058922265,0.013745364,-0.022907477,0.028347451,-0.040759467,-0.06820364,-0.061398294,-0.046857864,0.0060627847,0.010442015,-0.07662317,0.031950947,-0.107813306,-0.0016125626,-0.033413693,-0.09989722,-0.03580293,-0.036118146,-0.08276185,0.1280713,0.07596603,0.049225595,-0.026312606,0.0031930343,-0.0012469466,0.000872793,-0.040353082,-0.034516506,-0.036047976,0.004999689,-0.01570079,-0.09783203,1.4186912e-33,-0.0130903,0.04742377,0.02889633,0.09809088,0.043639127,-0.0747531,0.03402747,0.029147558,0.019848268,0.111865185,0.04845458,-0.016222177,0.025501896,0.09813583,-0.002833195,0.0038929228,0.048537854,0.014111116,0.036570966,-0.017512968,-0.079755045,0.043452248,0.10843892,0.061968718,-0.015678233,0.0060376576,0.010718569,-0.03907552,0.040307663,0.04553227,0.041731495,0.083179526,-0.0844024,-0.023447752,0.014485363,-0.00788756,-0.11410207,0.039309476,0.011220049,0.12889338,-0.07970701,0.010658111,-0.06657531,-0.022037188,-0.043459363,-0.030291978,0.05803613,0.06109021,-0.09415837,0.030727232,0.011858702,0.013526454,-0.049580045,-0.052486703,0.013218615,0.0050231596,0.07660412,-0.0046343356,-0.0692584,-0.03353194,0.022288544,0.022836983,0.01361394,-0.036331616,0.008194689,0.09226991,0.08229875,-0.047639843,-0.005033471,0.0066081914,-0.106666416,-0.06385503,0.006385333,0.06318358,0.03146985,-0.060764216,-0.043287244,-0.042385958,-0.07963737,-0.043478705,-0.031840656,0.060027804,0.027829666,0.03728562,-0.07014622,0.0649568,-0.052579004,-0.0034938955,-0.15390792,-0.04426162,0.028217599,0.05593145,0.023251446,0.029115746,-0.040321343,-1.8832651e-08,-0.037536032,0.0005056563,0.013919978,-0.038129315,0.061486673,0.005534046,-0.010768211,-0.16527015,0.061414365,0.019963786,-0.019666849,-0.059065994,0.00083181454,-0.016695915,0.01090875,0.02685501,-0.0008743374,-0.060967337,-0.049258173,0.012731965,-0.03973743,-0.0018210232,-0.03797001,0.00035104176,0.04477184,0.04185603,-0.03215796,-0.09103283,0.0467481,0.017810425,0.037437033,0.054782324,-0.0059442874,0.022557084,0.053393807,-0.03746794,-0.07080435,0.10548834,-0.04215435,0.042547524,0.05064807,0.07560324,0.04072184,-0.0011278372,0.00702018,-0.070807114,0.040035766,0.051417697,0.011316928,-0.028955372,0.04197099,-0.0026867494,-0.056857754,0.0761941,0.030621601,-0.052413527,-0.021492323,0.0957355,0.1013111,-0.082977295,-0.036080915,-0.03327806,0.0029404552,-0.03586191]'::vector)", "Filter": "(cohort = 'static'::text)" } ] } } ] } }, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "eligible": true, "protocol": "mixed-mutations-v2" }, "examples": [ { "event": 0, "operation": "insert", "doc_id": "mutation-000000", "scheduled_s": 0.0, "submitted_s": 0.00031441706232726574, "schedule_delay_ms": 0.31441706232726574, "precondition": true, "status": "success", "ack_ms": 24.474791018292308, "write_started_ms": 1.218290999531746, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 27.072957949712873, "stable_since_ms": 27.072957949712873, "confirmed_ms": 78.55712494347245, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 37.15975000523031, "stable_since_ms": 37.15975000523031, "confirmed_ms": 93.73454097658396, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 37.15975000523031, "confirmed_ms": 93.73454097658396, "search_minus_ack_ms": 12.684958986938 }, { "event": 1, "operation": "update", "doc_id": "mutation-000001", "scheduled_s": 0.025, "submitted_s": 0.025411167065612972, "schedule_delay_ms": 0.4111670656129704, "precondition": true, "status": "success", "ack_ms": 6.210707942955196, "write_started_ms": 0.12574996799230576, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 11.630082968622446, "stable_since_ms": 11.630082968622446, "confirmed_ms": 63.366750022396445, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 17.40537502337247, "stable_since_ms": 17.40537502337247, "confirmed_ms": 73.37520795408636, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 17.40537502337247, "confirmed_ms": 73.37520795408636, "search_minus_ack_ms": 11.194667080417275 }, { "event": 2, "operation": "delete", "doc_id": "mutation-000002", "scheduled_s": 0.05, "submitted_s": 0.05023704201448709, "schedule_delay_ms": 0.23704201448708495, "precondition": true, "status": "success", "ack_ms": 0.9110000682994723, "write_started_ms": 0.07362500764429569, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 11.34958304464817, "stable_since_ms": 11.34958304464817, "confirmed_ms": 62.96516605652869, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 18.599583068862557, "stable_since_ms": 18.599583068862557, "confirmed_ms": 74.66958300210536, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 18.599583068862557, "confirmed_ms": 74.66958300210536, "search_minus_ack_ms": 17.688583000563085 } ] }, { "run": "fb2_429e9a_011", "trial": 5, "backend": "qdrant", "scenario": "nq-10k", "status": "complete", "eligible": true, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "summary": { "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "wall_s": 30.059059915947728, "operations": { "insert": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 14.634291990660131, "p50": 67.29954149341211, "p95": 70.91808158438653, "p99": 73.34605470183324, "max": 108.05512510705739 }, "confirmed_ms": { "min": 107.91858297307044, "p50": 213.29093701206148, "p95": 218.39708117768168, "p99": 220.80784937716106, "max": 253.86725005228072 }, "ack_ms": { "min": 3.750207950361073, "p50": 10.142458020709455, "p95": 12.477983656572178, "p99": 14.979392114328236, "max": 17.963457969017327 }, "search_minus_ack_ms": { "min": 0.6378750549629331, "p50": 57.260458008386195, "p95": 59.975982626201585, "p99": 61.69850300531855, "max": 101.18741705082357 }, "regressions": 0, "confirmed_within_100ms": 0.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "update": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 17.722500022500753, "p50": 66.77264551399276, "p95": 70.77660204377025, "p99": 73.38626718265017, "max": 96.79324994795024 }, "confirmed_ms": { "min": 111.84858297929168, "p50": 212.42054150206968, "p95": 219.40074206795543, "p99": 223.43874502461412, "max": 241.63366691209376 }, "ack_ms": { "min": 5.802833940833807, "p50": 9.622062556445599, "p95": 12.284681212622672, "p99": 17.39319673972216, "max": 39.877458941191435 }, "search_minus_ack_ms": { "min": 1.5766670694574714, "p50": 56.96787452325225, "p95": 60.258871188852936, "p99": 60.67529768566601, "max": 60.91108394321054 }, "regressions": 0, "confirmed_within_100ms": 0.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "delete": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 14.51558293774724, "p50": 18.454895995091647, "p95": 23.390629212371934, "p99": 66.23961411300115, "max": 66.25091598834842 }, "confirmed_ms": { "min": 105.11916596442461, "p50": 153.11174996895716, "p95": 158.10969201265834, "p99": 211.70892408699729, "max": 212.63774996623397 }, "ack_ms": { "min": 3.4867500653490424, "p50": 8.214812551159412, "p95": 10.271116625517607, "p99": 10.955965634202585, "max": 10.997999925166368 }, "search_minus_ack_ms": { "min": -2.4443339789286256, "p50": 0.1915625180117786, "p95": 5.258672346826537, "p99": 57.32017608825118, "max": 58.150208089500666 }, "regressions": 0, "confirmed_within_100ms": 0.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 } }, "total_operations": 300, "statuses": { "success": 300, "timeout": 0, "error": 0, "invalid_precondition": 0, "client_overload": 0 }, "query": { "count": 300, "success": 300, "ann_recall_at_10": 0.999, "ndcg_at_10": 0.948294881680092, "latency_ms": { "min": 4.100749967619777, "p50": 8.537729503586888, "p95": 11.17367711267434, "p99": 47.57016439340077, "max": 51.69254192151129 }, "dispatch_delay_ms": { "min": 0.2556659746915102, "p50": 1.478124933782965, "p95": 2.256224979646504, "p99": 2.5140692666172972, "max": 3.0430409824475646 } }, "diagnostics": { "schedule_delay_ms": { "min": 0.010124966502544908, "p50": 0.37589545827376725, "p95": 0.7360312668606817, "p99": 0.7701063640416248, "max": 0.9056660113855308 }, "probe_dispatch_ms": { "min": 0.022666994482278824, "p50": 0.4636875237338245, "p95": 1.7668420332483945, "p99": 2.5539988337550312, "max": 5.610083928331733 }, "probe_service_ms": { "min": 0.9779159445315599, "p50": 6.324229529127479, "p95": 48.45942510291934, "p99": 49.774126700358465, "max": 55.919500067830086 }, "write_service_ms": { "min": 3.281707991845906, "p50": 8.600770554039627, "p95": 11.309652484487742, "p99": 14.234947285149225, "max": 39.35933404136449 } }, "actual_submission_rate": 9.99997109808376, "metadata": { "backend": "qdrant", "collection": "fb2_429e9a_011", "dataset": { "dataset": "nq", "source": "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/nq.zip", "source_sha256": "2553bf7bfbab47b1436ca00a34bce57320e18e611fd00999a2a3a1b4714be752", "source_md5": "d4d3d2e48787a744b6f6e691ff534307", "records": 124000, "queries": 100, "relevant_documents_retained": 120, "seed": 20261028, "license": "CC-BY-SA-3.0", "license_source": "https://raw.githubusercontent.com/google-research-datasets/natural-questions/master/nq_open/README.md", "attribution": "BEIR and original dataset authors; Wikipedia contributors for NQ, SciFact authors and cited article authors for SciFact.", "transformation": "200-word title+text prefix; seeded reservoir of documents with relevant documents for selected test queries retained at front. Simulated mutations; no historical replay.", "model": "sentence-transformers/all-MiniLM-L6-v2", "model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "dimensions": 384, "normalized": true, "max_tokens": 256, "device": "mps", "files": { "corpus.jsonl": "9dc487d785e6ee493eccaf764daa3bf8720a8f4fc8c406246065e1ad33a91f5d", "queries.json": "5603f52876ada5d36118b65dea490f1d43900bc159e1804e637796eb9115e999", "vectors.npy": "cb7421eb6ca5964231403f0a922843088f6fd0e01cbf67f2a1076cc7101005db", "query_vectors.npy": "5e666f8d966d8db6b0fd93d4963234d2dffa0ee3910b8fad069a8ee201e8b5d4", "UPSTREAM-LICENSE.md": "8b675521da3fa142ac6bda38c9bcfcc8cff0e2d6128be53a062ed55fb53f7b87" } }, "base_count": 10000, "profile": { "dataset": "nq", "base_count": 10000, "rate": 10, "query_rate": 10, "arrival": "steady" }, "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "preloaded_mutable": 200, "started_utc": "2026-10-29T02:54:56Z", "setup_s": 3.3881729160202667, "backend_before": { "version": "1.20.0", "collection": { "status": "green", "optimizer_status": "ok", "indexed_vectors_count": 10200, "points_count": 10200, "segments_count": 2, "config": { "params": { "vectors": { "size": 384, "distance": "Cosine" }, "shard_number": 1, "replication_factor": 1, "write_consistency_factor": 1, "on_disk_payload": true }, "hnsw_config": { "m": 16, "ef_construct": 128, "full_scan_threshold": 10, "max_indexing_threads": 0, "on_disk": false }, "optimizer_config": { "deleted_threshold": 0.2, "vacuum_min_vector_number": 1000, "default_segment_number": 0, "max_segment_size": null, "memmap_threshold": null, "indexing_threshold": 1024, "flush_interval_sec": 5, "max_optimization_threads": null, "prevent_unoptimized": null }, "wal_config": { "wal_capacity_mb": 32, "wal_segments_ahead": 0, "wal_retain_closed": 1 }, "quantization_config": null }, "payload_schema": { "cohort": { "data_type": "keyword", "points": 10200 } }, "update_queue": { "length": 0 }, "created_at": "2026-10-29T02:54:56.034Z" }, "ack": "wait=true", "transport": "HTTP REST, pooled httpx" }, "count_before": 10200, "idle_duration_s": 10.0, "idle_queries": { "count": 100, "ann_recall": 0.9990000000000001, "ndcg": 0.948294881680092, "latency_ms": { "min": 3.2859580824151635, "p50": 7.956354005727917, "p95": 9.580133890267462, "p99": 11.718323995592256, "max": 47.101708012633026 } }, "exact_ndcg": 0.9481505328688745, "resources_before": { "usage_usec": 73322874, "user_usec": 69789168, "system_usec": 3533705, "nice_usec": 57008000, "nr_periods": 1389, "nr_throttled": 83, "throttled_usec": 316106, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 361041920, "pids": 31 }, "resources_after": { "usage_usec": 79441344, "user_usec": 75155028, "system_usec": 4286316, "nice_usec": 57008000, "nr_periods": 1691, "nr_throttled": 83, "throttled_usec": 316106, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 347176960, "pids": 33 }, "count_after": 10200, "backend_after": { "version": "1.20.0", "collection": { "status": "green", "optimizer_status": "ok", "indexed_vectors_count": 10200, "points_count": 10200, "segments_count": 2, "config": { "params": { "vectors": { "size": 384, "distance": "Cosine" }, "shard_number": 1, "replication_factor": 1, "write_consistency_factor": 1, "on_disk_payload": true }, "hnsw_config": { "m": 16, "ef_construct": 128, "full_scan_threshold": 10, "max_indexing_threads": 0, "on_disk": false }, "optimizer_config": { "deleted_threshold": 0.2, "vacuum_min_vector_number": 1000, "default_segment_number": 0, "max_segment_size": null, "memmap_threshold": null, "indexing_threshold": 1024, "flush_interval_sec": 5, "max_optimization_threads": null, "prevent_unoptimized": null }, "wal_config": { "wal_capacity_mb": 32, "wal_segments_ahead": 0, "wal_retain_closed": 1 }, "quantization_config": null }, "payload_schema": { "cohort": { "data_type": "keyword", "points": 10200 } }, "update_queue": { "length": 0 }, "created_at": "2026-10-29T02:54:56.034Z" }, "ack": "wait=true", "transport": "HTTP REST, pooled httpx" } }, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "eligible": true, "protocol": "mixed-mutations-v2" }, "examples": [ { "event": 0, "operation": "insert", "doc_id": "mutation-000000", "scheduled_s": 0.0, "submitted_s": 0.00020733301062136889, "schedule_delay_ms": 0.20733301062136889, "precondition": true, "status": "success", "ack_ms": 6.2578750075772405, "write_started_ms": 1.2395830126479268, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 14.85604199115187, "stable_since_ms": 14.85604199115187, "confirmed_ms": 69.72662499174476, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 59.76333294529468, "stable_since_ms": 59.76333294529468, "confirmed_ms": 203.4578329185024, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 59.76333294529468, "confirmed_ms": 203.4578329185024, "search_minus_ack_ms": 53.50545793771744 }, { "event": 1, "operation": "update", "doc_id": "mutation-000001", "scheduled_s": 0.1, "submitted_s": 0.10047016595490277, "schedule_delay_ms": 0.4701659549027626, "precondition": true, "status": "success", "ack_ms": 7.95137498062104, "write_started_ms": 0.8476250804960728, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 17.607292043976486, "stable_since_ms": 17.607292043976486, "confirmed_ms": 73.98183399345726, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 61.35358405299485, "stable_since_ms": 61.35358405299485, "confirmed_ms": 208.42216699384153, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 61.35358405299485, "confirmed_ms": 208.42216699384153, "search_minus_ack_ms": 53.40220907237381 }, { "event": 2, "operation": "delete", "doc_id": "mutation-000002", "scheduled_s": 0.2, "submitted_s": 0.20032175001688302, "schedule_delay_ms": 0.32175001688300453, "precondition": true, "status": "success", "ack_ms": 9.48499992955476, "write_started_ms": 1.362374983727932, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 19.85524990595877, "stable_since_ms": 19.85524990595877, "confirmed_ms": 77.83441594801843, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 50.01574999187142, "stable_since_ms": 50.01574999187142, "confirmed_ms": 153.9020409109071, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 50.01574999187142, "confirmed_ms": 153.9020409109071, "search_minus_ack_ms": 40.530750062316656 } ] }, { "run": "fb2_429e9a_012", "trial": 3, "backend": "pgvector", "scenario": "nq-10k", "status": "complete", "eligible": true, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "summary": { "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "wall_s": 30.001199833932333, "operations": { "insert": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 17.35070801805705, "p50": 23.051333555486053, "p95": 25.75722665642388, "p99": 27.329397477442413, "max": 29.992250027135015 }, "confirmed_ms": { "min": 72.85204203799367, "p50": 87.78739598346874, "p95": 91.60564132616855, "p99": 93.1711075350177, "max": 101.9927499582991 }, "ack_ms": { "min": 4.5218749437481165, "p50": 10.192478948738426, "p95": 12.161341676255688, "p99": 13.077033367007996, "max": 17.78283400926739 }, "search_minus_ack_ms": { "min": 9.239334031008184, "p50": 12.725125008728355, "p95": 14.712968067033216, "p99": 15.747391667682683, "max": 16.78854099009186 }, "regressions": 0, "confirmed_within_100ms": 0.99, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "update": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 15.799834043718874, "p50": 23.350354516878724, "p95": 26.040380902122706, "p99": 27.065694099292163, "max": 27.62525004800409 }, "confirmed_ms": { "min": 71.18687499314547, "p50": 87.43262552889064, "p95": 90.79717673012055, "p99": 92.28061478585005, "max": 93.60779100097716 }, "ack_ms": { "min": 5.433833925053477, "p50": 9.963770513422787, "p95": 11.9587104243692, "p99": 13.70202245539987, "max": 16.51337498333305 }, "search_minus_ack_ms": { "min": 10.255208006128669, "p50": 13.206291943788528, "p95": 15.276044484926388, "p99": 16.647048975573853, "max": 16.903458046726882 }, "regressions": 0, "confirmed_within_100ms": 1.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "delete": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 11.316874995827675, "p50": 13.793354504741728, "p95": 23.81669223541394, "p99": 24.559648175491024, "max": 25.201290962286294 }, "confirmed_ms": { "min": 63.85337491519749, "p50": 72.8718540049158, "p95": 89.6310298005119, "p99": 90.97800662391819, "max": 91.3622910156846 }, "ack_ms": { "min": 0.9345419239252806, "p50": 2.559270476922393, "p95": 4.073771490948275, "p99": 4.4286769325845, "max": 5.440250039100647 }, "search_minus_ack_ms": { "min": 1.68208300601691, "p50": 4.655228985939175, "p95": 20.807060453807935, "p99": 21.493420863989744, "max": 22.32708304654807 }, "regressions": 0, "confirmed_within_100ms": 1.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 } }, "total_operations": 300, "statuses": { "success": 300, "timeout": 0, "error": 0, "invalid_precondition": 0, "client_overload": 0 }, "query": { "count": 300, "success": 300, "ann_recall_at_10": 1.0, "ndcg_at_10": 0.9481505328688746, "latency_ms": { "min": 2.327832975424826, "p50": 5.987541982904077, "p95": 7.6106333115603775, "p99": 9.510383006418122, "max": 14.027750003151596 }, "dispatch_delay_ms": { "min": 0.12429198250174522, "p50": 1.1199169093742967, "p95": 2.0093312312383205, "p99": 2.2806243342347443, "max": 5.518374964594841 } }, "diagnostics": { "schedule_delay_ms": { "min": 0.009499909356947, "p50": 0.4062919528227127, "p95": 0.7202023873104313, "p99": 0.7432555656407434, "max": 1.3786669820543551 }, "probe_dispatch_ms": { "min": 0.023541972041130066, "p50": 0.4499999340623617, "p95": 1.5504169277846813, "p99": 2.3409585934132333, "max": 2.613167045637965 }, "probe_service_ms": { "min": 0.39120810106396675, "p50": 2.7280839858576655, "p95": 7.53545796032995, "p99": 8.39170820545405, "max": 21.949166897684336 }, "write_service_ms": { "min": 0.7782089523971081, "p50": 8.742375008296221, "p95": 11.460893513867632, "p99": 12.690402544103558, "max": 16.62829192355275 } }, "actual_submission_rate": 9.999941513704877, "metadata": { "backend": "pgvector", "collection": "fb2_429e9a_012", "dataset": { "dataset": "nq", "source": "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/nq.zip", "source_sha256": "2553bf7bfbab47b1436ca00a34bce57320e18e611fd00999a2a3a1b4714be752", "source_md5": "d4d3d2e48787a744b6f6e691ff534307", "records": 124000, "queries": 100, "relevant_documents_retained": 120, "seed": 20261028, "license": "CC-BY-SA-3.0", "license_source": "https://raw.githubusercontent.com/google-research-datasets/natural-questions/master/nq_open/README.md", "attribution": "BEIR and original dataset authors; Wikipedia contributors for NQ, SciFact authors and cited article authors for SciFact.", "transformation": "200-word title+text prefix; seeded reservoir of documents with relevant documents for selected test queries retained at front. Simulated mutations; no historical replay.", "model": "sentence-transformers/all-MiniLM-L6-v2", "model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "dimensions": 384, "normalized": true, "max_tokens": 256, "device": "mps", "files": { "corpus.jsonl": "9dc487d785e6ee493eccaf764daa3bf8720a8f4fc8c406246065e1ad33a91f5d", "queries.json": "5603f52876ada5d36118b65dea490f1d43900bc159e1804e637796eb9115e999", "vectors.npy": "cb7421eb6ca5964231403f0a922843088f6fd0e01cbf67f2a1076cc7101005db", "query_vectors.npy": "5e666f8d966d8db6b0fd93d4963234d2dffa0ee3910b8fad069a8ee201e8b5d4", "UPSTREAM-LICENSE.md": "8b675521da3fa142ac6bda38c9bcfcc8cff0e2d6128be53a062ed55fb53f7b87" } }, "base_count": 10000, "profile": { "dataset": "nq", "base_count": 10000, "rate": 10, "query_rate": 10, "arrival": "steady" }, "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "preloaded_mutable": 200, "started_utc": "2026-10-29T02:55:45Z", "setup_s": 3.5895684169372544, "backend_before": { "version": "0.8.7", "postgres": "PostgreSQL 16.15 (Debian 16.15-1.pgdg12+2) on aarch64-unknown-linux-gnu, compiled by gcc (Debian 12.2.0-14+deb12u1) 12.2.0, 64-bit", "ef_search": 128, "ack": "autocommit, READ COMMITTED", "transport": "psycopg, connection per worker", "query_plan": [ { "Plan": { "Node Type": "Limit", "Parallel Aware": false, "Async Capable": false, "Startup Cost": 1340.05, "Total Cost": 1356.3, "Plan Rows": 10, "Plan Width": 18, "Plans": [ { "Node Type": "Index Scan", "Parent Relationship": "Outer", "Parallel Aware": false, "Async Capable": false, "Scan Direction": "Forward", "Index Name": "fb2_429e9a_012_hnsw", "Relation Name": "fb2_429e9a_012", "Alias": "fb2_429e9a_012", "Startup Cost": 1340.05, "Total Cost": 17585.0, "Plan Rows": 10000, "Plan Width": 18, "Order By": "(embedding <=> '[-0.08946895,-0.094362676,0.092023626,-0.0040844884,0.043233823,0.00062519487,0.1075197,-0.035732996,0.013746996,-0.024103122,0.0021706754,-0.04720602,0.04787615,-0.02043222,0.058315646,0.081252135,-0.05375015,-0.030799072,-0.0077205584,0.035735283,-0.04474784,0.01614922,-0.06905586,0.04194401,-0.08832908,0.007873556,0.0044992934,0.021968655,0.010953364,0.07212316,0.023185596,0.001176476,0.04725166,-0.03580423,-0.032344446,0.004752835,-0.056179125,-0.0353235,-0.037251893,-0.005143068,-0.027599342,0.024403147,0.031338032,0.0056410627,0.008425455,-0.0087741045,-0.039870497,-0.011831836,0.042621233,0.048142947,0.01603842,0.06745542,-0.038018145,0.064212486,-0.009636566,0.01607475,-0.0027868534,-0.03761097,-0.011889431,-0.01084577,0.083403684,0.028946918,0.061870236,-0.045382615,0.03812586,0.060426526,-0.017003024,0.02328925,-0.093020774,-0.024072766,0.042720392,-0.022444837,-0.011597358,0.01209231,-0.005953629,0.029691145,-0.011711495,-0.053345133,-0.081757374,-0.049303155,0.032193717,-0.038507868,-0.0060101775,-0.031470332,-0.07362443,0.03575578,0.0245484,0.01985519,-0.15116754,0.050386053,-0.0389634,0.07548389,-0.09264168,-0.012602816,-0.0035539444,-0.046416666,0.015100493,0.100691915,-0.037529,0.04600518,-0.021859422,0.014675989,-0.097722545,-0.053200893,-0.017040461,-0.01629616,-0.012339222,0.06543302,0.07252074,-0.044301156,0.03054835,-0.038570553,0.019310065,-0.005527747,-0.029386865,-0.058248505,0.065443166,0.028506057,0.0878254,0.058387812,0.081721246,-0.0005962058,0.0569388,0.00960514,0.011667701,0.014456492,-0.07218306,-3.0444767e-33,0.06687367,-0.019213954,0.058202963,0.025171444,-0.028259544,-0.04171671,0.03127722,0.04865431,-0.0721459,-0.050143186,-0.029907746,0.06248994,-0.05096076,-0.065192886,0.010210447,-0.077256374,-0.043370593,-0.0901922,-0.066522665,0.026072454,-0.022315364,0.075375885,-0.029968103,-0.0050771856,-0.0046955296,0.0065650316,-0.06791897,0.07368851,0.056069646,-0.021026025,-0.011352883,0.038811427,-0.090847075,-0.056490287,-0.039881036,0.032885592,-0.029586185,0.04753608,0.006341778,-0.0034275919,0.03681623,0.09326265,-0.0532089,0.032396164,0.012605483,0.045778517,-0.0006472902,0.029721813,0.0734947,-0.037117425,-0.09356724,-0.0071634455,-0.032400057,0.010453378,0.06286513,0.006053158,0.060524322,0.091866635,0.063640274,0.055919155,0.053222507,-0.019174587,0.089296214,-0.058922265,0.013745364,-0.022907477,0.028347451,-0.040759467,-0.06820364,-0.061398294,-0.046857864,0.0060627847,0.010442015,-0.07662317,0.031950947,-0.107813306,-0.0016125626,-0.033413693,-0.09989722,-0.03580293,-0.036118146,-0.08276185,0.1280713,0.07596603,0.049225595,-0.026312606,0.0031930343,-0.0012469466,0.000872793,-0.040353082,-0.034516506,-0.036047976,0.004999689,-0.01570079,-0.09783203,1.4186912e-33,-0.0130903,0.04742377,0.02889633,0.09809088,0.043639127,-0.0747531,0.03402747,0.029147558,0.019848268,0.111865185,0.04845458,-0.016222177,0.025501896,0.09813583,-0.002833195,0.0038929228,0.048537854,0.014111116,0.036570966,-0.017512968,-0.079755045,0.043452248,0.10843892,0.061968718,-0.015678233,0.0060376576,0.010718569,-0.03907552,0.040307663,0.04553227,0.041731495,0.083179526,-0.0844024,-0.023447752,0.014485363,-0.00788756,-0.11410207,0.039309476,0.011220049,0.12889338,-0.07970701,0.010658111,-0.06657531,-0.022037188,-0.043459363,-0.030291978,0.05803613,0.06109021,-0.09415837,0.030727232,0.011858702,0.013526454,-0.049580045,-0.052486703,0.013218615,0.0050231596,0.07660412,-0.0046343356,-0.0692584,-0.03353194,0.022288544,0.022836983,0.01361394,-0.036331616,0.008194689,0.09226991,0.08229875,-0.047639843,-0.005033471,0.0066081914,-0.106666416,-0.06385503,0.006385333,0.06318358,0.03146985,-0.060764216,-0.043287244,-0.042385958,-0.07963737,-0.043478705,-0.031840656,0.060027804,0.027829666,0.03728562,-0.07014622,0.0649568,-0.052579004,-0.0034938955,-0.15390792,-0.04426162,0.028217599,0.05593145,0.023251446,0.029115746,-0.040321343,-1.8832651e-08,-0.037536032,0.0005056563,0.013919978,-0.038129315,0.061486673,0.005534046,-0.010768211,-0.16527015,0.061414365,0.019963786,-0.019666849,-0.059065994,0.00083181454,-0.016695915,0.01090875,0.02685501,-0.0008743374,-0.060967337,-0.049258173,0.012731965,-0.03973743,-0.0018210232,-0.03797001,0.00035104176,0.04477184,0.04185603,-0.03215796,-0.09103283,0.0467481,0.017810425,0.037437033,0.054782324,-0.0059442874,0.022557084,0.053393807,-0.03746794,-0.07080435,0.10548834,-0.04215435,0.042547524,0.05064807,0.07560324,0.04072184,-0.0011278372,0.00702018,-0.070807114,0.040035766,0.051417697,0.011316928,-0.028955372,0.04197099,-0.0026867494,-0.056857754,0.0761941,0.030621601,-0.052413527,-0.021492323,0.0957355,0.1013111,-0.082977295,-0.036080915,-0.03327806,0.0029404552,-0.03586191]'::vector)", "Filter": "(cohort = 'static'::text)" } ] } } ] }, "count_before": 10200, "idle_duration_s": 10.0, "idle_queries": { "count": 100, "ann_recall": 1.0, "ndcg": 0.9481505328688745, "latency_ms": { "min": 3.593832952901721, "p50": 7.784874993376434, "p95": 9.275098296348006, "p99": 10.426114610163538, "max": 18.66683305706829 } }, "exact_ndcg": 0.9481505328688745, "resources_before": { "usage_usec": 217310197, "user_usec": 202655321, "system_usec": 14654875, "nice_usec": 0, "nr_periods": 2829, "nr_throttled": 854, "throttled_usec": 90183406, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 1053270016, "pids": 23 }, "resources_after": { "usage_usec": 224935215, "user_usec": 209796729, "system_usec": 15138486, "nice_usec": 0, "nr_periods": 3130, "nr_throttled": 854, "throttled_usec": 90183406, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 1066266624, "pids": 28 }, "count_after": 10200, "backend_after": { "version": "0.8.7", "postgres": "PostgreSQL 16.15 (Debian 16.15-1.pgdg12+2) on aarch64-unknown-linux-gnu, compiled by gcc (Debian 12.2.0-14+deb12u1) 12.2.0, 64-bit", "ef_search": 128, "ack": "autocommit, READ COMMITTED", "transport": "psycopg, connection per worker", "query_plan": [ { "Plan": { "Node Type": "Limit", "Parallel Aware": false, "Async Capable": false, "Startup Cost": 1350.72, "Total Cost": 1367.04, "Plan Rows": 10, "Plan Width": 18, "Plans": [ { "Node Type": "Index Scan", "Parent Relationship": "Outer", "Parallel Aware": false, "Async Capable": false, "Scan Direction": "Forward", "Index Name": "fb2_429e9a_012_hnsw", "Relation Name": "fb2_429e9a_012", "Alias": "fb2_429e9a_012", "Startup Cost": 1350.72, "Total Cost": 17879.94, "Plan Rows": 10128, "Plan Width": 18, "Order By": "(embedding <=> '[-0.08946895,-0.094362676,0.092023626,-0.0040844884,0.043233823,0.00062519487,0.1075197,-0.035732996,0.013746996,-0.024103122,0.0021706754,-0.04720602,0.04787615,-0.02043222,0.058315646,0.081252135,-0.05375015,-0.030799072,-0.0077205584,0.035735283,-0.04474784,0.01614922,-0.06905586,0.04194401,-0.08832908,0.007873556,0.0044992934,0.021968655,0.010953364,0.07212316,0.023185596,0.001176476,0.04725166,-0.03580423,-0.032344446,0.004752835,-0.056179125,-0.0353235,-0.037251893,-0.005143068,-0.027599342,0.024403147,0.031338032,0.0056410627,0.008425455,-0.0087741045,-0.039870497,-0.011831836,0.042621233,0.048142947,0.01603842,0.06745542,-0.038018145,0.064212486,-0.009636566,0.01607475,-0.0027868534,-0.03761097,-0.011889431,-0.01084577,0.083403684,0.028946918,0.061870236,-0.045382615,0.03812586,0.060426526,-0.017003024,0.02328925,-0.093020774,-0.024072766,0.042720392,-0.022444837,-0.011597358,0.01209231,-0.005953629,0.029691145,-0.011711495,-0.053345133,-0.081757374,-0.049303155,0.032193717,-0.038507868,-0.0060101775,-0.031470332,-0.07362443,0.03575578,0.0245484,0.01985519,-0.15116754,0.050386053,-0.0389634,0.07548389,-0.09264168,-0.012602816,-0.0035539444,-0.046416666,0.015100493,0.100691915,-0.037529,0.04600518,-0.021859422,0.014675989,-0.097722545,-0.053200893,-0.017040461,-0.01629616,-0.012339222,0.06543302,0.07252074,-0.044301156,0.03054835,-0.038570553,0.019310065,-0.005527747,-0.029386865,-0.058248505,0.065443166,0.028506057,0.0878254,0.058387812,0.081721246,-0.0005962058,0.0569388,0.00960514,0.011667701,0.014456492,-0.07218306,-3.0444767e-33,0.06687367,-0.019213954,0.058202963,0.025171444,-0.028259544,-0.04171671,0.03127722,0.04865431,-0.0721459,-0.050143186,-0.029907746,0.06248994,-0.05096076,-0.065192886,0.010210447,-0.077256374,-0.043370593,-0.0901922,-0.066522665,0.026072454,-0.022315364,0.075375885,-0.029968103,-0.0050771856,-0.0046955296,0.0065650316,-0.06791897,0.07368851,0.056069646,-0.021026025,-0.011352883,0.038811427,-0.090847075,-0.056490287,-0.039881036,0.032885592,-0.029586185,0.04753608,0.006341778,-0.0034275919,0.03681623,0.09326265,-0.0532089,0.032396164,0.012605483,0.045778517,-0.0006472902,0.029721813,0.0734947,-0.037117425,-0.09356724,-0.0071634455,-0.032400057,0.010453378,0.06286513,0.006053158,0.060524322,0.091866635,0.063640274,0.055919155,0.053222507,-0.019174587,0.089296214,-0.058922265,0.013745364,-0.022907477,0.028347451,-0.040759467,-0.06820364,-0.061398294,-0.046857864,0.0060627847,0.010442015,-0.07662317,0.031950947,-0.107813306,-0.0016125626,-0.033413693,-0.09989722,-0.03580293,-0.036118146,-0.08276185,0.1280713,0.07596603,0.049225595,-0.026312606,0.0031930343,-0.0012469466,0.000872793,-0.040353082,-0.034516506,-0.036047976,0.004999689,-0.01570079,-0.09783203,1.4186912e-33,-0.0130903,0.04742377,0.02889633,0.09809088,0.043639127,-0.0747531,0.03402747,0.029147558,0.019848268,0.111865185,0.04845458,-0.016222177,0.025501896,0.09813583,-0.002833195,0.0038929228,0.048537854,0.014111116,0.036570966,-0.017512968,-0.079755045,0.043452248,0.10843892,0.061968718,-0.015678233,0.0060376576,0.010718569,-0.03907552,0.040307663,0.04553227,0.041731495,0.083179526,-0.0844024,-0.023447752,0.014485363,-0.00788756,-0.11410207,0.039309476,0.011220049,0.12889338,-0.07970701,0.010658111,-0.06657531,-0.022037188,-0.043459363,-0.030291978,0.05803613,0.06109021,-0.09415837,0.030727232,0.011858702,0.013526454,-0.049580045,-0.052486703,0.013218615,0.0050231596,0.07660412,-0.0046343356,-0.0692584,-0.03353194,0.022288544,0.022836983,0.01361394,-0.036331616,0.008194689,0.09226991,0.08229875,-0.047639843,-0.005033471,0.0066081914,-0.106666416,-0.06385503,0.006385333,0.06318358,0.03146985,-0.060764216,-0.043287244,-0.042385958,-0.07963737,-0.043478705,-0.031840656,0.060027804,0.027829666,0.03728562,-0.07014622,0.0649568,-0.052579004,-0.0034938955,-0.15390792,-0.04426162,0.028217599,0.05593145,0.023251446,0.029115746,-0.040321343,-1.8832651e-08,-0.037536032,0.0005056563,0.013919978,-0.038129315,0.061486673,0.005534046,-0.010768211,-0.16527015,0.061414365,0.019963786,-0.019666849,-0.059065994,0.00083181454,-0.016695915,0.01090875,0.02685501,-0.0008743374,-0.060967337,-0.049258173,0.012731965,-0.03973743,-0.0018210232,-0.03797001,0.00035104176,0.04477184,0.04185603,-0.03215796,-0.09103283,0.0467481,0.017810425,0.037437033,0.054782324,-0.0059442874,0.022557084,0.053393807,-0.03746794,-0.07080435,0.10548834,-0.04215435,0.042547524,0.05064807,0.07560324,0.04072184,-0.0011278372,0.00702018,-0.070807114,0.040035766,0.051417697,0.011316928,-0.028955372,0.04197099,-0.0026867494,-0.056857754,0.0761941,0.030621601,-0.052413527,-0.021492323,0.0957355,0.1013111,-0.082977295,-0.036080915,-0.03327806,0.0029404552,-0.03586191]'::vector)", "Filter": "(cohort = 'static'::text)" } ] } } ] } }, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "eligible": true, "protocol": "mixed-mutations-v2" }, "examples": [ { "event": 0, "operation": "insert", "doc_id": "mutation-000000", "scheduled_s": 0.0, "submitted_s": 0.00025524990633130074, "schedule_delay_ms": 0.25524990633130074, "precondition": true, "status": "success", "ack_ms": 17.78283400926739, "write_started_ms": 1.1545420857146382, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 25.21329198498279, "stable_since_ms": 25.21329198498279, "confirmed_ms": 78.07637506630272, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 29.992250027135015, "stable_since_ms": 29.992250027135015, "confirmed_ms": 85.20433399826288, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 29.992250027135015, "confirmed_ms": 85.20433399826288, "search_minus_ack_ms": 12.209416017867625 }, { "event": 1, "operation": "update", "doc_id": "mutation-000001", "scheduled_s": 0.1, "submitted_s": 0.10052312491461635, "schedule_delay_ms": 0.5231249146163408, "precondition": true, "status": "success", "ack_ms": 5.5383340222761035, "write_started_ms": 0.1723750028759241, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 12.159749981947243, "stable_since_ms": 12.159749981947243, "confirmed_ms": 65.78716705553234, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 15.799834043718874, "stable_since_ms": 15.799834043718874, "confirmed_ms": 77.33879203442484, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 15.799834043718874, "confirmed_ms": 77.33879203442484, "search_minus_ack_ms": 10.261500021442771 }, { "event": 2, "operation": "delete", "doc_id": "mutation-000002", "scheduled_s": 0.2, "submitted_s": 0.20012574992142618, "schedule_delay_ms": 0.12574992142616592, "precondition": true, "status": "success", "ack_ms": 3.18100000731647, "write_started_ms": 0.1275839749723673, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 12.6062500057742, "stable_since_ms": 12.6062500057742, "confirmed_ms": 65.8297500340268, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 22.853209055028856, "stable_since_ms": 22.853209055028856, "confirmed_ms": 88.55362504255027, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 22.853209055028856, "confirmed_ms": 88.55362504255027, "search_minus_ack_ms": 19.672209047712386 } ] }, { "run": "fb2_429e9a_013", "trial": 2, "backend": "weaviate", "scenario": "nq-100k", "status": "complete", "eligible": true, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "summary": { "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "wall_s": 30.0015816250816, "operations": { "insert": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 7.706290925852954, "p50": 17.939707962796092, "p95": 35.34893480245955, "p99": 42.24187642452317, "max": 62.01320898253471 }, "confirmed_ms": { "min": 68.79958300851285, "p50": 86.32695849519223, "p95": 106.5389686904382, "p99": 113.96565034170642, "max": 128.26129200402647 }, "ack_ms": { "min": 5.128374905325472, "p50": 10.9619380091317, "p95": 17.436002125032246, "p99": 23.62466719467205, "max": 47.17024997808039 }, "search_minus_ack_ms": { "min": 0.11362496297806501, "p50": 4.396520496811718, "p95": 20.380889560328797, "p99": 21.624516152078293, "max": 22.731542005203664 }, "regressions": 0, "confirmed_within_100ms": 0.74, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "update": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 8.573916973546147, "p50": 17.88433297770098, "p95": 31.810292328009382, "p99": 33.68559091934004, "max": 35.33224994316697 }, "confirmed_ms": { "min": 68.58262501191348, "p50": 85.19170852378011, "p95": 104.41020692815073, "p99": 107.19204407185316, "max": 107.85633290652186 }, "ack_ms": { "min": 5.008499952964485, "p50": 10.608249984215945, "p95": 15.340222901431844, "p99": 15.810415614396337, "max": 16.511499998159707 }, "search_minus_ack_ms": { "min": -0.30641595367342234, "p50": 3.488083486445248, "p95": 19.863555987831205, "p99": 21.043486605631188, "max": 21.376291988417506 }, "regressions": 0, "confirmed_within_100ms": 0.89, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "delete": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 5.783207947388291, "p50": 15.9442710573785, "p95": 25.11313746799715, "p99": 33.318317959783606, "max": 35.11941700708121 }, "confirmed_ms": { "min": 68.68266593664885, "p50": 82.09120848914608, "p95": 94.06139792408794, "p99": 106.07852249522693, "max": 107.12845891248435 }, "ack_ms": { "min": 1.7909170128405094, "p50": 5.369416496250778, "p95": 11.280995520064605, "p99": 13.125633280724289, "max": 13.42345797456801 }, "search_minus_ack_ms": { "min": 0.1585839781910181, "p50": 5.872124980669469, "p95": 10.51199822686611, "p99": 20.851526326732717, "max": 21.99679205659777 }, "regressions": 0, "confirmed_within_100ms": 0.96, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 } }, "total_operations": 300, "statuses": { "success": 300, "timeout": 0, "error": 0, "invalid_precondition": 0, "client_overload": 0 }, "query": { "count": 300, "success": 300, "ann_recall_at_10": 0.991, "ndcg_at_10": 0.7931268675492565, "latency_ms": { "min": 3.7923339987173676, "p50": 11.365437414497137, "p95": 15.99512090906501, "p99": 21.70470589888281, "max": 41.724166949279606 }, "dispatch_delay_ms": { "min": 0.08200004231184721, "p50": 1.5132505795918405, "p95": 2.3537110071629286, "p99": 2.6653798611368984, "max": 3.2334589632228017 } }, "diagnostics": { "schedule_delay_ms": { "min": 0.016209040768266902, "p50": 0.39225057698866195, "p95": 0.7400233065702366, "p99": 0.7829936514612067, "max": 1.368750026449539 }, "probe_dispatch_ms": { "min": 0.028165988624095917, "p50": 0.48122898442670703, "p95": 1.9691562338266522, "p99": 2.676564775174483, "max": 3.2939170487225056 }, "probe_service_ms": { "min": 0.851999968290329, "p50": 5.582499958109111, "p95": 12.257635506102815, "p99": 14.785697759361925, "max": 41.704250033944845 }, "write_service_ms": { "min": 1.5052499948069453, "p50": 9.292207949329168, "p95": 14.622694393619895, "p99": 19.391944499220678, "max": 46.54312506318092 } }, "actual_submission_rate": 9.999820571808968, "metadata": { "backend": "weaviate", "collection": "fb2_429e9a_013", "dataset": { "dataset": "nq", "source": "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/nq.zip", "source_sha256": "2553bf7bfbab47b1436ca00a34bce57320e18e611fd00999a2a3a1b4714be752", "source_md5": "d4d3d2e48787a744b6f6e691ff534307", "records": 124000, "queries": 100, "relevant_documents_retained": 120, "seed": 20261028, "license": "CC-BY-SA-3.0", "license_source": "https://raw.githubusercontent.com/google-research-datasets/natural-questions/master/nq_open/README.md", "attribution": "BEIR and original dataset authors; Wikipedia contributors for NQ, SciFact authors and cited article authors for SciFact.", "transformation": "200-word title+text prefix; seeded reservoir of documents with relevant documents for selected test queries retained at front. Simulated mutations; no historical replay.", "model": "sentence-transformers/all-MiniLM-L6-v2", "model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "dimensions": 384, "normalized": true, "max_tokens": 256, "device": "mps", "files": { "corpus.jsonl": "9dc487d785e6ee493eccaf764daa3bf8720a8f4fc8c406246065e1ad33a91f5d", "queries.json": "5603f52876ada5d36118b65dea490f1d43900bc159e1804e637796eb9115e999", "vectors.npy": "cb7421eb6ca5964231403f0a922843088f6fd0e01cbf67f2a1076cc7101005db", "query_vectors.npy": "5e666f8d966d8db6b0fd93d4963234d2dffa0ee3910b8fad069a8ee201e8b5d4", "UPSTREAM-LICENSE.md": "8b675521da3fa142ac6bda38c9bcfcc8cff0e2d6128be53a062ed55fb53f7b87" } }, "base_count": 100000, "profile": { "dataset": "nq", "base_count": 100000, "rate": 10, "query_rate": 10, "arrival": "steady" }, "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "preloaded_mutable": 200, "started_utc": "2026-10-29T02:56:33Z", "setup_s": 37.24745983304456, "backend_before": { "version": "1.40.3", "schema": { "class": "Ffb2429e9a013", "invertedIndexConfig": { "bm25": { "b": 0.75, "k1": 1.2 }, "cleanupIntervalSeconds": 60, "stopwords": { "additions": null, "preset": "en", "removals": null }, "usingBlockMaxWAND": true }, "multiTenancyConfig": { "autoTenantActivation": false, "autoTenantCreation": false, "enabled": false }, "properties": [ { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "doc_id", "tokenization": "field" }, { "dataType": [ "int" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": false, "name": "version" }, { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "cohort", "tokenization": "field" }, { "dataType": [ "text" ], "indexFilterable": false, "indexRangeFilters": false, "indexSearchable": false, "name": "text", "tokenization": "word" } ], "shardingConfig": { "actualCount": 1, "actualVirtualCount": 128, "desiredCount": 1, "desiredVirtualCount": 128, "function": "murmur3", "key": "_id", "strategy": "hash", "virtualPerPhysical": 128 }, "vectorIndexConfig": { "bq": { "enabled": false }, "cleanupIntervalSeconds": 300, "distance": "cosine", "dynamicEfFactor": 8, "dynamicEfMax": 500, "dynamicEfMin": 100, "ef": 128, "efConstruction": 128, "filterStrategy": "acorn", "flatSearchCutoff": 0, "maxConnections": 16, "multivector": { "aggregation": "maxSim", "enabled": false, "muvera": { "dprojections": 16, "enabled": false, "ksim": 4, "repetitions": 10 } }, "pq": { "bitCompression": false, "centroids": 256, "enabled": false, "encoder": { "distribution": "log-normal", "type": "kmeans" }, "segments": 0, "trainingLimit": 100000 }, "rq": { "bits": 8, "centering": false, "enabled": false, "rescoreLimit": 20, "trainingLimit": 10000 }, "skip": false, "skipDefaultQuantization": false, "sq": { "enabled": false, "rescoreLimit": 20, "trainingLimit": 100000 }, "trackDefaultQuantization": false, "vectorCacheMaxObjects": 1000000000000 }, "vectorIndexType": "hnsw", "vectorizer": "none", "replicationConfig": { "deletionStrategy": "TimeBasedResolution", "factor": 1, "asyncEnabled": false } }, "shards": [ { "name": "MiTateffcqvD", "per_node_status": { "node1": "READY" }, "status": "READY", "vectorQueueSize": 0 } ], "ack": "synchronous indexing, object PUT/POST/DELETE", "transport": "HTTP REST writes + GraphQL search, pooled httpx" }, "count_before": 100200, "idle_duration_s": 10.0, "idle_queries": { "count": 100, "ann_recall": 0.991, "ndcg": 0.7931268675492565, "latency_ms": { "min": 4.032582975924015, "p50": 11.73387496965006, "p95": 13.481597660575062, "p99": 17.286503532668583, "max": 18.454166944138706 } }, "exact_ndcg": 0.7931268675492565, "resources_before": { "usage_usec": 160131319, "user_usec": 156993444, "system_usec": 3137874, "nice_usec": 0, "nr_periods": 3699, "nr_throttled": 242, "throttled_usec": 1084957, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 1239240704, "pids": 10 }, "resources_after": { "usage_usec": 170081911, "user_usec": 166486501, "system_usec": 3595409, "nice_usec": 0, "nr_periods": 4000, "nr_throttled": 242, "throttled_usec": 1084957, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 1141141504, "pids": 10 }, "count_after": 100200, "backend_after": { "version": "1.40.3", "schema": { "class": "Ffb2429e9a013", "invertedIndexConfig": { "bm25": { "b": 0.75, "k1": 1.2 }, "cleanupIntervalSeconds": 60, "stopwords": { "additions": null, "preset": "en", "removals": null }, "usingBlockMaxWAND": true }, "multiTenancyConfig": { "autoTenantActivation": false, "autoTenantCreation": false, "enabled": false }, "properties": [ { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "doc_id", "tokenization": "field" }, { "dataType": [ "int" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": false, "name": "version" }, { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "cohort", "tokenization": "field" }, { "dataType": [ "text" ], "indexFilterable": false, "indexRangeFilters": false, "indexSearchable": false, "name": "text", "tokenization": "word" } ], "shardingConfig": { "actualCount": 1, "actualVirtualCount": 128, "desiredCount": 1, "desiredVirtualCount": 128, "function": "murmur3", "key": "_id", "strategy": "hash", "virtualPerPhysical": 128 }, "vectorIndexConfig": { "bq": { "enabled": false }, "cleanupIntervalSeconds": 300, "distance": "cosine", "dynamicEfFactor": 8, "dynamicEfMax": 500, "dynamicEfMin": 100, "ef": 128, "efConstruction": 128, "filterStrategy": "acorn", "flatSearchCutoff": 0, "maxConnections": 16, "multivector": { "aggregation": "maxSim", "enabled": false, "muvera": { "dprojections": 16, "enabled": false, "ksim": 4, "repetitions": 10 } }, "pq": { "bitCompression": false, "centroids": 256, "enabled": false, "encoder": { "distribution": "log-normal", "type": "kmeans" }, "segments": 0, "trainingLimit": 100000 }, "rq": { "bits": 8, "centering": false, "enabled": false, "rescoreLimit": 20, "trainingLimit": 10000 }, "skip": false, "skipDefaultQuantization": false, "sq": { "enabled": false, "rescoreLimit": 20, "trainingLimit": 100000 }, "trackDefaultQuantization": false, "vectorCacheMaxObjects": 1000000000000 }, "vectorIndexType": "hnsw", "vectorizer": "none", "replicationConfig": { "deletionStrategy": "TimeBasedResolution", "factor": 1, "asyncEnabled": false } }, "shards": [ { "name": "MiTateffcqvD", "per_node_status": { "node1": "READY" }, "status": "READY", "vectorQueueSize": 0 } ], "ack": "synchronous indexing, object PUT/POST/DELETE", "transport": "HTTP REST writes + GraphQL search, pooled httpx" } }, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "eligible": true, "protocol": "mixed-mutations-v2" }, "examples": [ { "event": 0, "operation": "insert", "doc_id": "mutation-000000", "scheduled_s": 0.0, "submitted_s": 0.00022591708693653345, "schedule_delay_ms": 0.22591708693653345, "precondition": true, "status": "success", "ack_ms": 8.962082909420133, "write_started_ms": 1.686332980170846, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 14.830791973508894, "stable_since_ms": 14.830791973508894, "confirmed_ms": 68.71187500655651, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 23.76062492839992, "stable_since_ms": 23.76062492839992, "confirmed_ms": 84.51091696042567, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 23.76062492839992, "confirmed_ms": 84.51091696042567, "search_minus_ack_ms": 14.798542018979788 }, { "event": 1, "operation": "update", "doc_id": "mutation-000001", "scheduled_s": 0.1, "submitted_s": 0.10056908405385911, "schedule_delay_ms": 0.5690840538591091, "precondition": true, "status": "success", "ack_ms": 7.693790947087109, "write_started_ms": 0.656957970932126, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 16.294040950015187, "stable_since_ms": 16.294040950015187, "confirmed_ms": 72.72950001060963, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 10.778457974083722, "stable_since_ms": 10.778457974083722, "confirmed_ms": 77.92633294593543, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 16.294040950015187, "confirmed_ms": 77.92633294593543, "search_minus_ack_ms": 3.0846670269966125 }, { "event": 2, "operation": "delete", "doc_id": "mutation-000002", "scheduled_s": 0.2, "submitted_s": 0.20045429200399667, "schedule_delay_ms": 0.45429200399665914, "precondition": true, "status": "success", "ack_ms": 6.2292920192703605, "write_started_ms": 0.8478329982608557, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 6.318042054772377, "stable_since_ms": 6.318042054772377, "confirmed_ms": 63.79604199901223, "consecutive": 3, "regressions": 0, "requests": 3 }, "search": { "first_ms": 12.397458078339696, "stable_since_ms": 12.397458078339696, "confirmed_ms": 80.64816705882549, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 12.397458078339696, "confirmed_ms": 80.64816705882549, "search_minus_ack_ms": 6.1681660590693355 } ] }, { "run": "fb2_429e9a_014", "trial": 2, "backend": "qdrant", "scenario": "scifact-small", "status": "complete", "eligible": true, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "summary": { "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "wall_s": 30.056455583078787, "operations": { "insert": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 19.574415986426175, "p50": 66.38831249438226, "p95": 69.95584963588044, "p99": 108.7601960648317, "max": 112.3352920403704 }, "confirmed_ms": { "min": 112.46366600971669, "p50": 211.20258345035836, "p95": 217.67850241158155, "p99": 251.31642948021184, "max": 254.42795897834003 }, "ack_ms": { "min": 5.240250029601157, "p50": 10.001187503803521, "p95": 12.503318669041617, "p99": 13.438935924787078, "max": 14.191624941304326 }, "search_minus_ack_ms": { "min": 0.21275004837661982, "p50": 56.25145847443491, "p95": 59.93428180809132, "p99": 98.79140203236604, "max": 101.97029099799693 }, "regressions": 0, "confirmed_within_100ms": 0.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "update": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 16.616874956525862, "p50": 64.85764600802213, "p95": 68.37719439063221, "p99": 70.48421854735354, "max": 71.93304202519357 }, "confirmed_ms": { "min": 110.90187495574355, "p50": 210.05731250625104, "p95": 213.1075666868128, "p99": 215.18821425270292, "max": 219.44166696630418 }, "ack_ms": { "min": 4.810750018805265, "p50": 8.732812537346035, "p95": 11.734754184726626, "p99": 12.400660166749763, "max": 18.736082944087684 }, "search_minus_ack_ms": { "min": 1.2910410296171904, "p50": 55.84433348849416, "p95": 58.37034785072319, "p99": 60.207404118264094, "max": 60.861999983899295 }, "regressions": 0, "confirmed_within_100ms": 0.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "delete": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 3.5689580254256725, "p50": 18.721437023486942, "p95": 60.77254583942703, "p99": 73.88423079042697, "max": 75.72620804421604 }, "confirmed_ms": { "min": 103.79458300303668, "p50": 114.57720800535753, "p95": 204.97401639004238, "p99": 211.3839572190773, "max": 218.6479999218136 }, "ack_ms": { "min": 3.2701670425012708, "p50": 7.8814789885655046, "p95": 10.209041915368289, "p99": 17.292355931131237, "max": 30.243042041547596 }, "search_minus_ack_ms": { "min": -2.6529169408604503, "p50": 0.21658401237800717, "p95": 53.580821451032534, "p99": 56.76430498948322, "max": 59.58708294201642 }, "regressions": 0, "confirmed_within_100ms": 0.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 } }, "total_operations": 300, "statuses": { "success": 300, "timeout": 0, "error": 0, "invalid_precondition": 0, "client_overload": 0 }, "query": { "count": 300, "success": 300, "ann_recall_at_10": 1.0, "ndcg_at_10": 0.6567157762068453, "latency_ms": { "min": 3.2952919136732817, "p50": 7.619250507559627, "p95": 10.314891714369878, "p99": 44.80607805657196, "max": 75.23870805744082 }, "dispatch_delay_ms": { "min": 0.14150002971291542, "p50": 1.4966460294090211, "p95": 2.4719045381061733, "p99": 2.6446053537074476, "max": 3.051375038921833 } }, "diagnostics": { "schedule_delay_ms": { "min": 0.004957988856801876, "p50": 0.42908355826543065, "p95": 0.7471313001594648, "p99": 0.7962758925281387, "max": 1.3103749835863265 }, "probe_dispatch_ms": { "min": 0.022999942302703857, "p50": 0.45804097317159176, "p95": 1.7288423026911914, "p99": 2.6073294994421303, "max": 6.835290929302573 }, "probe_service_ms": { "min": 1.148457988165319, "p50": 6.038374966010451, "p95": 47.86870366660878, "p99": 49.23218311741948, "max": 75.73508296627551 }, "write_service_ms": { "min": 2.972166985273361, "p50": 8.157583535648882, "p95": 11.359181243460627, "p99": 13.584604247007492, "max": 29.841833980754018 } }, "actual_submission_rate": 10.000030378993678, "metadata": { "backend": "qdrant", "collection": "fb2_429e9a_014", "dataset": { "dataset": "scifact", "source": "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/scifact.zip", "source_sha256": "536e14446a0ba56ed1398ab1055f39fe852686ecad24a6306c80c490fa8e0165", "source_md5": "5f7d1de60b170fc8027bb7898e2efca1", "records": 5183, "queries": 100, "relevant_documents_retained": 107, "seed": 20261028, "license": "Corpus: ODC-By-1.0; claims/evidence: CC-BY-4.0", "license_source": "https://raw.githubusercontent.com/allenai/scifact/master/LICENSE.md", "attribution": "BEIR and original dataset authors; Wikipedia contributors for NQ, SciFact authors and cited article authors for SciFact.", "transformation": "200-word title+text prefix; seeded reservoir of documents with relevant documents for selected test queries retained at front. Simulated mutations; no historical replay.", "model": "sentence-transformers/all-MiniLM-L6-v2", "model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "dimensions": 384, "normalized": true, "max_tokens": 256, "device": "mps", "files": { "corpus.jsonl": "189d58f273468352c9f73c6333c5120dcfb7ba1695984f073f0ebb6c100ed0cd", "queries.json": "c8b333c73353a8c4aeb7d6b63067c9a241aa5d39a0314f0c315ec63019ee9558", "vectors.npy": "b6b21404977151ce565e7d6d57d9931a4679e1ecf4aa44fcd9b29eb54955d09e", "query_vectors.npy": "02324eb7c3b3d2e2f0ec3208453e2ec0efd5d61d9fc845b1f8b7950e913f2730", "UPSTREAM-LICENSE.md": "56ee69bc639cbab8f0b2e4b67091ec5e9390aa99a1d65f0a35e4274686c9a806" } }, "base_count": 4000, "profile": { "dataset": "scifact", "base_count": 4000, "rate": 10, "query_rate": 10, "arrival": "steady" }, "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "preloaded_mutable": 200, "started_utc": "2026-10-29T02:57:55Z", "setup_s": 1.7504542090464383, "backend_before": { "version": "1.20.0", "collection": { "status": "green", "optimizer_status": "ok", "indexed_vectors_count": 4200, "points_count": 4200, "segments_count": 2, "config": { "params": { "vectors": { "size": 384, "distance": "Cosine" }, "shard_number": 1, "replication_factor": 1, "write_consistency_factor": 1, "on_disk_payload": true }, "hnsw_config": { "m": 16, "ef_construct": 128, "full_scan_threshold": 10, "max_indexing_threads": 0, "on_disk": false }, "optimizer_config": { "deleted_threshold": 0.2, "vacuum_min_vector_number": 1000, "default_segment_number": 0, "max_segment_size": null, "memmap_threshold": null, "indexing_threshold": 1024, "flush_interval_sec": 5, "max_optimization_threads": null, "prevent_unoptimized": null }, "wal_config": { "wal_capacity_mb": 32, "wal_segments_ahead": 0, "wal_retain_closed": 1 }, "quantization_config": null }, "payload_schema": { "cohort": { "data_type": "keyword", "points": 4200 } }, "update_queue": { "length": 0 }, "created_at": "2026-10-29T02:57:55.155Z" }, "ack": "wait=true", "transport": "HTTP REST, pooled httpx" }, "count_before": 4200, "idle_duration_s": 10.0, "idle_queries": { "count": 100, "ann_recall": 1.0, "ndcg": 0.6567157762068453, "latency_ms": { "min": 2.9665830079466105, "p50": 7.532895542681217, "p95": 8.625241689151153, "p99": 9.291590899229275, "max": 53.021457977592945 } }, "exact_ndcg": 0.6567157762068453, "resources_before": { "usage_usec": 81250230, "user_usec": 76677181, "system_usec": 4573049, "nice_usec": 57411000, "nr_periods": 1897, "nr_throttled": 83, "throttled_usec": 316106, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 314392576, "pids": 32 }, "resources_after": { "usage_usec": 86413983, "user_usec": 81106342, "system_usec": 5307640, "nice_usec": 57411000, "nr_periods": 2199, "nr_throttled": 83, "throttled_usec": 316106, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 310951936, "pids": 33 }, "count_after": 4200, "backend_after": { "version": "1.20.0", "collection": { "status": "green", "optimizer_status": "ok", "indexed_vectors_count": 4200, "points_count": 4200, "segments_count": 2, "config": { "params": { "vectors": { "size": 384, "distance": "Cosine" }, "shard_number": 1, "replication_factor": 1, "write_consistency_factor": 1, "on_disk_payload": true }, "hnsw_config": { "m": 16, "ef_construct": 128, "full_scan_threshold": 10, "max_indexing_threads": 0, "on_disk": false }, "optimizer_config": { "deleted_threshold": 0.2, "vacuum_min_vector_number": 1000, "default_segment_number": 0, "max_segment_size": null, "memmap_threshold": null, "indexing_threshold": 1024, "flush_interval_sec": 5, "max_optimization_threads": null, "prevent_unoptimized": null }, "wal_config": { "wal_capacity_mb": 32, "wal_segments_ahead": 0, "wal_retain_closed": 1 }, "quantization_config": null }, "payload_schema": { "cohort": { "data_type": "keyword", "points": 4200 } }, "update_queue": { "length": 0 }, "created_at": "2026-10-29T02:57:55.155Z" }, "ack": "wait=true", "transport": "HTTP REST, pooled httpx" } }, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "eligible": true, "protocol": "mixed-mutations-v2" }, "examples": [ { "event": 0, "operation": "insert", "doc_id": "mutation-000000", "scheduled_s": 0.0, "submitted_s": 0.00021349999587982893, "schedule_delay_ms": 0.21349999587982893, "precondition": true, "status": "success", "ack_ms": 7.548708003014326, "write_started_ms": 1.4206250198185444, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 14.65358305722475, "stable_since_ms": 14.65358305722475, "confirmed_ms": 70.35912503488362, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 58.381958049722016, "stable_since_ms": 58.381958049722016, "confirmed_ms": 202.6626250008121, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 58.381958049722016, "confirmed_ms": 202.6626250008121, "search_minus_ack_ms": 50.83325004670769 }, { "event": 1, "operation": "update", "doc_id": "mutation-000001", "scheduled_s": 0.1, "submitted_s": 0.1004815420601517, "schedule_delay_ms": 0.48154206015169065, "precondition": true, "status": "success", "ack_ms": 10.922541026957333, "write_started_ms": 1.5560409519821405, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 21.883832989260554, "stable_since_ms": 21.883832989260554, "confirmed_ms": 77.70087500102818, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 66.40862498898059, "stable_since_ms": 66.40862498898059, "confirmed_ms": 208.55633297469467, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 66.40862498898059, "confirmed_ms": 208.55633297469467, "search_minus_ack_ms": 55.48608396202326 }, { "event": 2, "operation": "delete", "doc_id": "mutation-000002", "scheduled_s": 0.2, "submitted_s": 0.20027150004170835, "schedule_delay_ms": 0.2715000417083391, "precondition": true, "status": "success", "ack_ms": 5.8839579578489065, "write_started_ms": 0.9572079870849848, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 18.31629197113216, "stable_since_ms": 18.31629197113216, "confirmed_ms": 74.45325003936887, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 46.10875004436821, "stable_since_ms": 46.10875004436821, "confirmed_ms": 190.0252919876948, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 46.10875004436821, "confirmed_ms": 190.0252919876948, "search_minus_ack_ms": 40.2247920865193 } ] }, { "run": "fb2_429e9a_015", "trial": 5, "backend": "pgvector", "scenario": "scifact-small", "status": "complete", "eligible": true, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "summary": { "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "wall_s": 30.001418125000782, "operations": { "insert": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 15.676749986596406, "p50": 20.252500020433217, "p95": 23.293929535429925, "p99": 25.137451390037324, "max": 27.463292004540563 }, "confirmed_ms": { "min": 70.5480829346925, "p50": 81.07324998127297, "p95": 85.2607398875989, "p99": 86.17834568023684, "max": 90.83666699007154 }, "ack_ms": { "min": 3.3134579425677657, "p50": 8.193792018573731, "p95": 9.986423526424913, "p99": 16.011058756848808, "max": 18.182499916292727 }, "search_minus_ack_ms": { "min": 2.276916056871414, "p50": 11.924291495233774, "p95": 14.16529135312885, "p99": 15.060515527147805, "max": 16.00670802872628 }, "regressions": 0, "confirmed_within_100ms": 1.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "update": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 14.501375029794872, "p50": 20.074375031981617, "p95": 22.61715138447471, "p99": 26.43397622159683, "max": 27.928999974392354 }, "confirmed_ms": { "min": 70.0137090170756, "p50": 80.84777049953118, "p95": 84.79269753443077, "p99": 86.86490674503149, "max": 89.23583396244794 }, "ack_ms": { "min": 3.013417008332908, "p50": 7.976874941959977, "p95": 9.966343070846051, "p99": 15.983040576102237, "max": 20.65650001168251 }, "search_minus_ack_ms": { "min": 5.762374959886074, "p50": 11.989958002232015, "p95": 13.40989655582234, "p99": 14.246665465179834, "max": 14.807541039772332 }, "regressions": 0, "confirmed_within_100ms": 1.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "delete": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 12.20700005069375, "p50": 14.453208073973656, "p95": 21.458284824620932, "p99": 22.751964947674423, "max": 22.921749972738326 }, "confirmed_ms": { "min": 64.58554102573544, "p50": 68.38683347450569, "p95": 82.703517266782, "p99": 84.16770842042752, "max": 84.57612502388656 }, "ack_ms": { "min": 1.2450000504031777, "p50": 2.9254580149427056, "p95": 4.008327331393957, "p99": 4.536923418054357, "max": 4.607749986462295 }, "search_minus_ack_ms": { "min": 1.0263749863952398, "p50": 2.900541527196765, "p95": 17.835247953189537, "p99": 18.41796123888344, "max": 18.834874965250492 }, "regressions": 0, "confirmed_within_100ms": 1.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 } }, "total_operations": 300, "statuses": { "success": 300, "timeout": 0, "error": 0, "invalid_precondition": 0, "client_overload": 0 }, "query": { "count": 300, "success": 300, "ann_recall_at_10": 0.999, "ndcg_at_10": 0.6567157762068453, "latency_ms": { "min": 1.8152500269934535, "p50": 4.535541462246329, "p95": 5.816012190189213, "p99": 6.919099549995706, "max": 14.728749985806644 }, "dispatch_delay_ms": { "min": 0.11479202657938004, "p50": 1.2005000025965273, "p95": 2.0966871816199277, "p99": 2.3887304472737014, "max": 2.9088750015944242 } }, "diagnostics": { "schedule_delay_ms": { "min": 0.009917002170567457, "p50": 0.4048960749063468, "p95": 0.7206187280839772, "p99": 0.7565159273803168, "max": 0.8042500820018006 }, "probe_dispatch_ms": { "min": 0.03020802978426218, "p50": 0.45745796523988247, "p95": 1.7048230220098048, "p99": 2.523968176683411, "max": 3.0462080612778664 }, "probe_service_ms": { "min": 0.4456250462681055, "p50": 2.823937509674579, "p95": 5.783791246358305, "p99": 6.717178982216869, "max": 16.23629208188504 }, "write_service_ms": { "min": 1.1089160107076168, "p50": 7.018187490757555, "p95": 9.298692020820456, "p99": 15.549438287271187, "max": 20.280790980905294 } }, "actual_submission_rate": 10.000004696320879, "metadata": { "backend": "pgvector", "collection": "fb2_429e9a_015", "dataset": { "dataset": "scifact", "source": "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/scifact.zip", "source_sha256": "536e14446a0ba56ed1398ab1055f39fe852686ecad24a6306c80c490fa8e0165", "source_md5": "5f7d1de60b170fc8027bb7898e2efca1", "records": 5183, "queries": 100, "relevant_documents_retained": 107, "seed": 20261028, "license": "Corpus: ODC-By-1.0; claims/evidence: CC-BY-4.0", "license_source": "https://raw.githubusercontent.com/allenai/scifact/master/LICENSE.md", "attribution": "BEIR and original dataset authors; Wikipedia contributors for NQ, SciFact authors and cited article authors for SciFact.", "transformation": "200-word title+text prefix; seeded reservoir of documents with relevant documents for selected test queries retained at front. Simulated mutations; no historical replay.", "model": "sentence-transformers/all-MiniLM-L6-v2", "model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "dimensions": 384, "normalized": true, "max_tokens": 256, "device": "mps", "files": { "corpus.jsonl": "189d58f273468352c9f73c6333c5120dcfb7ba1695984f073f0ebb6c100ed0cd", "queries.json": "c8b333c73353a8c4aeb7d6b63067c9a241aa5d39a0314f0c315ec63019ee9558", "vectors.npy": "b6b21404977151ce565e7d6d57d9931a4679e1ecf4aa44fcd9b29eb54955d09e", "query_vectors.npy": "02324eb7c3b3d2e2f0ec3208453e2ec0efd5d61d9fc845b1f8b7950e913f2730", "UPSTREAM-LICENSE.md": "56ee69bc639cbab8f0b2e4b67091ec5e9390aa99a1d65f0a35e4274686c9a806" } }, "base_count": 4000, "profile": { "dataset": "scifact", "base_count": 4000, "rate": 10, "query_rate": 10, "arrival": "steady" }, "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "preloaded_mutable": 200, "started_utc": "2026-10-29T02:58:42Z", "setup_s": 1.6686612910125405, "backend_before": { "version": "0.8.7", "postgres": "PostgreSQL 16.15 (Debian 16.15-1.pgdg12+2) on aarch64-unknown-linux-gnu, compiled by gcc (Debian 12.2.0-14+deb12u1) 12.2.0, 64-bit", "ef_search": 128, "ack": "autocommit, READ COMMITTED", "transport": "psycopg, connection per worker", "query_plan": [ { "Plan": { "Node Type": "Limit", "Parallel Aware": false, "Async Capable": false, "Startup Cost": 1216.56, "Total Cost": 1229.86, "Plan Rows": 10, "Plan Width": 16, "Plans": [ { "Node Type": "Index Scan", "Parent Relationship": "Outer", "Parallel Aware": false, "Async Capable": false, "Scan Direction": "Forward", "Index Name": "fb2_429e9a_015_hnsw", "Relation Name": "fb2_429e9a_015", "Alias": "fb2_429e9a_015", "Startup Cost": 1216.56, "Total Cost": 6538.0, "Plan Rows": 4000, "Plan Width": 16, "Order By": "(embedding <=> '[-0.025241137,-0.0037172192,-0.046351995,0.046476826,0.01966622,0.015340689,0.01234306,0.07297605,-0.018329926,-0.013840419,-0.07423897,0.0226357,-0.060784698,-0.028650705,0.05173048,-0.03129233,0.07885052,-0.024390804,0.034739245,0.002474597,0.06695868,-0.03807076,0.016879125,-0.007840027,0.026065528,0.014293832,-0.07275669,0.053708147,-0.020156316,0.07862689,-0.122224644,0.023270043,-0.03987285,-0.029884664,0.008407885,0.06157951,-0.045617852,-0.0013952592,-0.1272149,0.012878671,0.03199118,-0.054489113,-0.07601716,0.016042225,-0.02170239,0.0031456149,-0.03146653,0.036349196,0.030850349,0.101148084,-0.04483407,-0.02210456,0.015648471,0.008023243,0.02329231,-0.047311362,-0.00033133395,-0.060376305,-0.031292018,-0.052068412,-0.055494312,0.03995384,-0.05426452,-0.027163653,0.060765423,-0.029587762,0.062952094,-0.032716926,0.03284741,0.03383774,0.025922673,0.09215419,-0.045252554,0.033018086,-0.0063568754,0.003192685,0.036198895,0.071328856,0.061275735,-0.03834579,0.04380842,0.10884386,-0.06443366,0.027753524,0.016000912,0.031735342,-0.048004173,0.04857545,-0.058097336,0.043568186,-0.014455384,0.008579895,0.023373613,0.0023358837,0.070720226,-0.008269884,-0.011370413,-0.016605737,-0.002399004,-0.03909848,0.017561054,0.042421922,0.011341604,0.025210828,0.09824794,0.0038353435,0.015525245,0.082866095,0.0066154706,0.027564062,0.051245574,-0.009089501,0.13282458,0.04902157,-0.013380719,0.042171925,0.033162206,0.048019677,0.029081784,0.116638325,0.00094674755,0.07029037,0.004596228,-0.02894104,-0.0037263758,0.0041222423,-0.04681563,2.8784053e-33,0.09736768,0.02274426,-0.05078067,0.03864741,0.052899208,0.0496459,-0.060387712,-0.08358956,0.07020227,0.042317823,-0.037229892,-0.1324623,0.08240096,0.039485864,-0.09648351,-0.04383094,-0.06106865,0.025340972,0.031598456,0.04274742,-0.0010978275,0.0007333723,0.041061625,0.011878232,-0.12491255,0.06554766,-0.06920273,0.05781889,-0.05118846,0.043497268,-0.029946037,-0.08991292,0.095012516,-0.12172844,-0.048556324,-0.022020115,-0.014565633,-0.045546,-0.05527385,0.06144455,-0.040042106,0.011806558,-0.13014492,-0.024195487,0.008652814,-0.05887302,0.074748434,-0.03390347,-0.026171094,-0.046370335,-0.02871423,0.01613992,0.050530907,-0.072944105,0.0142246755,-0.0029443551,-0.028645331,0.024415584,-0.021900453,-0.035877362,-0.012359783,0.0469807,0.062022846,0.09110963,-0.011605537,-0.02338493,0.0334725,-0.03523439,-0.039391812,0.020923749,0.045133665,-0.0050133714,0.05868334,0.05490907,0.002556749,0.04097332,0.008711031,-0.06200574,0.03028164,-0.11109333,0.0780934,0.012912109,0.07097299,-0.04068633,0.008197527,0.030907633,0.05238045,0.0011536852,-0.01918761,-0.15324447,0.016286323,-0.05679703,-0.04369561,-0.06417541,0.02090238,-3.0734922e-33,-0.005520113,0.007917555,0.051423088,-0.08971186,0.009558501,0.04202714,-0.032392696,0.007947915,-0.033026196,0.057292204,0.049505785,-0.017359108,-0.087639086,-0.047633022,0.012320442,0.036888227,-0.032912884,-0.05555789,-0.030620035,0.040593192,0.056201898,-0.0435655,0.031078862,0.0809475,0.0595156,-0.008109408,-0.10603491,0.0726334,-0.06864153,-0.034877256,0.023989158,-0.04435082,-0.019972347,-0.12077245,0.044683173,0.014618874,-0.020796062,0.114251524,0.01766482,0.0160371,0.012701173,-0.0053563626,0.074527174,-0.021048209,-0.022659509,0.08406541,-0.06423724,-0.071950026,0.05369397,0.01140933,-0.088458955,-0.01212312,0.0024443534,0.12023613,-0.027182862,-0.022696352,0.020270811,0.08858118,-0.10382351,0.086645365,-0.06383489,-0.04042209,0.04494627,0.016389402,0.13325493,0.08580276,0.060353823,0.03661967,0.041859947,0.021768665,0.0073611615,-0.009300088,-0.033082195,-0.040072612,0.06993921,0.02415639,0.012099922,0.034952287,-0.10813522,-0.024580343,0.05526578,-0.043421194,-0.00020226264,0.013380121,-0.013031641,-0.0023380888,0.020561703,-0.026072076,-0.058541298,-0.027999775,-0.024194289,-0.07036205,-0.05425848,-0.07206627,-0.026656583,-2.3484924e-08,-0.037731905,0.015433777,-0.059194244,-0.062783524,-0.047979638,-0.024383545,-0.1138203,0.040749338,-0.038158197,0.008040415,0.024223538,-0.018507883,0.108268395,0.025551548,0.068713784,-0.043465756,0.017785417,0.07973369,-0.0075122416,-0.04605511,-0.047828607,-0.017772881,0.05118608,0.0022625374,0.03168905,0.0143064195,0.017061999,-0.05608364,0.0064040497,0.0072717075,-0.013301113,0.018809363,-0.017954899,0.08202268,-0.06270375,0.029983893,0.0048537604,-0.035181895,-0.06037397,-0.0063187,0.0034529127,0.011674293,0.013009845,0.012574133,0.031151973,-0.087101504,0.030415354,0.036086276,-0.06438181,-0.08931628,0.0122331055,-0.0099123195,0.008941321,-0.024656717,-0.025116868,-0.0044285995,-0.065338664,0.0398255,0.039999187,0.011154338,0.010968719,-0.048901215,0.096052974,-0.08210191]'::vector)", "Filter": "(cohort = 'static'::text)" } ] } } ] }, "count_before": 4200, "idle_duration_s": 10.0, "idle_queries": { "count": 100, "ann_recall": 0.9990000000000001, "ndcg": 0.6567157762068453, "latency_ms": { "min": 2.380124991759658, "p50": 6.2431879341602325, "p95": 7.320660422556102, "p99": 8.114059764193426, "max": 18.605749937705696 } }, "exact_ndcg": 0.6567157762068453, "resources_before": { "usage_usec": 226854776, "user_usec": 211539414, "system_usec": 15315361, "nice_usec": 0, "nr_periods": 3290, "nr_throttled": 855, "throttled_usec": 90184282, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 1056600064, "pids": 23 }, "resources_after": { "usage_usec": 232127928, "user_usec": 216345506, "system_usec": 15782421, "nice_usec": 0, "nr_periods": 3591, "nr_throttled": 855, "throttled_usec": 90184282, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 1067700224, "pids": 27 }, "count_after": 4200, "backend_after": { "version": "0.8.7", "postgres": "PostgreSQL 16.15 (Debian 16.15-1.pgdg12+2) on aarch64-unknown-linux-gnu, compiled by gcc (Debian 12.2.0-14+deb12u1) 12.2.0, 64-bit", "ef_search": 128, "ack": "autocommit, READ COMMITTED", "transport": "psycopg, connection per worker", "query_plan": [ { "Plan": { "Node Type": "Limit", "Parallel Aware": false, "Async Capable": false, "Startup Cost": 1246.69, "Total Cost": 1260.24, "Plan Rows": 10, "Plan Width": 16, "Plans": [ { "Node Type": "Index Scan", "Parent Relationship": "Outer", "Parallel Aware": false, "Async Capable": false, "Scan Direction": "Forward", "Index Name": "fb2_429e9a_015_hnsw", "Relation Name": "fb2_429e9a_015", "Alias": "fb2_429e9a_015", "Startup Cost": 1246.69, "Total Cost": 6804.35, "Plan Rows": 4100, "Plan Width": 16, "Order By": "(embedding <=> '[-0.025241137,-0.0037172192,-0.046351995,0.046476826,0.01966622,0.015340689,0.01234306,0.07297605,-0.018329926,-0.013840419,-0.07423897,0.0226357,-0.060784698,-0.028650705,0.05173048,-0.03129233,0.07885052,-0.024390804,0.034739245,0.002474597,0.06695868,-0.03807076,0.016879125,-0.007840027,0.026065528,0.014293832,-0.07275669,0.053708147,-0.020156316,0.07862689,-0.122224644,0.023270043,-0.03987285,-0.029884664,0.008407885,0.06157951,-0.045617852,-0.0013952592,-0.1272149,0.012878671,0.03199118,-0.054489113,-0.07601716,0.016042225,-0.02170239,0.0031456149,-0.03146653,0.036349196,0.030850349,0.101148084,-0.04483407,-0.02210456,0.015648471,0.008023243,0.02329231,-0.047311362,-0.00033133395,-0.060376305,-0.031292018,-0.052068412,-0.055494312,0.03995384,-0.05426452,-0.027163653,0.060765423,-0.029587762,0.062952094,-0.032716926,0.03284741,0.03383774,0.025922673,0.09215419,-0.045252554,0.033018086,-0.0063568754,0.003192685,0.036198895,0.071328856,0.061275735,-0.03834579,0.04380842,0.10884386,-0.06443366,0.027753524,0.016000912,0.031735342,-0.048004173,0.04857545,-0.058097336,0.043568186,-0.014455384,0.008579895,0.023373613,0.0023358837,0.070720226,-0.008269884,-0.011370413,-0.016605737,-0.002399004,-0.03909848,0.017561054,0.042421922,0.011341604,0.025210828,0.09824794,0.0038353435,0.015525245,0.082866095,0.0066154706,0.027564062,0.051245574,-0.009089501,0.13282458,0.04902157,-0.013380719,0.042171925,0.033162206,0.048019677,0.029081784,0.116638325,0.00094674755,0.07029037,0.004596228,-0.02894104,-0.0037263758,0.0041222423,-0.04681563,2.8784053e-33,0.09736768,0.02274426,-0.05078067,0.03864741,0.052899208,0.0496459,-0.060387712,-0.08358956,0.07020227,0.042317823,-0.037229892,-0.1324623,0.08240096,0.039485864,-0.09648351,-0.04383094,-0.06106865,0.025340972,0.031598456,0.04274742,-0.0010978275,0.0007333723,0.041061625,0.011878232,-0.12491255,0.06554766,-0.06920273,0.05781889,-0.05118846,0.043497268,-0.029946037,-0.08991292,0.095012516,-0.12172844,-0.048556324,-0.022020115,-0.014565633,-0.045546,-0.05527385,0.06144455,-0.040042106,0.011806558,-0.13014492,-0.024195487,0.008652814,-0.05887302,0.074748434,-0.03390347,-0.026171094,-0.046370335,-0.02871423,0.01613992,0.050530907,-0.072944105,0.0142246755,-0.0029443551,-0.028645331,0.024415584,-0.021900453,-0.035877362,-0.012359783,0.0469807,0.062022846,0.09110963,-0.011605537,-0.02338493,0.0334725,-0.03523439,-0.039391812,0.020923749,0.045133665,-0.0050133714,0.05868334,0.05490907,0.002556749,0.04097332,0.008711031,-0.06200574,0.03028164,-0.11109333,0.0780934,0.012912109,0.07097299,-0.04068633,0.008197527,0.030907633,0.05238045,0.0011536852,-0.01918761,-0.15324447,0.016286323,-0.05679703,-0.04369561,-0.06417541,0.02090238,-3.0734922e-33,-0.005520113,0.007917555,0.051423088,-0.08971186,0.009558501,0.04202714,-0.032392696,0.007947915,-0.033026196,0.057292204,0.049505785,-0.017359108,-0.087639086,-0.047633022,0.012320442,0.036888227,-0.032912884,-0.05555789,-0.030620035,0.040593192,0.056201898,-0.0435655,0.031078862,0.0809475,0.0595156,-0.008109408,-0.10603491,0.0726334,-0.06864153,-0.034877256,0.023989158,-0.04435082,-0.019972347,-0.12077245,0.044683173,0.014618874,-0.020796062,0.114251524,0.01766482,0.0160371,0.012701173,-0.0053563626,0.074527174,-0.021048209,-0.022659509,0.08406541,-0.06423724,-0.071950026,0.05369397,0.01140933,-0.088458955,-0.01212312,0.0024443534,0.12023613,-0.027182862,-0.022696352,0.020270811,0.08858118,-0.10382351,0.086645365,-0.06383489,-0.04042209,0.04494627,0.016389402,0.13325493,0.08580276,0.060353823,0.03661967,0.041859947,0.021768665,0.0073611615,-0.009300088,-0.033082195,-0.040072612,0.06993921,0.02415639,0.012099922,0.034952287,-0.10813522,-0.024580343,0.05526578,-0.043421194,-0.00020226264,0.013380121,-0.013031641,-0.0023380888,0.020561703,-0.026072076,-0.058541298,-0.027999775,-0.024194289,-0.07036205,-0.05425848,-0.07206627,-0.026656583,-2.3484924e-08,-0.037731905,0.015433777,-0.059194244,-0.062783524,-0.047979638,-0.024383545,-0.1138203,0.040749338,-0.038158197,0.008040415,0.024223538,-0.018507883,0.108268395,0.025551548,0.068713784,-0.043465756,0.017785417,0.07973369,-0.0075122416,-0.04605511,-0.047828607,-0.017772881,0.05118608,0.0022625374,0.03168905,0.0143064195,0.017061999,-0.05608364,0.0064040497,0.0072717075,-0.013301113,0.018809363,-0.017954899,0.08202268,-0.06270375,0.029983893,0.0048537604,-0.035181895,-0.06037397,-0.0063187,0.0034529127,0.011674293,0.013009845,0.012574133,0.031151973,-0.087101504,0.030415354,0.036086276,-0.06438181,-0.08931628,0.0122331055,-0.0099123195,0.008941321,-0.024656717,-0.025116868,-0.0044285995,-0.065338664,0.0398255,0.039999187,0.011154338,0.010968719,-0.048901215,0.096052974,-0.08210191]'::vector)", "Filter": "(cohort = 'static'::text)" } ] } } ] } }, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "eligible": true, "protocol": "mixed-mutations-v2" }, "examples": [ { "event": 0, "operation": "insert", "doc_id": "mutation-000000", "scheduled_s": 0.0, "submitted_s": 0.0003183750668540597, "schedule_delay_ms": 0.3183750668540597, "precondition": true, "status": "success", "ack_ms": 18.182499916292727, "write_started_ms": 1.4611249789595604, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 24.895082926377654, "stable_since_ms": 24.895082926377654, "confirmed_ms": 77.04516698140651, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 27.463292004540563, "stable_since_ms": 27.463292004540563, "confirmed_ms": 82.12141692638397, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 27.463292004540563, "confirmed_ms": 82.12141692638397, "search_minus_ack_ms": 9.280792088247836 }, { "event": 1, "operation": "update", "doc_id": "mutation-000001", "scheduled_s": 0.1, "submitted_s": 0.10009595798328519, "schedule_delay_ms": 0.09595798328518312, "precondition": true, "status": "success", "ack_ms": 5.713209044188261, "write_started_ms": 0.15591701958328485, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 13.62233399413526, "stable_since_ms": 13.62233399413526, "confirmed_ms": 68.11379210557789, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 17.421542084775865, "stable_since_ms": 17.421542084775865, "confirmed_ms": 76.69400004670024, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 17.421542084775865, "confirmed_ms": 76.69400004670024, "search_minus_ack_ms": 11.708333040587604 }, { "event": 2, "operation": "delete", "doc_id": "mutation-000002", "scheduled_s": 0.2, "submitted_s": 0.20063933299388736, "schedule_delay_ms": 0.6393329938873538, "precondition": true, "status": "success", "ack_ms": 3.152125049382448, "write_started_ms": 0.7716250838711858, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 13.927209074608982, "stable_since_ms": 13.927209074608982, "confirmed_ms": 68.37112503126264, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 18.71991704683751, "stable_since_ms": 18.71991704683751, "confirmed_ms": 79.51158401556313, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 18.71991704683751, "confirmed_ms": 79.51158401556313, "search_minus_ack_ms": 15.56779199745506 } ] }, { "run": "fb2_429e9a_016", "trial": 3, "backend": "weaviate", "scenario": "scifact-small", "status": "complete", "eligible": true, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "summary": { "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "wall_s": 30.000871125026606, "operations": { "insert": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 8.92404094338417, "p50": 18.603332981001586, "p95": 30.845760786905885, "p99": 33.42099231318573, "max": 36.91820800304413 }, "confirmed_ms": { "min": 65.81870804075152, "p50": 81.61568798823282, "p95": 102.79049620730802, "p99": 103.71762247523293, "max": 104.36087497510016 }, "ack_ms": { "min": 3.2172080827876925, "p50": 9.640104544814676, "p95": 14.212421188130975, "p99": 16.009052840527175, "max": 21.273666061460972 }, "search_minus_ack_ms": { "min": -0.9918330470100045, "p50": 3.275396069511771, "p95": 18.180966371437535, "p99": 18.970532084349564, "max": 22.7975829038769 }, "regressions": 0, "confirmed_within_100ms": 0.92, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "update": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 8.29770800191909, "p50": 17.895833065267652, "p95": 31.2450127734337, "p99": 33.63883301150054, "max": 34.294708049856126 }, "confirmed_ms": { "min": 68.41608404647559, "p50": 79.13433347130194, "p95": 103.50165865966119, "p99": 104.15459014242515, "max": 105.43408303055912 }, "ack_ms": { "min": 3.9486250607296824, "p50": 9.177874948363751, "p95": 13.099108630558476, "p99": 14.342250407207763, "max": 15.282790991477668 }, "search_minus_ack_ms": { "min": 0.11662498582154512, "p50": 2.6594999944791198, "p95": 18.733350047841668, "p99": 21.146586770191792, "max": 22.06270908936858 }, "regressions": 0, "confirmed_within_100ms": 0.92, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "delete": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 4.725291975773871, "p50": 16.884229553397745, "p95": 24.59532503271475, "p99": 27.827091972576458, "max": 28.211499913595617 }, "confirmed_ms": { "min": 62.82416696194559, "p50": 79.04314598999918, "p95": 88.75638328609057, "p99": 93.18285913323052, "max": 93.89904199633747 }, "ack_ms": { "min": 2.1485830657184124, "p50": 5.489270493853837, "p95": 11.124448629561812, "p99": 12.857340752379969, "max": 12.985915993340313 }, "search_minus_ack_ms": { "min": -0.11604093015193939, "p50": 4.437500028871, "p95": 15.256063075503333, "p99": 17.987719607772306, "max": 18.071333062835038 }, "regressions": 0, "confirmed_within_100ms": 1.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 } }, "total_operations": 300, "statuses": { "success": 300, "timeout": 0, "error": 0, "invalid_precondition": 0, "client_overload": 0 }, "query": { "count": 300, "success": 300, "ann_recall_at_10": 1.0, "ndcg_at_10": 0.6567157762068453, "latency_ms": { "min": 4.225917044095695, "p50": 9.77766647702083, "p95": 15.334347618045285, "p99": 19.717212050454684, "max": 23.741375072859228 }, "dispatch_delay_ms": { "min": 0.134709058329463, "p50": 1.4834795147180557, "p95": 2.354298485442996, "p99": 2.942777823191136, "max": 3.4234169870615005 } }, "diagnostics": { "schedule_delay_ms": { "min": 0.013666972517256681, "p50": 0.46214598696669995, "p95": 0.7438211829880359, "p99": 0.7655687273942124, "max": 0.918875052594359 }, "probe_dispatch_ms": { "min": 0.01850002445280552, "p50": 0.5057500675320625, "p95": 1.969541481230408, "p99": 2.732717338949442, "max": 3.560040961019695 }, "probe_service_ms": { "min": 0.6910410011187196, "p50": 5.499750026501715, "p95": 10.75587549712509, "p99": 13.671841658651795, "max": 22.748582996428013 }, "write_service_ms": { "min": 1.8422920256853104, "p50": 7.656228961423039, "p95": 12.453903548885137, "p99": 14.89432974369265, "max": 20.550083019770682 } }, "actual_submission_rate": 9.999958988444185, "metadata": { "backend": "weaviate", "collection": "fb2_429e9a_016", "dataset": { "dataset": "scifact", "source": "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/scifact.zip", "source_sha256": "536e14446a0ba56ed1398ab1055f39fe852686ecad24a6306c80c490fa8e0165", "source_md5": "5f7d1de60b170fc8027bb7898e2efca1", "records": 5183, "queries": 100, "relevant_documents_retained": 107, "seed": 20261028, "license": "Corpus: ODC-By-1.0; claims/evidence: CC-BY-4.0", "license_source": "https://raw.githubusercontent.com/allenai/scifact/master/LICENSE.md", "attribution": "BEIR and original dataset authors; Wikipedia contributors for NQ, SciFact authors and cited article authors for SciFact.", "transformation": "200-word title+text prefix; seeded reservoir of documents with relevant documents for selected test queries retained at front. Simulated mutations; no historical replay.", "model": "sentence-transformers/all-MiniLM-L6-v2", "model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "dimensions": 384, "normalized": true, "max_tokens": 256, "device": "mps", "files": { "corpus.jsonl": "189d58f273468352c9f73c6333c5120dcfb7ba1695984f073f0ebb6c100ed0cd", "queries.json": "c8b333c73353a8c4aeb7d6b63067c9a241aa5d39a0314f0c315ec63019ee9558", "vectors.npy": "b6b21404977151ce565e7d6d57d9931a4679e1ecf4aa44fcd9b29eb54955d09e", "query_vectors.npy": "02324eb7c3b3d2e2f0ec3208453e2ec0efd5d61d9fc845b1f8b7950e913f2730", "UPSTREAM-LICENSE.md": "56ee69bc639cbab8f0b2e4b67091ec5e9390aa99a1d65f0a35e4274686c9a806" } }, "base_count": 4000, "profile": { "dataset": "scifact", "base_count": 4000, "rate": 10, "query_rate": 10, "arrival": "steady" }, "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "preloaded_mutable": 200, "started_utc": "2026-10-29T02:59:28Z", "setup_s": 1.148817582987249, "backend_before": { "version": "1.40.3", "schema": { "class": "Ffb2429e9a016", "invertedIndexConfig": { "bm25": { "b": 0.75, "k1": 1.2 }, "cleanupIntervalSeconds": 60, "stopwords": { "additions": null, "preset": "en", "removals": null }, "usingBlockMaxWAND": true }, "multiTenancyConfig": { "autoTenantActivation": false, "autoTenantCreation": false, "enabled": false }, "properties": [ { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "doc_id", "tokenization": "field" }, { "dataType": [ "int" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": false, "name": "version" }, { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "cohort", "tokenization": "field" }, { "dataType": [ "text" ], "indexFilterable": false, "indexRangeFilters": false, "indexSearchable": false, "name": "text", "tokenization": "word" } ], "shardingConfig": { "actualCount": 1, "actualVirtualCount": 128, "desiredCount": 1, "desiredVirtualCount": 128, "function": "murmur3", "key": "_id", "strategy": "hash", "virtualPerPhysical": 128 }, "vectorIndexConfig": { "bq": { "enabled": false }, "cleanupIntervalSeconds": 300, "distance": "cosine", "dynamicEfFactor": 8, "dynamicEfMax": 500, "dynamicEfMin": 100, "ef": 128, "efConstruction": 128, "filterStrategy": "acorn", "flatSearchCutoff": 0, "maxConnections": 16, "multivector": { "aggregation": "maxSim", "enabled": false, "muvera": { "dprojections": 16, "enabled": false, "ksim": 4, "repetitions": 10 } }, "pq": { "bitCompression": false, "centroids": 256, "enabled": false, "encoder": { "distribution": "log-normal", "type": "kmeans" }, "segments": 0, "trainingLimit": 100000 }, "rq": { "bits": 8, "centering": false, "enabled": false, "rescoreLimit": 20, "trainingLimit": 10000 }, "skip": false, "skipDefaultQuantization": false, "sq": { "enabled": false, "rescoreLimit": 20, "trainingLimit": 100000 }, "trackDefaultQuantization": false, "vectorCacheMaxObjects": 1000000000000 }, "vectorIndexType": "hnsw", "vectorizer": "none", "replicationConfig": { "deletionStrategy": "TimeBasedResolution", "factor": 1, "asyncEnabled": false } }, "shards": [ { "name": "tZn3pO4pR3xm", "per_node_status": { "node1": "READY" }, "status": "READY", "vectorQueueSize": 0 } ], "ack": "synchronous indexing, object PUT/POST/DELETE", "transport": "HTTP REST writes + GraphQL search, pooled httpx" }, "count_before": 4200, "idle_duration_s": 10.0, "idle_queries": { "count": 100, "ann_recall": 1.0, "ndcg": 0.6567157762068453, "latency_ms": { "min": 4.144249949604273, "p50": 10.384125460404903, "p95": 12.243893748382106, "p99": 14.23429949791171, "max": 15.03941707778722 } }, "exact_ndcg": 0.6567157762068453, "resources_before": { "usage_usec": 172996598, "user_usec": 169235408, "system_usec": 3761190, "nice_usec": 0, "nr_periods": 4389, "nr_throttled": 243, "throttled_usec": 1085801, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 629088256, "pids": 11 }, "resources_after": { "usage_usec": 181600514, "user_usec": 177403070, "system_usec": 4197443, "nice_usec": 0, "nr_periods": 4690, "nr_throttled": 243, "throttled_usec": 1085801, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 627240960, "pids": 11 }, "count_after": 4200, "backend_after": { "version": "1.40.3", "schema": { "class": "Ffb2429e9a016", "invertedIndexConfig": { "bm25": { "b": 0.75, "k1": 1.2 }, "cleanupIntervalSeconds": 60, "stopwords": { "additions": null, "preset": "en", "removals": null }, "usingBlockMaxWAND": true }, "multiTenancyConfig": { "autoTenantActivation": false, "autoTenantCreation": false, "enabled": false }, "properties": [ { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "doc_id", "tokenization": "field" }, { "dataType": [ "int" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": false, "name": "version" }, { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "cohort", "tokenization": "field" }, { "dataType": [ "text" ], "indexFilterable": false, "indexRangeFilters": false, "indexSearchable": false, "name": "text", "tokenization": "word" } ], "shardingConfig": { "actualCount": 1, "actualVirtualCount": 128, "desiredCount": 1, "desiredVirtualCount": 128, "function": "murmur3", "key": "_id", "strategy": "hash", "virtualPerPhysical": 128 }, "vectorIndexConfig": { "bq": { "enabled": false }, "cleanupIntervalSeconds": 300, "distance": "cosine", "dynamicEfFactor": 8, "dynamicEfMax": 500, "dynamicEfMin": 100, "ef": 128, "efConstruction": 128, "filterStrategy": "acorn", "flatSearchCutoff": 0, "maxConnections": 16, "multivector": { "aggregation": "maxSim", "enabled": false, "muvera": { "dprojections": 16, "enabled": false, "ksim": 4, "repetitions": 10 } }, "pq": { "bitCompression": false, "centroids": 256, "enabled": false, "encoder": { "distribution": "log-normal", "type": "kmeans" }, "segments": 0, "trainingLimit": 100000 }, "rq": { "bits": 8, "centering": false, "enabled": false, "rescoreLimit": 20, "trainingLimit": 10000 }, "skip": false, "skipDefaultQuantization": false, "sq": { "enabled": false, "rescoreLimit": 20, "trainingLimit": 100000 }, "trackDefaultQuantization": false, "vectorCacheMaxObjects": 1000000000000 }, "vectorIndexType": "hnsw", "vectorizer": "none", "replicationConfig": { "deletionStrategy": "TimeBasedResolution", "factor": 1, "asyncEnabled": false } }, "shards": [ { "name": "tZn3pO4pR3xm", "per_node_status": { "node1": "READY" }, "status": "READY", "vectorQueueSize": 0 } ], "ack": "synchronous indexing, object PUT/POST/DELETE", "transport": "HTTP REST writes + GraphQL search, pooled httpx" } }, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "eligible": true, "protocol": "mixed-mutations-v2" }, "examples": [ { "event": 0, "operation": "insert", "doc_id": "mutation-000000", "scheduled_s": 0.0, "submitted_s": 0.0002903339918702841, "schedule_delay_ms": 0.2903339918702841, "precondition": true, "status": "success", "ack_ms": 6.136666052043438, "write_started_ms": 2.1151250693947077, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 13.653791043907404, "stable_since_ms": 13.653791043907404, "confirmed_ms": 67.1959159662947, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 6.840415997430682, "stable_since_ms": 6.840415997430682, "confirmed_ms": 67.10329104680568, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 13.653791043907404, "confirmed_ms": 67.1959159662947, "search_minus_ack_ms": 0.7037499453872442 }, { "event": 1, "operation": "update", "doc_id": "mutation-000001", "scheduled_s": 0.1, "submitted_s": 0.10010174999479204, "schedule_delay_ms": 0.10174999479203861, "precondition": true, "status": "success", "ack_ms": 10.574209038168192, "write_started_ms": 0.7821250474080443, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 19.05579201411456, "stable_since_ms": 19.05579201411456, "confirmed_ms": 75.44208399485797, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 14.505875064060092, "stable_since_ms": 14.505875064060092, "confirmed_ms": 78.09495902620256, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 19.05579201411456, "confirmed_ms": 78.09495902620256, "search_minus_ack_ms": 3.9316660258919 }, { "event": 2, "operation": "delete", "doc_id": "mutation-000002", "scheduled_s": 0.2, "submitted_s": 0.2005097500514239, "schedule_delay_ms": 0.5097500514238962, "precondition": true, "status": "success", "ack_ms": 4.095416981726885, "write_started_ms": 0.4812499973922968, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 16.671374905854464, "stable_since_ms": 16.671374905854464, "confirmed_ms": 70.35762490704656, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 8.060291991569102, "stable_since_ms": 8.060291991569102, "confirmed_ms": 70.842916960828, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 16.671374905854464, "confirmed_ms": 70.842916960828, "search_minus_ack_ms": 3.964875009842217 } ] }, { "run": "fb2_429e9a_017", "trial": 1, "backend": "weaviate", "scenario": "scifact-small", "status": "complete", "eligible": true, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "summary": { "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "wall_s": 30.0008368329145, "operations": { "insert": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 6.945707951672375, "p50": 17.875770514365286, "p95": 31.52760628145188, "p99": 33.36950402241204, "max": 40.71650002151728 }, "confirmed_ms": { "min": 68.2729579275474, "p50": 80.95593797042966, "p95": 102.88502874900587, "p99": 104.40290787140843, "max": 112.95312503352761 }, "ack_ms": { "min": 4.872167017310858, "p50": 9.580938029102981, "p95": 13.31262887106277, "p99": 14.459732411196493, "max": 21.767458994872868 }, "search_minus_ack_ms": { "min": 0.12616603635251522, "p50": 3.003957972396165, "p95": 18.742685083998367, "p99": 20.542211360298104, "max": 21.631541894748807 }, "regressions": 0, "confirmed_within_100ms": 0.91, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "update": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 8.969833026640117, "p50": 18.173207994550467, "p95": 30.44567526085302, "p99": 31.859036055393528, "max": 33.59916596673429 }, "confirmed_ms": { "min": 67.08804203663021, "p50": 80.56002046214417, "p95": 102.80484800459817, "p99": 103.82540227030404, "max": 104.18291704263538 }, "ack_ms": { "min": 4.003167035989463, "p50": 9.528666501864791, "p95": 13.618444069288671, "p99": 14.941887498134756, "max": 16.498208045959473 }, "search_minus_ack_ms": { "min": -0.2846249844878912, "p50": 3.324812510982156, "p95": 18.133493448840454, "p99": 19.183822995983064, "max": 19.422125071287155 }, "regressions": 0, "confirmed_within_100ms": 0.93, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "delete": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 6.627500057220459, "p50": 16.970728989690542, "p95": 24.4229735690169, "p99": 29.5452438283246, "max": 29.734416981227696 }, "confirmed_ms": { "min": 69.58112504798919, "p50": 78.66560452384874, "p95": 91.46205619326791, "p99": 98.04643293027766, "max": 98.32441702019423 }, "ack_ms": { "min": 1.6860419418662786, "p50": 5.586874962318689, "p95": 11.504544038325548, "p99": 12.074309297604511, "max": 16.8899999698624 }, "search_minus_ack_ms": { "min": 0.12799992691725492, "p50": 4.144270496908575, "p95": 14.911502460017795, "p99": 18.06695325183682, "max": 18.34279205650091 }, "regressions": 0, "confirmed_within_100ms": 1.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 } }, "total_operations": 300, "statuses": { "success": 300, "timeout": 0, "error": 0, "invalid_precondition": 0, "client_overload": 0 }, "query": { "count": 300, "success": 300, "ann_recall_at_10": 1.0, "ndcg_at_10": 0.6567157762068453, "latency_ms": { "min": 4.77937504183501, "p50": 9.698375477455556, "p95": 13.32284351810813, "p99": 16.93870631046592, "max": 23.73808401171118 }, "dispatch_delay_ms": { "min": 0.0660829246044159, "p50": 1.3892084825783968, "p95": 2.445849939249456, "p99": 2.8441408544313154, "max": 3.4074579598382115 } }, "diagnostics": { "schedule_delay_ms": { "min": 0.006832997314631939, "p50": 0.35870797000825405, "p95": 0.7405065279456391, "p99": 0.7672246077563472, "max": 0.7758329622447047 }, "probe_dispatch_ms": { "min": 0.02629193477332592, "p50": 0.4909579874947667, "p95": 2.02930844388902, "p99": 2.682233694940805, "max": 3.6117499694228172 }, "probe_service_ms": { "min": 0.9314999915659428, "p50": 5.5043750908225775, "p95": 10.740949586033816, "p99": 13.599886959418651, "max": 19.623083993792534 }, "write_service_ms": { "min": 1.5583339845761657, "p50": 8.146186999510974, "p95": 12.364233040716499, "p99": 14.482429655035945, "max": 20.697833970189095 } }, "actual_submission_rate": 10.000082651193118, "metadata": { "backend": "weaviate", "collection": "fb2_429e9a_017", "dataset": { "dataset": "scifact", "source": "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/scifact.zip", "source_sha256": "536e14446a0ba56ed1398ab1055f39fe852686ecad24a6306c80c490fa8e0165", "source_md5": "5f7d1de60b170fc8027bb7898e2efca1", "records": 5183, "queries": 100, "relevant_documents_retained": 107, "seed": 20261028, "license": "Corpus: ODC-By-1.0; claims/evidence: CC-BY-4.0", "license_source": "https://raw.githubusercontent.com/allenai/scifact/master/LICENSE.md", "attribution": "BEIR and original dataset authors; Wikipedia contributors for NQ, SciFact authors and cited article authors for SciFact.", "transformation": "200-word title+text prefix; seeded reservoir of documents with relevant documents for selected test queries retained at front. Simulated mutations; no historical replay.", "model": "sentence-transformers/all-MiniLM-L6-v2", "model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "dimensions": 384, "normalized": true, "max_tokens": 256, "device": "mps", "files": { "corpus.jsonl": "189d58f273468352c9f73c6333c5120dcfb7ba1695984f073f0ebb6c100ed0cd", "queries.json": "c8b333c73353a8c4aeb7d6b63067c9a241aa5d39a0314f0c315ec63019ee9558", "vectors.npy": "b6b21404977151ce565e7d6d57d9931a4679e1ecf4aa44fcd9b29eb54955d09e", "query_vectors.npy": "02324eb7c3b3d2e2f0ec3208453e2ec0efd5d61d9fc845b1f8b7950e913f2730", "UPSTREAM-LICENSE.md": "56ee69bc639cbab8f0b2e4b67091ec5e9390aa99a1d65f0a35e4274686c9a806" } }, "base_count": 4000, "profile": { "dataset": "scifact", "base_count": 4000, "rate": 10, "query_rate": 10, "arrival": "steady" }, "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "preloaded_mutable": 200, "started_utc": "2026-10-29T03:00:14Z", "setup_s": 1.1806183330481872, "backend_before": { "version": "1.40.3", "schema": { "class": "Ffb2429e9a017", "invertedIndexConfig": { "bm25": { "b": 0.75, "k1": 1.2 }, "cleanupIntervalSeconds": 60, "stopwords": { "additions": null, "preset": "en", "removals": null }, "usingBlockMaxWAND": true }, "multiTenancyConfig": { "autoTenantActivation": false, "autoTenantCreation": false, "enabled": false }, "properties": [ { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "doc_id", "tokenization": "field" }, { "dataType": [ "int" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": false, "name": "version" }, { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "cohort", "tokenization": "field" }, { "dataType": [ "text" ], "indexFilterable": false, "indexRangeFilters": false, "indexSearchable": false, "name": "text", "tokenization": "word" } ], "shardingConfig": { "actualCount": 1, "actualVirtualCount": 128, "desiredCount": 1, "desiredVirtualCount": 128, "function": "murmur3", "key": "_id", "strategy": "hash", "virtualPerPhysical": 128 }, "vectorIndexConfig": { "bq": { "enabled": false }, "cleanupIntervalSeconds": 300, "distance": "cosine", "dynamicEfFactor": 8, "dynamicEfMax": 500, "dynamicEfMin": 100, "ef": 128, "efConstruction": 128, "filterStrategy": "acorn", "flatSearchCutoff": 0, "maxConnections": 16, "multivector": { "aggregation": "maxSim", "enabled": false, "muvera": { "dprojections": 16, "enabled": false, "ksim": 4, "repetitions": 10 } }, "pq": { "bitCompression": false, "centroids": 256, "enabled": false, "encoder": { "distribution": "log-normal", "type": "kmeans" }, "segments": 0, "trainingLimit": 100000 }, "rq": { "bits": 8, "centering": false, "enabled": false, "rescoreLimit": 20, "trainingLimit": 10000 }, "skip": false, "skipDefaultQuantization": false, "sq": { "enabled": false, "rescoreLimit": 20, "trainingLimit": 100000 }, "trackDefaultQuantization": false, "vectorCacheMaxObjects": 1000000000000 }, "vectorIndexType": "hnsw", "vectorizer": "none", "replicationConfig": { "deletionStrategy": "TimeBasedResolution", "factor": 1, "asyncEnabled": false } }, "shards": [ { "name": "bjhUfjsjAmt2", "per_node_status": { "node1": "READY" }, "status": "READY", "vectorQueueSize": 0 } ], "ack": "synchronous indexing, object PUT/POST/DELETE", "transport": "HTTP REST writes + GraphQL search, pooled httpx" }, "count_before": 4200, "idle_duration_s": 10.0, "idle_queries": { "count": 100, "ann_recall": 1.0, "ndcg": 0.6567157762068453, "latency_ms": { "min": 4.831166006624699, "p50": 10.135812452062964, "p95": 12.13232510490343, "p99": 14.166149741504345, "max": 14.977041981182992 } }, "exact_ndcg": 0.6567157762068453, "resources_before": { "usage_usec": 184342736, "user_usec": 180047100, "system_usec": 4295635, "nice_usec": 0, "nr_periods": 4833, "nr_throttled": 243, "throttled_usec": 1085801, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 550170624, "pids": 11 }, "resources_after": { "usage_usec": 192908699, "user_usec": 188213417, "system_usec": 4695282, "nice_usec": 0, "nr_periods": 5134, "nr_throttled": 243, "throttled_usec": 1085801, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 552837120, "pids": 11 }, "count_after": 4200, "backend_after": { "version": "1.40.3", "schema": { "class": "Ffb2429e9a017", "invertedIndexConfig": { "bm25": { "b": 0.75, "k1": 1.2 }, "cleanupIntervalSeconds": 60, "stopwords": { "additions": null, "preset": "en", "removals": null }, "usingBlockMaxWAND": true }, "multiTenancyConfig": { "autoTenantActivation": false, "autoTenantCreation": false, "enabled": false }, "properties": [ { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "doc_id", "tokenization": "field" }, { "dataType": [ "int" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": false, "name": "version" }, { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "cohort", "tokenization": "field" }, { "dataType": [ "text" ], "indexFilterable": false, "indexRangeFilters": false, "indexSearchable": false, "name": "text", "tokenization": "word" } ], "shardingConfig": { "actualCount": 1, "actualVirtualCount": 128, "desiredCount": 1, "desiredVirtualCount": 128, "function": "murmur3", "key": "_id", "strategy": "hash", "virtualPerPhysical": 128 }, "vectorIndexConfig": { "bq": { "enabled": false }, "cleanupIntervalSeconds": 300, "distance": "cosine", "dynamicEfFactor": 8, "dynamicEfMax": 500, "dynamicEfMin": 100, "ef": 128, "efConstruction": 128, "filterStrategy": "acorn", "flatSearchCutoff": 0, "maxConnections": 16, "multivector": { "aggregation": "maxSim", "enabled": false, "muvera": { "dprojections": 16, "enabled": false, "ksim": 4, "repetitions": 10 } }, "pq": { "bitCompression": false, "centroids": 256, "enabled": false, "encoder": { "distribution": "log-normal", "type": "kmeans" }, "segments": 0, "trainingLimit": 100000 }, "rq": { "bits": 8, "centering": false, "enabled": false, "rescoreLimit": 20, "trainingLimit": 10000 }, "skip": false, "skipDefaultQuantization": false, "sq": { "enabled": false, "rescoreLimit": 20, "trainingLimit": 100000 }, "trackDefaultQuantization": false, "vectorCacheMaxObjects": 1000000000000 }, "vectorIndexType": "hnsw", "vectorizer": "none", "replicationConfig": { "deletionStrategy": "TimeBasedResolution", "factor": 1, "asyncEnabled": false } }, "shards": [ { "name": "bjhUfjsjAmt2", "per_node_status": { "node1": "READY" }, "status": "READY", "vectorQueueSize": 0 } ], "ack": "synchronous indexing, object PUT/POST/DELETE", "transport": "HTTP REST writes + GraphQL search, pooled httpx" } }, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "eligible": true, "protocol": "mixed-mutations-v2" }, "examples": [ { "event": 0, "operation": "insert", "doc_id": "mutation-000000", "scheduled_s": 0.0, "submitted_s": 0.00036237493623048067, "schedule_delay_ms": 0.36237493623048067, "precondition": true, "status": "success", "ack_ms": 6.932624964974821, "write_started_ms": 1.7901670653373003, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 14.824875048361719, "stable_since_ms": 14.824875048361719, "confirmed_ms": 69.60345804691315, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 7.279708050191402, "stable_since_ms": 7.279708050191402, "confirmed_ms": 65.84079202730209, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 14.824875048361719, "confirmed_ms": 69.60345804691315, "search_minus_ack_ms": 0.3470830852165818 }, { "event": 1, "operation": "update", "doc_id": "mutation-000001", "scheduled_s": 0.1, "submitted_s": 0.10013845795765519, "schedule_delay_ms": 0.13845795765518587, "precondition": true, "status": "success", "ack_ms": 8.918417035602033, "write_started_ms": 0.6039589643478394, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 17.73812505416572, "stable_since_ms": 17.73812505416572, "confirmed_ms": 76.52454194612801, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 12.060374952852726, "stable_since_ms": 12.060374952852726, "confirmed_ms": 80.50849998835474, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 17.73812505416572, "confirmed_ms": 80.50849998835474, "search_minus_ack_ms": 3.141957917250693 }, { "event": 2, "operation": "delete", "doc_id": "mutation-000002", "scheduled_s": 0.2, "submitted_s": 0.20051299990154803, "schedule_delay_ms": 0.5129999015480169, "precondition": true, "status": "success", "ack_ms": 5.232417024672031, "write_started_ms": 0.6918750004842877, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 17.971042077988386, "stable_since_ms": 17.971042077988386, "confirmed_ms": 72.59141700342298, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 10.71954204235226, "stable_since_ms": 10.71954204235226, "confirmed_ms": 76.52708305977285, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 17.971042077988386, "confirmed_ms": 76.52708305977285, "search_minus_ack_ms": 5.487125017680228 } ] }, { "run": "fb2_429e9a_018", "trial": 1, "backend": "pgvector", "scenario": "nq-100k-burst", "status": "complete", "eligible": true, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "summary": { "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "burst", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "wall_s": 30.001325875055045, "operations": { "insert": { "count": 130, "success": 130, "success_rate": 1.0, "first_observed_ms": { "min": 16.735333018004894, "p50": 28.201624925713986, "p95": 34.40463942242786, "p99": 35.596845304826275, "max": 44.520707917399704 }, "confirmed_ms": { "min": 72.50100001692772, "p50": 88.84277049219236, "p95": 96.82332848897204, "p99": 99.22646936960518, "max": 101.34304198436439 }, "ack_ms": { "min": 5.226665991358459, "p50": 16.991499986033887, "p95": 23.37302037049085, "p99": 24.135437959339466, "max": 34.55174993723631 }, "search_minus_ack_ms": { "min": 4.336500074714422, "p50": 11.747958487831056, "p95": 12.86771030863747, "p99": 14.280741881811988, "max": 15.90166601818055 }, "regressions": 0, "confirmed_within_100ms": 0.9923076923076923, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "update": { "count": 130, "success": 130, "success_rate": 1.0, "first_observed_ms": { "min": 14.950584038160741, "p50": 25.159186974633485, "p95": 30.471292044967413, "p99": 32.54509266233072, "max": 33.321249997243285 }, "confirmed_ms": { "min": 70.5073750577867, "p50": 85.92191652860492, "p95": 91.92856453591958, "p99": 94.44863086217083, "max": 95.10670800227672 }, "ack_ms": { "min": 4.325125017203391, "p50": 13.267770525999367, "p95": 19.000016810605302, "p99": 20.463731867494058, "max": 20.77125001233071 }, "search_minus_ack_ms": { "min": 8.886667084880173, "p50": 11.836312478408217, "p95": 13.39186440454796, "p99": 13.70880975271575, "max": 14.29954101331532 }, "regressions": 0, "confirmed_within_100ms": 1.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "delete": { "count": 130, "success": 130, "success_rate": 1.0, "first_observed_ms": { "min": 6.149958004243672, "p50": 13.183686998672783, "p95": 29.853814776288345, "p99": 33.22159750154241, "max": 34.908207948319614 }, "confirmed_ms": { "min": 61.95337499957532, "p50": 75.18239598721266, "p95": 94.3726536759641, "p99": 100.4722632654011, "max": 101.85375006403774 }, "ack_ms": { "min": 0.6227090489119291, "p50": 1.9544170354492962, "p95": 5.264991818694398, "p99": 6.437657490605486, "max": 6.836083950474858 }, "search_minus_ack_ms": { "min": 2.681249985471368, "p50": 10.668375005479902, "p95": 26.399258099263534, "p99": 30.042652643751328, "max": 32.08083298522979 }, "regressions": 0, "confirmed_within_100ms": 0.9846153846153847, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 } }, "total_operations": 390, "statuses": { "success": 390, "timeout": 0, "error": 0, "invalid_precondition": 0, "client_overload": 0 }, "query": { "count": 300, "success": 300, "ann_recall_at_10": 0.9929999999999999, "ndcg_at_10": 0.7931268675492565, "latency_ms": { "min": 3.4682919504120946, "p50": 10.084312001708895, "p95": 14.714500034460801, "p99": 17.168100810376913, "max": 22.039584000594914 }, "dispatch_delay_ms": { "min": 0.07695797830820084, "p50": 1.0467915562912822, "p95": 1.7107180960010737, "p99": 2.0213826058898117, "max": 2.174791065044701 } }, "diagnostics": { "schedule_delay_ms": { "min": 0.005000014789757756, "p50": 0.41458301711827517, "p95": 0.733483128716017, "p99": 0.7715496965679236, "max": 0.814208039082609 }, "probe_dispatch_ms": { "min": 0.02237502485513687, "p50": 0.4673328949138522, "p95": 1.2315625208429992, "p99": 1.6557547962293029, "max": 2.978249918669462 }, "probe_service_ms": { "min": 0.22066698875278234, "p50": 1.9257079111412168, "p95": 10.491687979083508, "p99": 14.900858304463325, "max": 18.868207931518555 }, "write_service_ms": { "min": 0.5438749212771654, "p50": 8.587312477175146, "p95": 20.391349983401597, "p99": 23.132596871582795, "max": 34.269999945536256 } }, "actual_submission_rate": 13.009816487630664, "metadata": { "backend": "pgvector", "collection": "fb2_429e9a_018", "dataset": { "dataset": "nq", "source": "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/nq.zip", "source_sha256": "2553bf7bfbab47b1436ca00a34bce57320e18e611fd00999a2a3a1b4714be752", "source_md5": "d4d3d2e48787a744b6f6e691ff534307", "records": 124000, "queries": 100, "relevant_documents_retained": 120, "seed": 20261028, "license": "CC-BY-SA-3.0", "license_source": "https://raw.githubusercontent.com/google-research-datasets/natural-questions/master/nq_open/README.md", "attribution": "BEIR and original dataset authors; Wikipedia contributors for NQ, SciFact authors and cited article authors for SciFact.", "transformation": "200-word title+text prefix; seeded reservoir of documents with relevant documents for selected test queries retained at front. Simulated mutations; no historical replay.", "model": "sentence-transformers/all-MiniLM-L6-v2", "model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "dimensions": 384, "normalized": true, "max_tokens": 256, "device": "mps", "files": { "corpus.jsonl": "9dc487d785e6ee493eccaf764daa3bf8720a8f4fc8c406246065e1ad33a91f5d", "queries.json": "5603f52876ada5d36118b65dea490f1d43900bc159e1804e637796eb9115e999", "vectors.npy": "cb7421eb6ca5964231403f0a922843088f6fd0e01cbf67f2a1076cc7101005db", "query_vectors.npy": "5e666f8d966d8db6b0fd93d4963234d2dffa0ee3910b8fad069a8ee201e8b5d4", "UPSTREAM-LICENSE.md": "8b675521da3fa142ac6bda38c9bcfcc8cff0e2d6128be53a062ed55fb53f7b87" } }, "base_count": 100000, "profile": { "dataset": "nq", "base_count": 100000, "rate": 10, "query_rate": 10, "arrival": "burst" }, "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "burst", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "preloaded_mutable": 260, "started_utc": "2026-10-29T03:00:59Z", "setup_s": 58.326709583052434, "backend_before": { "version": "0.8.7", "postgres": "PostgreSQL 16.15 (Debian 16.15-1.pgdg12+2) on aarch64-unknown-linux-gnu, compiled by gcc (Debian 12.2.0-14+deb12u1) 12.2.0, 64-bit", "ef_search": 128, "ack": "autocommit, READ COMMITTED", "transport": "psycopg, connection per worker", "query_plan": [ { "Plan": { "Node Type": "Limit", "Parallel Aware": false, "Async Capable": false, "Startup Cost": 1674.28, "Total Cost": 1691.41, "Plan Rows": 10, "Plan Width": 18, "Plans": [ { "Node Type": "Index Scan", "Parent Relationship": "Outer", "Parallel Aware": false, "Async Capable": false, "Scan Direction": "Forward", "Index Name": "fb2_429e9a_018_hnsw", "Relation Name": "fb2_429e9a_018", "Alias": "fb2_429e9a_018", "Startup Cost": 1674.28, "Total Cost": 172927.21, "Plan Rows": 100006, "Plan Width": 18, "Order By": "(embedding <=> '[-0.08946895,-0.094362676,0.092023626,-0.0040844884,0.043233823,0.00062519487,0.1075197,-0.035732996,0.013746996,-0.024103122,0.0021706754,-0.04720602,0.04787615,-0.02043222,0.058315646,0.081252135,-0.05375015,-0.030799072,-0.0077205584,0.035735283,-0.04474784,0.01614922,-0.06905586,0.04194401,-0.08832908,0.007873556,0.0044992934,0.021968655,0.010953364,0.07212316,0.023185596,0.001176476,0.04725166,-0.03580423,-0.032344446,0.004752835,-0.056179125,-0.0353235,-0.037251893,-0.005143068,-0.027599342,0.024403147,0.031338032,0.0056410627,0.008425455,-0.0087741045,-0.039870497,-0.011831836,0.042621233,0.048142947,0.01603842,0.06745542,-0.038018145,0.064212486,-0.009636566,0.01607475,-0.0027868534,-0.03761097,-0.011889431,-0.01084577,0.083403684,0.028946918,0.061870236,-0.045382615,0.03812586,0.060426526,-0.017003024,0.02328925,-0.093020774,-0.024072766,0.042720392,-0.022444837,-0.011597358,0.01209231,-0.005953629,0.029691145,-0.011711495,-0.053345133,-0.081757374,-0.049303155,0.032193717,-0.038507868,-0.0060101775,-0.031470332,-0.07362443,0.03575578,0.0245484,0.01985519,-0.15116754,0.050386053,-0.0389634,0.07548389,-0.09264168,-0.012602816,-0.0035539444,-0.046416666,0.015100493,0.100691915,-0.037529,0.04600518,-0.021859422,0.014675989,-0.097722545,-0.053200893,-0.017040461,-0.01629616,-0.012339222,0.06543302,0.07252074,-0.044301156,0.03054835,-0.038570553,0.019310065,-0.005527747,-0.029386865,-0.058248505,0.065443166,0.028506057,0.0878254,0.058387812,0.081721246,-0.0005962058,0.0569388,0.00960514,0.011667701,0.014456492,-0.07218306,-3.0444767e-33,0.06687367,-0.019213954,0.058202963,0.025171444,-0.028259544,-0.04171671,0.03127722,0.04865431,-0.0721459,-0.050143186,-0.029907746,0.06248994,-0.05096076,-0.065192886,0.010210447,-0.077256374,-0.043370593,-0.0901922,-0.066522665,0.026072454,-0.022315364,0.075375885,-0.029968103,-0.0050771856,-0.0046955296,0.0065650316,-0.06791897,0.07368851,0.056069646,-0.021026025,-0.011352883,0.038811427,-0.090847075,-0.056490287,-0.039881036,0.032885592,-0.029586185,0.04753608,0.006341778,-0.0034275919,0.03681623,0.09326265,-0.0532089,0.032396164,0.012605483,0.045778517,-0.0006472902,0.029721813,0.0734947,-0.037117425,-0.09356724,-0.0071634455,-0.032400057,0.010453378,0.06286513,0.006053158,0.060524322,0.091866635,0.063640274,0.055919155,0.053222507,-0.019174587,0.089296214,-0.058922265,0.013745364,-0.022907477,0.028347451,-0.040759467,-0.06820364,-0.061398294,-0.046857864,0.0060627847,0.010442015,-0.07662317,0.031950947,-0.107813306,-0.0016125626,-0.033413693,-0.09989722,-0.03580293,-0.036118146,-0.08276185,0.1280713,0.07596603,0.049225595,-0.026312606,0.0031930343,-0.0012469466,0.000872793,-0.040353082,-0.034516506,-0.036047976,0.004999689,-0.01570079,-0.09783203,1.4186912e-33,-0.0130903,0.04742377,0.02889633,0.09809088,0.043639127,-0.0747531,0.03402747,0.029147558,0.019848268,0.111865185,0.04845458,-0.016222177,0.025501896,0.09813583,-0.002833195,0.0038929228,0.048537854,0.014111116,0.036570966,-0.017512968,-0.079755045,0.043452248,0.10843892,0.061968718,-0.015678233,0.0060376576,0.010718569,-0.03907552,0.040307663,0.04553227,0.041731495,0.083179526,-0.0844024,-0.023447752,0.014485363,-0.00788756,-0.11410207,0.039309476,0.011220049,0.12889338,-0.07970701,0.010658111,-0.06657531,-0.022037188,-0.043459363,-0.030291978,0.05803613,0.06109021,-0.09415837,0.030727232,0.011858702,0.013526454,-0.049580045,-0.052486703,0.013218615,0.0050231596,0.07660412,-0.0046343356,-0.0692584,-0.03353194,0.022288544,0.022836983,0.01361394,-0.036331616,0.008194689,0.09226991,0.08229875,-0.047639843,-0.005033471,0.0066081914,-0.106666416,-0.06385503,0.006385333,0.06318358,0.03146985,-0.060764216,-0.043287244,-0.042385958,-0.07963737,-0.043478705,-0.031840656,0.060027804,0.027829666,0.03728562,-0.07014622,0.0649568,-0.052579004,-0.0034938955,-0.15390792,-0.04426162,0.028217599,0.05593145,0.023251446,0.029115746,-0.040321343,-1.8832651e-08,-0.037536032,0.0005056563,0.013919978,-0.038129315,0.061486673,0.005534046,-0.010768211,-0.16527015,0.061414365,0.019963786,-0.019666849,-0.059065994,0.00083181454,-0.016695915,0.01090875,0.02685501,-0.0008743374,-0.060967337,-0.049258173,0.012731965,-0.03973743,-0.0018210232,-0.03797001,0.00035104176,0.04477184,0.04185603,-0.03215796,-0.09103283,0.0467481,0.017810425,0.037437033,0.054782324,-0.0059442874,0.022557084,0.053393807,-0.03746794,-0.07080435,0.10548834,-0.04215435,0.042547524,0.05064807,0.07560324,0.04072184,-0.0011278372,0.00702018,-0.070807114,0.040035766,0.051417697,0.011316928,-0.028955372,0.04197099,-0.0026867494,-0.056857754,0.0761941,0.030621601,-0.052413527,-0.021492323,0.0957355,0.1013111,-0.082977295,-0.036080915,-0.03327806,0.0029404552,-0.03586191]'::vector)", "Filter": "(cohort = 'static'::text)" } ] } } ] }, "count_before": 100260, "idle_duration_s": 10.0, "idle_queries": { "count": 100, "ann_recall": 0.993, "ndcg": 0.7931268675492565, "latency_ms": { "min": 5.290750064887106, "p50": 18.314416462089866, "p95": 22.09521274198778, "p99": 23.364626184338707, "max": 23.8679590402171 } }, "exact_ndcg": 0.7931268675492565, "resources_before": { "usage_usec": 324365102, "user_usec": 305166706, "system_usec": 19198395, "nice_usec": 0, "nr_periods": 4344, "nr_throttled": 1277, "throttled_usec": 134802760, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 1584603136, "pids": 24 }, "resources_after": { "usage_usec": 335810666, "user_usec": 313083520, "system_usec": 22727146, "nice_usec": 0, "nr_periods": 4645, "nr_throttled": 1277, "throttled_usec": 134802760, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 1609740288, "pids": 34 }, "count_after": 100260, "backend_after": { "version": "0.8.7", "postgres": "PostgreSQL 16.15 (Debian 16.15-1.pgdg12+2) on aarch64-unknown-linux-gnu, compiled by gcc (Debian 12.2.0-14+deb12u1) 12.2.0, 64-bit", "ef_search": 128, "ack": "autocommit, READ COMMITTED", "transport": "psycopg, connection per worker", "query_plan": [ { "Plan": { "Node Type": "Limit", "Parallel Aware": false, "Async Capable": false, "Startup Cost": 1675.82, "Total Cost": 1692.95, "Plan Rows": 10, "Plan Width": 18, "Plans": [ { "Node Type": "Index Scan", "Parent Relationship": "Outer", "Parallel Aware": false, "Async Capable": false, "Scan Direction": "Forward", "Index Name": "fb2_429e9a_018_hnsw", "Relation Name": "fb2_429e9a_018", "Alias": "fb2_429e9a_018", "Startup Cost": 1675.82, "Total Cost": 173311.33, "Plan Rows": 100189, "Plan Width": 18, "Order By": "(embedding <=> '[-0.08946895,-0.094362676,0.092023626,-0.0040844884,0.043233823,0.00062519487,0.1075197,-0.035732996,0.013746996,-0.024103122,0.0021706754,-0.04720602,0.04787615,-0.02043222,0.058315646,0.081252135,-0.05375015,-0.030799072,-0.0077205584,0.035735283,-0.04474784,0.01614922,-0.06905586,0.04194401,-0.08832908,0.007873556,0.0044992934,0.021968655,0.010953364,0.07212316,0.023185596,0.001176476,0.04725166,-0.03580423,-0.032344446,0.004752835,-0.056179125,-0.0353235,-0.037251893,-0.005143068,-0.027599342,0.024403147,0.031338032,0.0056410627,0.008425455,-0.0087741045,-0.039870497,-0.011831836,0.042621233,0.048142947,0.01603842,0.06745542,-0.038018145,0.064212486,-0.009636566,0.01607475,-0.0027868534,-0.03761097,-0.011889431,-0.01084577,0.083403684,0.028946918,0.061870236,-0.045382615,0.03812586,0.060426526,-0.017003024,0.02328925,-0.093020774,-0.024072766,0.042720392,-0.022444837,-0.011597358,0.01209231,-0.005953629,0.029691145,-0.011711495,-0.053345133,-0.081757374,-0.049303155,0.032193717,-0.038507868,-0.0060101775,-0.031470332,-0.07362443,0.03575578,0.0245484,0.01985519,-0.15116754,0.050386053,-0.0389634,0.07548389,-0.09264168,-0.012602816,-0.0035539444,-0.046416666,0.015100493,0.100691915,-0.037529,0.04600518,-0.021859422,0.014675989,-0.097722545,-0.053200893,-0.017040461,-0.01629616,-0.012339222,0.06543302,0.07252074,-0.044301156,0.03054835,-0.038570553,0.019310065,-0.005527747,-0.029386865,-0.058248505,0.065443166,0.028506057,0.0878254,0.058387812,0.081721246,-0.0005962058,0.0569388,0.00960514,0.011667701,0.014456492,-0.07218306,-3.0444767e-33,0.06687367,-0.019213954,0.058202963,0.025171444,-0.028259544,-0.04171671,0.03127722,0.04865431,-0.0721459,-0.050143186,-0.029907746,0.06248994,-0.05096076,-0.065192886,0.010210447,-0.077256374,-0.043370593,-0.0901922,-0.066522665,0.026072454,-0.022315364,0.075375885,-0.029968103,-0.0050771856,-0.0046955296,0.0065650316,-0.06791897,0.07368851,0.056069646,-0.021026025,-0.011352883,0.038811427,-0.090847075,-0.056490287,-0.039881036,0.032885592,-0.029586185,0.04753608,0.006341778,-0.0034275919,0.03681623,0.09326265,-0.0532089,0.032396164,0.012605483,0.045778517,-0.0006472902,0.029721813,0.0734947,-0.037117425,-0.09356724,-0.0071634455,-0.032400057,0.010453378,0.06286513,0.006053158,0.060524322,0.091866635,0.063640274,0.055919155,0.053222507,-0.019174587,0.089296214,-0.058922265,0.013745364,-0.022907477,0.028347451,-0.040759467,-0.06820364,-0.061398294,-0.046857864,0.0060627847,0.010442015,-0.07662317,0.031950947,-0.107813306,-0.0016125626,-0.033413693,-0.09989722,-0.03580293,-0.036118146,-0.08276185,0.1280713,0.07596603,0.049225595,-0.026312606,0.0031930343,-0.0012469466,0.000872793,-0.040353082,-0.034516506,-0.036047976,0.004999689,-0.01570079,-0.09783203,1.4186912e-33,-0.0130903,0.04742377,0.02889633,0.09809088,0.043639127,-0.0747531,0.03402747,0.029147558,0.019848268,0.111865185,0.04845458,-0.016222177,0.025501896,0.09813583,-0.002833195,0.0038929228,0.048537854,0.014111116,0.036570966,-0.017512968,-0.079755045,0.043452248,0.10843892,0.061968718,-0.015678233,0.0060376576,0.010718569,-0.03907552,0.040307663,0.04553227,0.041731495,0.083179526,-0.0844024,-0.023447752,0.014485363,-0.00788756,-0.11410207,0.039309476,0.011220049,0.12889338,-0.07970701,0.010658111,-0.06657531,-0.022037188,-0.043459363,-0.030291978,0.05803613,0.06109021,-0.09415837,0.030727232,0.011858702,0.013526454,-0.049580045,-0.052486703,0.013218615,0.0050231596,0.07660412,-0.0046343356,-0.0692584,-0.03353194,0.022288544,0.022836983,0.01361394,-0.036331616,0.008194689,0.09226991,0.08229875,-0.047639843,-0.005033471,0.0066081914,-0.106666416,-0.06385503,0.006385333,0.06318358,0.03146985,-0.060764216,-0.043287244,-0.042385958,-0.07963737,-0.043478705,-0.031840656,0.060027804,0.027829666,0.03728562,-0.07014622,0.0649568,-0.052579004,-0.0034938955,-0.15390792,-0.04426162,0.028217599,0.05593145,0.023251446,0.029115746,-0.040321343,-1.8832651e-08,-0.037536032,0.0005056563,0.013919978,-0.038129315,0.061486673,0.005534046,-0.010768211,-0.16527015,0.061414365,0.019963786,-0.019666849,-0.059065994,0.00083181454,-0.016695915,0.01090875,0.02685501,-0.0008743374,-0.060967337,-0.049258173,0.012731965,-0.03973743,-0.0018210232,-0.03797001,0.00035104176,0.04477184,0.04185603,-0.03215796,-0.09103283,0.0467481,0.017810425,0.037437033,0.054782324,-0.0059442874,0.022557084,0.053393807,-0.03746794,-0.07080435,0.10548834,-0.04215435,0.042547524,0.05064807,0.07560324,0.04072184,-0.0011278372,0.00702018,-0.070807114,0.040035766,0.051417697,0.011316928,-0.028955372,0.04197099,-0.0026867494,-0.056857754,0.0761941,0.030621601,-0.052413527,-0.021492323,0.0957355,0.1013111,-0.082977295,-0.036080915,-0.03327806,0.0029404552,-0.03586191]'::vector)", "Filter": "(cohort = 'static'::text)" } ] } } ] } }, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "eligible": true, "protocol": "mixed-mutations-v2" }, "examples": [ { "event": 0, "operation": "insert", "doc_id": "mutation-000000", "scheduled_s": 0.0, "submitted_s": 0.00019420799799263477, "schedule_delay_ms": 0.19420799799263477, "precondition": true, "status": "success", "ack_ms": 21.413875045254827, "write_started_ms": 0.9542079642415047, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 24.249125039204955, "stable_since_ms": 24.249125039204955, "confirmed_ms": 77.30204204563051, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 31.702292035333812, "stable_since_ms": 31.702292035333812, "confirmed_ms": 88.5974169941619, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 31.702292035333812, "confirmed_ms": 88.5974169941619, "search_minus_ack_ms": 10.288416990078986 }, { "event": 1, "operation": "update", "doc_id": "mutation-000001", "scheduled_s": 0.025, "submitted_s": 0.02526833303272724, "schedule_delay_ms": 0.26833303272724013, "precondition": true, "status": "success", "ack_ms": 5.50983299035579, "write_started_ms": 0.13508298434317112, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 11.17220800369978, "stable_since_ms": 11.17220800369978, "confirmed_ms": 69.23679204192013, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 16.746707959100604, "stable_since_ms": 16.746707959100604, "confirmed_ms": 72.4831250263378, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 16.746707959100604, "confirmed_ms": 72.4831250263378, "search_minus_ack_ms": 11.236874968744814 }, { "event": 2, "operation": "delete", "doc_id": "mutation-000002", "scheduled_s": 0.05, "submitted_s": 0.05014808301348239, "schedule_delay_ms": 0.14808301348238906, "precondition": true, "status": "success", "ack_ms": 1.4385420363396406, "write_started_ms": 0.14000001829117537, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 11.363000026904047, "stable_since_ms": 11.363000026904047, "confirmed_ms": 63.759083044715226, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 14.891208033077419, "stable_since_ms": 14.891208033077419, "confirmed_ms": 71.33670803159475, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 14.891208033077419, "confirmed_ms": 71.33670803159475, "search_minus_ack_ms": 13.452665996737778 } ] }, { "run": "fb2_429e9a_019", "trial": 2, "backend": "weaviate", "scenario": "nq-10k", "status": "complete", "eligible": true, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "summary": { "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "wall_s": 30.00138983305078, "operations": { "insert": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 7.464833091944456, "p50": 17.523395479656756, "p95": 31.516566639766097, "p99": 32.280489426339045, "max": 32.67129196319729 }, "confirmed_ms": { "min": 66.68133300263435, "p50": 81.76781254587695, "p95": 103.87551870662719, "p99": 104.44407377042808, "max": 105.28050002176315 }, "ack_ms": { "min": 4.656958975829184, "p50": 9.540936967823654, "p95": 14.17015184415504, "p99": 16.35523880133406, "max": 17.975250026211143 }, "search_minus_ack_ms": { "min": 0.10504107922315598, "p50": 3.6338125355541706, "p95": 17.422579153208062, "p99": 18.335603395244114, "max": 19.896874902769923 }, "regressions": 0, "confirmed_within_100ms": 0.92, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "update": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 8.823500014841557, "p50": 17.767854034900665, "p95": 31.648829899495468, "p99": 33.536730790510774, "max": 33.81529194302857 }, "confirmed_ms": { "min": 69.37912502326071, "p50": 82.20256259664893, "p95": 103.83872464881279, "p99": 106.72633778769524, "max": 110.27012497652322 }, "ack_ms": { "min": 4.838041961193085, "p50": 9.522125066723675, "p95": 13.379058573627844, "p99": 14.248728405218587, "max": 17.40220806095749 }, "search_minus_ack_ms": { "min": -0.3597079776227474, "p50": 3.0496669933199883, "p95": 18.653092288877815, "p99": 20.08993120864034, "max": 20.98233299329877 }, "regressions": 0, "confirmed_within_100ms": 0.87, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "delete": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 7.797374972142279, "p50": 15.781812486238778, "p95": 26.37408274458721, "p99": 31.81189752183855, "max": 32.38754207268357 }, "confirmed_ms": { "min": 68.55437497142702, "p50": 79.94285447057337, "p95": 93.39297685655765, "p99": 99.58147860481405, "max": 103.62191707827151 }, "ack_ms": { "min": 3.3190000103786588, "p50": 5.621604039333761, "p95": 12.1532125165686, "p99": 18.04153661476449, "max": 19.52600001823157 }, "search_minus_ack_ms": { "min": -0.4612499615177512, "p50": 4.910979012493044, "p95": 15.663887711707497, "p99": 19.045504732057456, "max": 19.767875084653497 }, "regressions": 0, "confirmed_within_100ms": 0.99, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 } }, "total_operations": 300, "statuses": { "success": 300, "timeout": 0, "error": 0, "invalid_precondition": 0, "client_overload": 0 }, "query": { "count": 300, "success": 300, "ann_recall_at_10": 0.996, "ndcg_at_10": 0.9481505328688746, "latency_ms": { "min": 4.588624928146601, "p50": 10.145041509531438, "p95": 15.589810459641745, "p99": 19.98534800601191, "max": 25.522999931126833 }, "dispatch_delay_ms": { "min": 0.07229205220937729, "p50": 1.4278959715738893, "p95": 2.338614582549781, "p99": 2.665071672527119, "max": 2.8643750119954348 } }, "diagnostics": { "schedule_delay_ms": { "min": 0.004832982085645199, "p50": 0.38893751334345694, "p95": 0.7456396217471628, "p99": 0.7637546327896416, "max": 0.8205420104787642 }, "probe_dispatch_ms": { "min": 0.030207913368940353, "p50": 0.492541934363544, "p95": 1.879333984106779, "p99": 2.5901501765474677, "max": 3.6234159488230944 }, "probe_service_ms": { "min": 0.8337500039488077, "p50": 5.565167055465281, "p95": 11.393625056371093, "p99": 14.670816855505096, "max": 38.59716700389981 }, "write_service_ms": { "min": 2.781457966193557, "p50": 8.15877056447789, "p95": 12.88170655607246, "p99": 16.594591755419962, "max": 19.0033339895308 } }, "actual_submission_rate": 9.999812364401768, "metadata": { "backend": "weaviate", "collection": "fb2_429e9a_019", "dataset": { "dataset": "nq", "source": "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/nq.zip", "source_sha256": "2553bf7bfbab47b1436ca00a34bce57320e18e611fd00999a2a3a1b4714be752", "source_md5": "d4d3d2e48787a744b6f6e691ff534307", "records": 124000, "queries": 100, "relevant_documents_retained": 120, "seed": 20261028, "license": "CC-BY-SA-3.0", "license_source": "https://raw.githubusercontent.com/google-research-datasets/natural-questions/master/nq_open/README.md", "attribution": "BEIR and original dataset authors; Wikipedia contributors for NQ, SciFact authors and cited article authors for SciFact.", "transformation": "200-word title+text prefix; seeded reservoir of documents with relevant documents for selected test queries retained at front. Simulated mutations; no historical replay.", "model": "sentence-transformers/all-MiniLM-L6-v2", "model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "dimensions": 384, "normalized": true, "max_tokens": 256, "device": "mps", "files": { "corpus.jsonl": "9dc487d785e6ee493eccaf764daa3bf8720a8f4fc8c406246065e1ad33a91f5d", "queries.json": "5603f52876ada5d36118b65dea490f1d43900bc159e1804e637796eb9115e999", "vectors.npy": "cb7421eb6ca5964231403f0a922843088f6fd0e01cbf67f2a1076cc7101005db", "query_vectors.npy": "5e666f8d966d8db6b0fd93d4963234d2dffa0ee3910b8fad069a8ee201e8b5d4", "UPSTREAM-LICENSE.md": "8b675521da3fa142ac6bda38c9bcfcc8cff0e2d6128be53a062ed55fb53f7b87" } }, "base_count": 10000, "profile": { "dataset": "nq", "base_count": 10000, "rate": 10, "query_rate": 10, "arrival": "steady" }, "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "preloaded_mutable": 200, "started_utc": "2026-10-29T03:02:43Z", "setup_s": 2.464031666982919, "backend_before": { "version": "1.40.3", "schema": { "class": "Ffb2429e9a019", "invertedIndexConfig": { "bm25": { "b": 0.75, "k1": 1.2 }, "cleanupIntervalSeconds": 60, "stopwords": { "additions": null, "preset": "en", "removals": null }, "usingBlockMaxWAND": true }, "multiTenancyConfig": { "autoTenantActivation": false, "autoTenantCreation": false, "enabled": false }, "properties": [ { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "doc_id", "tokenization": "field" }, { "dataType": [ "int" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": false, "name": "version" }, { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "cohort", "tokenization": "field" }, { "dataType": [ "text" ], "indexFilterable": false, "indexRangeFilters": false, "indexSearchable": false, "name": "text", "tokenization": "word" } ], "shardingConfig": { "actualCount": 1, "actualVirtualCount": 128, "desiredCount": 1, "desiredVirtualCount": 128, "function": "murmur3", "key": "_id", "strategy": "hash", "virtualPerPhysical": 128 }, "vectorIndexConfig": { "bq": { "enabled": false }, "cleanupIntervalSeconds": 300, "distance": "cosine", "dynamicEfFactor": 8, "dynamicEfMax": 500, "dynamicEfMin": 100, "ef": 128, "efConstruction": 128, "filterStrategy": "acorn", "flatSearchCutoff": 0, "maxConnections": 16, "multivector": { "aggregation": "maxSim", "enabled": false, "muvera": { "dprojections": 16, "enabled": false, "ksim": 4, "repetitions": 10 } }, "pq": { "bitCompression": false, "centroids": 256, "enabled": false, "encoder": { "distribution": "log-normal", "type": "kmeans" }, "segments": 0, "trainingLimit": 100000 }, "rq": { "bits": 8, "centering": false, "enabled": false, "rescoreLimit": 20, "trainingLimit": 10000 }, "skip": false, "skipDefaultQuantization": false, "sq": { "enabled": false, "rescoreLimit": 20, "trainingLimit": 100000 }, "trackDefaultQuantization": false, "vectorCacheMaxObjects": 1000000000000 }, "vectorIndexType": "hnsw", "vectorizer": "none", "replicationConfig": { "deletionStrategy": "TimeBasedResolution", "factor": 1, "asyncEnabled": false } }, "shards": [ { "name": "Kl74tG1yW6uk", "per_node_status": { "node1": "READY" }, "status": "READY", "vectorQueueSize": 0 } ], "ack": "synchronous indexing, object PUT/POST/DELETE", "transport": "HTTP REST writes + GraphQL search, pooled httpx" }, "count_before": 10200, "idle_duration_s": 10.0, "idle_queries": { "count": 100, "ann_recall": 0.996, "ndcg": 0.9481505328688745, "latency_ms": { "min": 5.234500044025481, "p50": 11.37537503382191, "p95": 13.3039646607358, "p99": 15.041862626094412, "max": 23.1009169947356 } }, "exact_ndcg": 0.9481505328688745, "resources_before": { "usage_usec": 198597971, "user_usec": 193724165, "system_usec": 4873805, "nice_usec": 0, "nr_periods": 5504, "nr_throttled": 246, "throttled_usec": 1085977, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 594632704, "pids": 11 }, "resources_after": { "usage_usec": 207648428, "user_usec": 202360567, "system_usec": 5287861, "nice_usec": 0, "nr_periods": 5805, "nr_throttled": 246, "throttled_usec": 1085977, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 598945792, "pids": 11 }, "count_after": 10200, "backend_after": { "version": "1.40.3", "schema": { "class": "Ffb2429e9a019", "invertedIndexConfig": { "bm25": { "b": 0.75, "k1": 1.2 }, "cleanupIntervalSeconds": 60, "stopwords": { "additions": null, "preset": "en", "removals": null }, "usingBlockMaxWAND": true }, "multiTenancyConfig": { "autoTenantActivation": false, "autoTenantCreation": false, "enabled": false }, "properties": [ { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "doc_id", "tokenization": "field" }, { "dataType": [ "int" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": false, "name": "version" }, { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "cohort", "tokenization": "field" }, { "dataType": [ "text" ], "indexFilterable": false, "indexRangeFilters": false, "indexSearchable": false, "name": "text", "tokenization": "word" } ], "shardingConfig": { "actualCount": 1, "actualVirtualCount": 128, "desiredCount": 1, "desiredVirtualCount": 128, "function": "murmur3", "key": "_id", "strategy": "hash", "virtualPerPhysical": 128 }, "vectorIndexConfig": { "bq": { "enabled": false }, "cleanupIntervalSeconds": 300, "distance": "cosine", "dynamicEfFactor": 8, "dynamicEfMax": 500, "dynamicEfMin": 100, "ef": 128, "efConstruction": 128, "filterStrategy": "acorn", "flatSearchCutoff": 0, "maxConnections": 16, "multivector": { "aggregation": "maxSim", "enabled": false, "muvera": { "dprojections": 16, "enabled": false, "ksim": 4, "repetitions": 10 } }, "pq": { "bitCompression": false, "centroids": 256, "enabled": false, "encoder": { "distribution": "log-normal", "type": "kmeans" }, "segments": 0, "trainingLimit": 100000 }, "rq": { "bits": 8, "centering": false, "enabled": false, "rescoreLimit": 20, "trainingLimit": 10000 }, "skip": false, "skipDefaultQuantization": false, "sq": { "enabled": false, "rescoreLimit": 20, "trainingLimit": 100000 }, "trackDefaultQuantization": false, "vectorCacheMaxObjects": 1000000000000 }, "vectorIndexType": "hnsw", "vectorizer": "none", "replicationConfig": { "deletionStrategy": "TimeBasedResolution", "factor": 1, "asyncEnabled": false } }, "shards": [ { "name": "Kl74tG1yW6uk", "per_node_status": { "node1": "READY" }, "status": "READY", "vectorQueueSize": 0 } ], "ack": "synchronous indexing, object PUT/POST/DELETE", "transport": "HTTP REST writes + GraphQL search, pooled httpx" } }, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "eligible": true, "protocol": "mixed-mutations-v2" }, "examples": [ { "event": 0, "operation": "insert", "doc_id": "mutation-000000", "scheduled_s": 0.0, "submitted_s": 0.00018554204143583775, "schedule_delay_ms": 0.18554204143583775, "precondition": true, "status": "success", "ack_ms": 4.993290989659727, "write_started_ms": 1.3333329698070884, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 14.346124953590333, "stable_since_ms": 14.346124953590333, "confirmed_ms": 66.68133300263435, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 5.415874999016523, "stable_since_ms": 5.415874999016523, "confirmed_ms": 61.84812495484948, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 14.346124953590333, "confirmed_ms": 66.68133300263435, "search_minus_ack_ms": 0.42258400935679674 }, { "event": 1, "operation": "update", "doc_id": "mutation-000001", "scheduled_s": 0.1, "submitted_s": 0.10011941695120186, "schedule_delay_ms": 0.11941695120185059, "precondition": true, "status": "success", "ack_ms": 5.678415996953845, "write_started_ms": 0.3963330527767539, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 6.205916055478156, "stable_since_ms": 6.205916055478156, "confirmed_ms": 61.39879103284329, "consecutive": 3, "regressions": 0, "requests": 3 }, "search": { "first_ms": 8.823500014841557, "stable_since_ms": 8.823500014841557, "confirmed_ms": 74.04920808039606, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 8.823500014841557, "confirmed_ms": 74.04920808039606, "search_minus_ack_ms": 3.1450840178877115 }, { "event": 2, "operation": "delete", "doc_id": "mutation-000002", "scheduled_s": 0.2, "submitted_s": 0.20018745795823634, "schedule_delay_ms": 0.1874579582363256, "precondition": true, "status": "success", "ack_ms": 4.164500045590103, "write_started_ms": 0.5251250695437193, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 4.2187090730294585, "stable_since_ms": 4.2187090730294585, "confirmed_ms": 59.805417084135115, "consecutive": 3, "regressions": 0, "requests": 3 }, "search": { "first_ms": 9.959625080227852, "stable_since_ms": 9.959625080227852, "confirmed_ms": 76.22904202435166, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 9.959625080227852, "confirmed_ms": 76.22904202435166, "search_minus_ack_ms": 5.795125034637749 } ] }, { "run": "fb2_429e9a_020", "trial": 3, "backend": "qdrant", "scenario": "nq-10k", "status": "complete", "eligible": true, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "summary": { "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "wall_s": 30.054821708938107, "operations": { "insert": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 15.56520804297179, "p50": 67.68937455490232, "p95": 71.65691102272831, "p99": 104.70977704389956, "max": 113.42445900663733 }, "confirmed_ms": { "min": 109.04466698411852, "p50": 213.24154152534902, "p95": 218.27049792627804, "p99": 231.28645037999394, "max": 261.6869170451537 }, "ack_ms": { "min": 4.541915957815945, "p50": 9.910541528370231, "p95": 13.160691305529326, "p99": 25.599405991379175, "max": 26.41516597941518 }, "search_minus_ack_ms": { "min": -1.8997499719262123, "p50": 57.17062501935288, "p95": 60.629183356650174, "p99": 95.85932782851161, "max": 98.0532499961555 }, "regressions": 0, "confirmed_within_100ms": 0.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "update": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 18.235958996228874, "p50": 66.51135405991226, "p95": 69.16556035284884, "p99": 69.94589800597169, "max": 70.2327910112217 }, "confirmed_ms": { "min": 126.21929205488414, "p50": 211.77543752128258, "p95": 216.60936185508035, "p99": 217.03073829063217, "max": 224.19470909517258 }, "ack_ms": { "min": 4.7188750468194485, "p50": 9.389124985318631, "p95": 11.125067004468292, "p99": 11.825232487171892, "max": 12.669166084378958 }, "search_minus_ack_ms": { "min": 1.5649170381948352, "p50": 56.79895845241845, "p95": 59.17027741088532, "p99": 60.012917628046125, "max": 60.429667006246746 }, "regressions": 0, "confirmed_within_100ms": 0.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "delete": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 13.477790984325111, "p50": 18.166500551160425, "p95": 21.049419423798096, "p99": 68.44010323402479, "max": 82.54949992988259 }, "confirmed_ms": { "min": 106.08845797833055, "p50": 151.2937499792315, "p95": 157.74073152570054, "p99": 213.94147323095245, "max": 227.65437501948327 }, "ack_ms": { "min": 2.96979199629277, "p50": 8.059396059252322, "p95": 10.154249938204885, "p99": 12.135430099442676, "max": 26.43816696945578 }, "search_minus_ack_ms": { "min": -3.1594160245731473, "p50": 0.2130205393768847, "p95": 3.092607943108305, "p99": 57.2813398076687, "max": 57.360416976734996 }, "regressions": 0, "confirmed_within_100ms": 0.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 } }, "total_operations": 300, "statuses": { "success": 300, "timeout": 0, "error": 0, "invalid_precondition": 0, "client_overload": 0 }, "query": { "count": 300, "success": 300, "ann_recall_at_10": 0.9970000000000001, "ndcg_at_10": 0.948294881680092, "latency_ms": { "min": 3.3878330141305923, "p50": 8.408332942053676, "p95": 10.9046729747206, "p99": 49.44943160749971, "max": 56.36987497564405 }, "dispatch_delay_ms": { "min": 0.14654197730123997, "p50": 1.4109374023973942, "p95": 2.2241068072617054, "p99": 2.584665665635839, "max": 3.372416947968304 } }, "diagnostics": { "schedule_delay_ms": { "min": 0.0008749077089476032, "p50": 0.3930630045942962, "p95": 0.7492957916110043, "p99": 0.7778847622692098, "max": 1.104834000580368 }, "probe_dispatch_ms": { "min": 0.03287498839199543, "p50": 0.45447901356965303, "p95": 1.6671875782776624, "p99": 2.376362488139422, "max": 9.619041928090155 }, "probe_service_ms": { "min": 1.0824999772012234, "p50": 6.092937546782196, "p95": 48.23757728445344, "p99": 49.20042003272101, "max": 57.485709083266556 }, "write_service_ms": { "min": 2.737874980084598, "p50": 8.536125009413809, "p95": 11.284991959109902, "p99": 17.05873117316507, "max": 25.854415958747268 } }, "actual_submission_rate": 9.999922325022304, "metadata": { "backend": "qdrant", "collection": "fb2_429e9a_020", "dataset": { "dataset": "nq", "source": "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/nq.zip", "source_sha256": "2553bf7bfbab47b1436ca00a34bce57320e18e611fd00999a2a3a1b4714be752", "source_md5": "d4d3d2e48787a744b6f6e691ff534307", "records": 124000, "queries": 100, "relevant_documents_retained": 120, "seed": 20261028, "license": "CC-BY-SA-3.0", "license_source": "https://raw.githubusercontent.com/google-research-datasets/natural-questions/master/nq_open/README.md", "attribution": "BEIR and original dataset authors; Wikipedia contributors for NQ, SciFact authors and cited article authors for SciFact.", "transformation": "200-word title+text prefix; seeded reservoir of documents with relevant documents for selected test queries retained at front. Simulated mutations; no historical replay.", "model": "sentence-transformers/all-MiniLM-L6-v2", "model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "dimensions": 384, "normalized": true, "max_tokens": 256, "device": "mps", "files": { "corpus.jsonl": "9dc487d785e6ee493eccaf764daa3bf8720a8f4fc8c406246065e1ad33a91f5d", "queries.json": "5603f52876ada5d36118b65dea490f1d43900bc159e1804e637796eb9115e999", "vectors.npy": "cb7421eb6ca5964231403f0a922843088f6fd0e01cbf67f2a1076cc7101005db", "query_vectors.npy": "5e666f8d966d8db6b0fd93d4963234d2dffa0ee3910b8fad069a8ee201e8b5d4", "UPSTREAM-LICENSE.md": "8b675521da3fa142ac6bda38c9bcfcc8cff0e2d6128be53a062ed55fb53f7b87" } }, "base_count": 10000, "profile": { "dataset": "nq", "base_count": 10000, "rate": 10, "query_rate": 10, "arrival": "steady" }, "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "preloaded_mutable": 200, "started_utc": "2026-10-29T03:03:30Z", "setup_s": 3.4409495409345254, "backend_before": { "version": "1.20.0", "collection": { "status": "green", "optimizer_status": "ok", "indexed_vectors_count": 10200, "points_count": 10200, "segments_count": 2, "config": { "params": { "vectors": { "size": 384, "distance": "Cosine" }, "shard_number": 1, "replication_factor": 1, "write_consistency_factor": 1, "on_disk_payload": true }, "hnsw_config": { "m": 16, "ef_construct": 128, "full_scan_threshold": 10, "max_indexing_threads": 0, "on_disk": false }, "optimizer_config": { "deleted_threshold": 0.2, "vacuum_min_vector_number": 1000, "default_segment_number": 0, "max_segment_size": null, "memmap_threshold": null, "indexing_threshold": 1024, "flush_interval_sec": 5, "max_optimization_threads": null, "prevent_unoptimized": null }, "wal_config": { "wal_capacity_mb": 32, "wal_segments_ahead": 0, "wal_retain_closed": 1 }, "quantization_config": null }, "payload_schema": { "cohort": { "data_type": "keyword", "points": 10200 } }, "update_queue": { "length": 0 }, "created_at": "2026-10-29T03:03:30.348Z" }, "ack": "wait=true", "transport": "HTTP REST, pooled httpx" }, "count_before": 10200, "idle_duration_s": 10.0, "idle_queries": { "count": 100, "ann_recall": 0.997, "ndcg": 0.948294881680092, "latency_ms": { "min": 4.821917042136192, "p50": 7.9695829772390425, "p95": 9.784028783906251, "p99": 36.48186427773914, "max": 48.587708035483956 } }, "exact_ndcg": 0.9481505328688745, "resources_before": { "usage_usec": 90150426, "user_usec": 84440099, "system_usec": 5710327, "nice_usec": 59241000, "nr_periods": 2486, "nr_throttled": 85, "throttled_usec": 330293, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 371085312, "pids": 31 }, "resources_after": { "usage_usec": 96158574, "user_usec": 89727836, "system_usec": 6430738, "nice_usec": 59241000, "nr_periods": 2788, "nr_throttled": 85, "throttled_usec": 330293, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 367099904, "pids": 32 }, "count_after": 10200, "backend_after": { "version": "1.20.0", "collection": { "status": "green", "optimizer_status": "ok", "indexed_vectors_count": 10200, "points_count": 10200, "segments_count": 2, "config": { "params": { "vectors": { "size": 384, "distance": "Cosine" }, "shard_number": 1, "replication_factor": 1, "write_consistency_factor": 1, "on_disk_payload": true }, "hnsw_config": { "m": 16, "ef_construct": 128, "full_scan_threshold": 10, "max_indexing_threads": 0, "on_disk": false }, "optimizer_config": { "deleted_threshold": 0.2, "vacuum_min_vector_number": 1000, "default_segment_number": 0, "max_segment_size": null, "memmap_threshold": null, "indexing_threshold": 1024, "flush_interval_sec": 5, "max_optimization_threads": null, "prevent_unoptimized": null }, "wal_config": { "wal_capacity_mb": 32, "wal_segments_ahead": 0, "wal_retain_closed": 1 }, "quantization_config": null }, "payload_schema": { "cohort": { "data_type": "keyword", "points": 10200 } }, "update_queue": { "length": 0 }, "created_at": "2026-10-29T03:03:30.348Z" }, "ack": "wait=true", "transport": "HTTP REST, pooled httpx" } }, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "eligible": true, "protocol": "mixed-mutations-v2" }, "examples": [ { "event": 0, "operation": "insert", "doc_id": "mutation-000000", "scheduled_s": 0.0, "submitted_s": 0.00023945898283272982, "schedule_delay_ms": 0.23945898283272982, "precondition": true, "status": "success", "ack_ms": 5.99358300678432, "write_started_ms": 1.290749991312623, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 14.728790963999927, "stable_since_ms": 14.728790963999927, "confirmed_ms": 69.04299999587238, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 61.92887492943555, "stable_since_ms": 61.92887492943555, "confirmed_ms": 205.4090410238132, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 61.92887492943555, "confirmed_ms": 205.4090410238132, "search_minus_ack_ms": 55.93529192265123 }, { "event": 1, "operation": "update", "doc_id": "mutation-000001", "scheduled_s": 0.1, "submitted_s": 0.10058949992526323, "schedule_delay_ms": 0.5894999252632205, "precondition": true, "status": "success", "ack_ms": 8.385792025364935, "write_started_ms": 1.3447500532492995, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 16.34729199577123, "stable_since_ms": 16.34729199577123, "confirmed_ms": 73.22958402801305, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 63.96704202052206, "stable_since_ms": 63.96704202052206, "confirmed_ms": 211.1735420767218, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 63.96704202052206, "confirmed_ms": 211.1735420767218, "search_minus_ack_ms": 55.58124999515712 }, { "event": 2, "operation": "delete", "doc_id": "mutation-000002", "scheduled_s": 0.2, "submitted_s": 0.20076787495054305, "schedule_delay_ms": 0.7678749505430349, "precondition": true, "status": "success", "ack_ms": 9.946584003046155, "write_started_ms": 1.2506250059232116, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 19.61287495214492, "stable_since_ms": 19.61287495214492, "confirmed_ms": 75.7301669800654, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 10.158417047932744, "stable_since_ms": 10.158417047932744, "confirmed_ms": 155.61133401934057, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 19.61287495214492, "confirmed_ms": 155.61133401934057, "search_minus_ack_ms": 0.21183304488658905 } ] }, { "run": "fb2_429e9a_021", "trial": 2, "backend": "qdrant", "scenario": "nq-100k-burst", "status": "complete", "eligible": true, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "summary": { "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "burst", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "wall_s": 30.061617291998118, "operations": { "insert": { "count": 130, "success": 130, "success_rate": 1.0, "first_observed_ms": { "min": 14.043624978512526, "p50": 68.75718757510185, "p95": 105.21178120397963, "p99": 111.04856995167208, "max": 114.01137500070035 }, "confirmed_ms": { "min": 109.80437498074025, "p50": 217.14047895511612, "p95": 243.3339831593912, "p99": 254.69005684484725, "max": 256.8663329584524 }, "ack_ms": { "min": 2.91216594632715, "p50": 8.062666980549693, "p95": 12.557989911874756, "p99": 14.670395342400283, "max": 16.131625045090914 }, "search_minus_ack_ms": { "min": 1.7895420314744115, "p50": 58.77258302643895, "p95": 98.46798951621167, "p99": 102.54612921271472, "max": 108.97241695784032 }, "regressions": 0, "confirmed_within_100ms": 0.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "update": { "count": 130, "success": 130, "success_rate": 1.0, "first_observed_ms": { "min": 17.216083011589944, "p50": 69.06212499598041, "p95": 102.86008297116496, "p99": 107.71036814432588, "max": 111.46062496118248 }, "confirmed_ms": { "min": 107.8080830629915, "p50": 217.37108350498602, "p95": 244.0894018043764, "p99": 255.19929985748598, "max": 261.2605419708416 }, "ack_ms": { "min": 2.3285410134121776, "p50": 9.920146025251597, "p95": 12.563773145666346, "p99": 15.901249231537781, "max": 40.217667003162205 }, "search_minus_ack_ms": { "min": 1.410915981978178, "p50": 57.943541498389095, "p95": 96.53091470245272, "p99": 98.4610727604013, "max": 99.07999995630234 }, "regressions": 0, "confirmed_within_100ms": 0.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "delete": { "count": 130, "success": 130, "success_rate": 1.0, "first_observed_ms": { "min": 8.163541904650629, "p50": 19.267666444648057, "p95": 46.640510723227635, "p99": 47.98697952297516, "max": 107.70091705489904 }, "confirmed_ms": { "min": 109.50987495016307, "p50": 157.7140205190517, "p95": 187.9890499811154, "p99": 191.70678650378247, "max": 251.44625001121312 }, "ack_ms": { "min": 2.6307080406695604, "p50": 7.98602047143504, "p95": 11.469214485259727, "p99": 12.521788915619252, "max": 18.943167058750987 }, "search_minus_ack_ms": { "min": -3.2711250241845846, "p50": 0.7665000157430768, "p95": 42.47181909158826, "p99": 43.14449375029653, "max": 99.31908396538347 }, "regressions": 0, "confirmed_within_100ms": 0.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 } }, "total_operations": 390, "statuses": { "success": 390, "timeout": 0, "error": 0, "invalid_precondition": 0, "client_overload": 0 }, "query": { "count": 300, "success": 300, "ann_recall_at_10": 0.9929999999999999, "ndcg_at_10": 0.7931268675492565, "latency_ms": { "min": 4.084583022631705, "p50": 9.76095802616328, "p95": 47.37376217963174, "p99": 49.91635785088873, "max": 54.20045799110085 }, "dispatch_delay_ms": { "min": 0.14266697689890862, "p50": 1.403770933393389, "p95": 2.176056138705462, "p99": 2.4323632323648776, "max": 2.521541900932789 } }, "diagnostics": { "schedule_delay_ms": { "min": 0.006666919216691269, "p50": 0.3797709476209832, "p95": 0.7461627654279822, "p99": 0.7998562059366784, "max": 1.5142499003566456 }, "probe_dispatch_ms": { "min": 0.02508307807147503, "p50": 0.47002098290249705, "p95": 1.5876188350375737, "p99": 2.2423749859444793, "max": 9.53983305953443 }, "probe_service_ms": { "min": 0.8357500191777945, "p50": 6.02535397047177, "p95": 49.0896440402139, "p99": 50.75782753294334, "max": 56.15383305121213 }, "write_service_ms": { "min": 2.19912501052022, "p50": 7.976354099810123, "p95": 11.83681035763584, "p99": 14.877203425858179, "max": 39.71845901105553 } }, "actual_submission_rate": 13.00981833681025, "metadata": { "backend": "qdrant", "collection": "fb2_429e9a_021", "dataset": { "dataset": "nq", "source": "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/nq.zip", "source_sha256": "2553bf7bfbab47b1436ca00a34bce57320e18e611fd00999a2a3a1b4714be752", "source_md5": "d4d3d2e48787a744b6f6e691ff534307", "records": 124000, "queries": 100, "relevant_documents_retained": 120, "seed": 20261028, "license": "CC-BY-SA-3.0", "license_source": "https://raw.githubusercontent.com/google-research-datasets/natural-questions/master/nq_open/README.md", "attribution": "BEIR and original dataset authors; Wikipedia contributors for NQ, SciFact authors and cited article authors for SciFact.", "transformation": "200-word title+text prefix; seeded reservoir of documents with relevant documents for selected test queries retained at front. Simulated mutations; no historical replay.", "model": "sentence-transformers/all-MiniLM-L6-v2", "model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "dimensions": 384, "normalized": true, "max_tokens": 256, "device": "mps", "files": { "corpus.jsonl": "9dc487d785e6ee493eccaf764daa3bf8720a8f4fc8c406246065e1ad33a91f5d", "queries.json": "5603f52876ada5d36118b65dea490f1d43900bc159e1804e637796eb9115e999", "vectors.npy": "cb7421eb6ca5964231403f0a922843088f6fd0e01cbf67f2a1076cc7101005db", "query_vectors.npy": "5e666f8d966d8db6b0fd93d4963234d2dffa0ee3910b8fad069a8ee201e8b5d4", "UPSTREAM-LICENSE.md": "8b675521da3fa142ac6bda38c9bcfcc8cff0e2d6128be53a062ed55fb53f7b87" } }, "base_count": 100000, "profile": { "dataset": "nq", "base_count": 100000, "rate": 10, "query_rate": 10, "arrival": "burst" }, "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "burst", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "preloaded_mutable": 260, "started_utc": "2026-10-29T03:04:19Z", "setup_s": 42.89376666606404, "backend_before": { "version": "1.20.0", "collection": { "status": "green", "optimizer_status": "ok", "indexed_vectors_count": 100260, "points_count": 100260, "segments_count": 2, "config": { "params": { "vectors": { "size": 384, "distance": "Cosine" }, "shard_number": 1, "replication_factor": 1, "write_consistency_factor": 1, "on_disk_payload": true }, "hnsw_config": { "m": 16, "ef_construct": 128, "full_scan_threshold": 10, "max_indexing_threads": 0, "on_disk": false }, "optimizer_config": { "deleted_threshold": 0.2, "vacuum_min_vector_number": 1000, "default_segment_number": 0, "max_segment_size": null, "memmap_threshold": null, "indexing_threshold": 1024, "flush_interval_sec": 5, "max_optimization_threads": null, "prevent_unoptimized": null }, "wal_config": { "wal_capacity_mb": 32, "wal_segments_ahead": 0, "wal_retain_closed": 1 }, "quantization_config": null }, "payload_schema": { "cohort": { "data_type": "keyword", "points": 100260 } }, "update_queue": { "length": 0 }, "created_at": "2026-10-29T03:04:19.860Z" }, "ack": "wait=true", "transport": "HTTP REST, pooled httpx" }, "count_before": 100260, "idle_duration_s": 10.0, "idle_queries": { "count": 100, "ann_recall": 0.993, "ndcg": 0.7931268675492565, "latency_ms": { "min": 4.861667053773999, "p50": 9.76433395408094, "p95": 11.264227709034456, "p99": 22.655054669594563, "max": 55.69766601547599 } }, "exact_ndcg": 0.7931268675492565, "resources_before": { "usage_usec": 159338081, "user_usec": 150676677, "system_usec": 8661403, "nice_usec": 115728000, "nr_periods": 3359, "nr_throttled": 228, "throttled_usec": 1203899, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 617238528, "pids": 28 }, "resources_after": { "usage_usec": 166791395, "user_usec": 157294661, "system_usec": 9496734, "nice_usec": 115728000, "nr_periods": 3661, "nr_throttled": 228, "throttled_usec": 1203899, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 599740416, "pids": 32 }, "count_after": 100260, "backend_after": { "version": "1.20.0", "collection": { "status": "green", "optimizer_status": "ok", "indexed_vectors_count": 100260, "points_count": 100260, "segments_count": 2, "config": { "params": { "vectors": { "size": 384, "distance": "Cosine" }, "shard_number": 1, "replication_factor": 1, "write_consistency_factor": 1, "on_disk_payload": true }, "hnsw_config": { "m": 16, "ef_construct": 128, "full_scan_threshold": 10, "max_indexing_threads": 0, "on_disk": false }, "optimizer_config": { "deleted_threshold": 0.2, "vacuum_min_vector_number": 1000, "default_segment_number": 0, "max_segment_size": null, "memmap_threshold": null, "indexing_threshold": 1024, "flush_interval_sec": 5, "max_optimization_threads": null, "prevent_unoptimized": null }, "wal_config": { "wal_capacity_mb": 32, "wal_segments_ahead": 0, "wal_retain_closed": 1 }, "quantization_config": null }, "payload_schema": { "cohort": { "data_type": "keyword", "points": 100260 } }, "update_queue": { "length": 0 }, "created_at": "2026-10-29T03:04:19.860Z" }, "ack": "wait=true", "transport": "HTTP REST, pooled httpx" } }, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "eligible": true, "protocol": "mixed-mutations-v2" }, "examples": [ { "event": 0, "operation": "insert", "doc_id": "mutation-000000", "scheduled_s": 0.0, "submitted_s": 0.00025287491735070944, "schedule_delay_ms": 0.25287491735070944, "precondition": true, "status": "success", "ack_ms": 5.661875009536743, "write_started_ms": 1.5267920680344105, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 14.84837499447167, "stable_since_ms": 14.84837499447167, "confirmed_ms": 68.63491702824831, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 60.53837505169213, "stable_since_ms": 60.53837505169213, "confirmed_ms": 202.67545897513628, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 60.53837505169213, "confirmed_ms": 202.67545897513628, "search_minus_ack_ms": 54.876500042155385 }, { "event": 1, "operation": "update", "doc_id": "mutation-000001", "scheduled_s": 0.025, "submitted_s": 0.025563208968378603, "schedule_delay_ms": 0.5632089683786021, "precondition": true, "status": "success", "ack_ms": 3.348332946188748, "write_started_ms": 0.5770409479737282, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 14.64474992826581, "stable_since_ms": 14.64474992826581, "confirmed_ms": 67.84337502904236, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 99.03220797423273, "stable_since_ms": 99.03220797423273, "confirmed_ms": 239.3325000302866, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 99.03220797423273, "confirmed_ms": 239.3325000302866, "search_minus_ack_ms": 95.68387502804399 }, { "event": 2, "operation": "delete", "doc_id": "mutation-000002", "scheduled_s": 0.05, "submitted_s": 0.05005774996243417, "schedule_delay_ms": 0.057749962434169855, "precondition": true, "status": "success", "ack_ms": 4.779667011462152, "write_started_ms": 0.724916928447783, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 13.57537496369332, "stable_since_ms": 13.57537496369332, "confirmed_ms": 68.87166702654213, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 45.4877499723807, "stable_since_ms": 45.4877499723807, "confirmed_ms": 146.98816696181893, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 45.4877499723807, "confirmed_ms": 146.98816696181893, "search_minus_ack_ms": 40.708082960918546 } ] }, { "run": "fb2_429e9a_022", "trial": 3, "backend": "qdrant", "scenario": "nq-100k-burst", "status": "complete", "eligible": true, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "summary": { "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "burst", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "wall_s": 30.064418375026435, "operations": { "insert": { "count": 130, "success": 130, "success_rate": 1.0, "first_observed_ms": { "min": 11.853792006149888, "p50": 69.18656296329573, "p95": 105.39951414102688, "p99": 110.08874783175997, "max": 110.67612504120916 }, "confirmed_ms": { "min": 113.39149996638298, "p50": 217.12818747619167, "p95": 246.85336668626405, "p99": 256.6976868675556, "max": 259.14829096291214 }, "ack_ms": { "min": 2.500167000107467, "p50": 8.271562517620623, "p95": 12.944162730127573, "p99": 14.063877479638904, "max": 14.97508306056261 }, "search_minus_ack_ms": { "min": 0.6299999076873064, "p50": 59.074833523482084, "p95": 97.71164988633245, "p99": 100.52149654133245, "max": 100.96004197839648 }, "regressions": 0, "confirmed_within_100ms": 0.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "update": { "count": 130, "success": 130, "success_rate": 1.0, "first_observed_ms": { "min": 17.11341692134738, "p50": 69.50093753403053, "p95": 102.69042053259909, "p99": 108.33456037216816, "max": 112.51025006640702 }, "confirmed_ms": { "min": 110.98829098045826, "p50": 216.54410450719297, "p95": 244.14518572739325, "p99": 255.70298142614766, "max": 260.92116604559124 }, "ack_ms": { "min": 2.951791975647211, "p50": 9.583103994373232, "p95": 12.706492008874193, "p99": 14.42644122522324, "max": 23.06154102552682 }, "search_minus_ack_ms": { "min": 1.1835829354822636, "p50": 58.30916651757434, "p95": 96.96826710714959, "p99": 98.68864232092164, "max": 102.4663340067491 }, "regressions": 0, "confirmed_within_100ms": 0.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "delete": { "count": 130, "success": 130, "success_rate": 1.0, "first_observed_ms": { "min": 13.548125047236681, "p50": 19.385083520319313, "p95": 48.08741247979924, "p99": 54.27804330945944, "max": 73.00099998246878 }, "confirmed_ms": { "min": 104.8613750608638, "p50": 158.3220204920508, "p95": 188.5254562657792, "p99": 199.52030940912664, "max": 217.5987500231713 }, "ack_ms": { "min": 2.5318749248981476, "p50": 8.109249989502132, "p95": 12.134712346596643, "p99": 17.90207161335273, "max": 20.99970809649676 }, "search_minus_ack_ms": { "min": -1.8285410478711128, "p50": 0.4446460516192019, "p95": 42.28788141044788, "p99": 43.816271689720445, "max": 54.5124999480322 }, "regressions": 0, "confirmed_within_100ms": 0.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 } }, "total_operations": 390, "statuses": { "success": 390, "timeout": 0, "error": 0, "invalid_precondition": 0, "client_overload": 0 }, "query": { "count": 300, "success": 300, "ann_recall_at_10": 0.991, "ndcg_at_10": 0.793271216360474, "latency_ms": { "min": 3.9091670187190175, "p50": 9.936562040820718, "p95": 49.00786668877117, "p99": 51.41776888398453, "max": 65.17991598229855 }, "dispatch_delay_ms": { "min": 0.0889170914888382, "p50": 1.3550835428759456, "p95": 2.2591256943996996, "p99": 2.5423474470153447, "max": 6.645542103797197 } }, "diagnostics": { "schedule_delay_ms": { "min": 0.004292069935019072, "p50": 0.38950054440611837, "p95": 0.7621358043975845, "p99": 0.8375231446705028, "max": 6.058000097982408 }, "probe_dispatch_ms": { "min": 0.022000051103532314, "p50": 0.4645834560506046, "p95": 1.5476750559173522, "p99": 2.3114556632935948, "max": 3.3417079830542207 }, "probe_service_ms": { "min": 0.76299998909235, "p50": 5.9593330370262265, "p95": 49.014640104724094, "p99": 50.75392852071673, "max": 71.09012501314282 }, "write_service_ms": { "min": 2.328749978914857, "p50": 8.036207989789546, "p95": 12.036493740743023, "p99": 14.71527505549603, "max": 22.491333074867725 } }, "actual_submission_rate": 13.009887337260459, "metadata": { "backend": "qdrant", "collection": "fb2_429e9a_022", "dataset": { "dataset": "nq", "source": "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/nq.zip", "source_sha256": "2553bf7bfbab47b1436ca00a34bce57320e18e611fd00999a2a3a1b4714be752", "source_md5": "d4d3d2e48787a744b6f6e691ff534307", "records": 124000, "queries": 100, "relevant_documents_retained": 120, "seed": 20261028, "license": "CC-BY-SA-3.0", "license_source": "https://raw.githubusercontent.com/google-research-datasets/natural-questions/master/nq_open/README.md", "attribution": "BEIR and original dataset authors; Wikipedia contributors for NQ, SciFact authors and cited article authors for SciFact.", "transformation": "200-word title+text prefix; seeded reservoir of documents with relevant documents for selected test queries retained at front. Simulated mutations; no historical replay.", "model": "sentence-transformers/all-MiniLM-L6-v2", "model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "dimensions": 384, "normalized": true, "max_tokens": 256, "device": "mps", "files": { "corpus.jsonl": "9dc487d785e6ee493eccaf764daa3bf8720a8f4fc8c406246065e1ad33a91f5d", "queries.json": "5603f52876ada5d36118b65dea490f1d43900bc159e1804e637796eb9115e999", "vectors.npy": "cb7421eb6ca5964231403f0a922843088f6fd0e01cbf67f2a1076cc7101005db", "query_vectors.npy": "5e666f8d966d8db6b0fd93d4963234d2dffa0ee3910b8fad069a8ee201e8b5d4", "UPSTREAM-LICENSE.md": "8b675521da3fa142ac6bda38c9bcfcc8cff0e2d6128be53a062ed55fb53f7b87" } }, "base_count": 100000, "profile": { "dataset": "nq", "base_count": 100000, "rate": 10, "query_rate": 10, "arrival": "burst" }, "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "burst", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "preloaded_mutable": 260, "started_utc": "2026-10-29T03:05:49Z", "setup_s": 43.11326970800292, "backend_before": { "version": "1.20.0", "collection": { "status": "green", "optimizer_status": "ok", "indexed_vectors_count": 100260, "points_count": 100260, "segments_count": 2, "config": { "params": { "vectors": { "size": 384, "distance": "Cosine" }, "shard_number": 1, "replication_factor": 1, "write_consistency_factor": 1, "on_disk_payload": true }, "hnsw_config": { "m": 16, "ef_construct": 128, "full_scan_threshold": 10, "max_indexing_threads": 0, "on_disk": false }, "optimizer_config": { "deleted_threshold": 0.2, "vacuum_min_vector_number": 1000, "default_segment_number": 0, "max_segment_size": null, "memmap_threshold": null, "indexing_threshold": 1024, "flush_interval_sec": 5, "max_optimization_threads": null, "prevent_unoptimized": null }, "wal_config": { "wal_capacity_mb": 32, "wal_segments_ahead": 0, "wal_retain_closed": 1 }, "quantization_config": null }, "payload_schema": { "cohort": { "data_type": "keyword", "points": 100260 } }, "update_queue": { "length": 0 }, "created_at": "2026-10-29T03:05:49.325Z" }, "ack": "wait=true", "transport": "HTTP REST, pooled httpx" }, "count_before": 100260, "idle_duration_s": 10.0, "idle_queries": { "count": 100, "ann_recall": 0.991, "ndcg": 0.793271216360474, "latency_ms": { "min": 4.297957988455892, "p50": 9.705749514978379, "p95": 11.063966614892706, "p99": 19.431270927889422, "max": 48.06087492033839 } }, "exact_ndcg": 0.7931268675492565, "resources_before": { "usage_usec": 230282211, "user_usec": 218535850, "system_usec": 11746361, "nice_usec": 172396000, "nr_periods": 4237, "nr_throttled": 380, "throttled_usec": 2075967, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 634396672, "pids": 30 }, "resources_after": { "usage_usec": 237713318, "user_usec": 225143854, "system_usec": 12569463, "nice_usec": 172396000, "nr_periods": 4539, "nr_throttled": 380, "throttled_usec": 2075967, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 612610048, "pids": 31 }, "count_after": 100260, "backend_after": { "version": "1.20.0", "collection": { "status": "green", "optimizer_status": "ok", "indexed_vectors_count": 100260, "points_count": 100260, "segments_count": 2, "config": { "params": { "vectors": { "size": 384, "distance": "Cosine" }, "shard_number": 1, "replication_factor": 1, "write_consistency_factor": 1, "on_disk_payload": true }, "hnsw_config": { "m": 16, "ef_construct": 128, "full_scan_threshold": 10, "max_indexing_threads": 0, "on_disk": false }, "optimizer_config": { "deleted_threshold": 0.2, "vacuum_min_vector_number": 1000, "default_segment_number": 0, "max_segment_size": null, "memmap_threshold": null, "indexing_threshold": 1024, "flush_interval_sec": 5, "max_optimization_threads": null, "prevent_unoptimized": null }, "wal_config": { "wal_capacity_mb": 32, "wal_segments_ahead": 0, "wal_retain_closed": 1 }, "quantization_config": null }, "payload_schema": { "cohort": { "data_type": "keyword", "points": 100260 } }, "update_queue": { "length": 0 }, "created_at": "2026-10-29T03:05:49.325Z" }, "ack": "wait=true", "transport": "HTTP REST, pooled httpx" } }, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "eligible": true, "protocol": "mixed-mutations-v2" }, "examples": [ { "event": 0, "operation": "insert", "doc_id": "mutation-000000", "scheduled_s": 0.0, "submitted_s": 0.00019979209173470736, "schedule_delay_ms": 0.19979209173470736, "precondition": true, "status": "success", "ack_ms": 6.8359579890966415, "write_started_ms": 1.329749939031899, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 14.645832940004766, "stable_since_ms": 14.645832940004766, "confirmed_ms": 67.94966699089855, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 60.54608291015029, "stable_since_ms": 60.54608291015029, "confirmed_ms": 160.36954196169972, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 60.54608291015029, "confirmed_ms": 160.36954196169972, "search_minus_ack_ms": 53.71012492105365 }, { "event": 1, "operation": "update", "doc_id": "mutation-000001", "scheduled_s": 0.025, "submitted_s": 0.025660542072728276, "schedule_delay_ms": 0.6605420727282749, "precondition": true, "status": "success", "ack_ms": 3.6833329359069467, "write_started_ms": 0.6790419574826956, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 14.770666952244937, "stable_since_ms": 14.770666952244937, "confirmed_ms": 68.3619580231607, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 99.51116691809148, "stable_since_ms": 99.51116691809148, "confirmed_ms": 242.7994169993326, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 99.51116691809148, "confirmed_ms": 242.7994169993326, "search_minus_ack_ms": 95.82783398218453 }, { "event": 2, "operation": "delete", "doc_id": "mutation-000002", "scheduled_s": 0.05, "submitted_s": 0.05024675000458956, "schedule_delay_ms": 0.24675000458955487, "precondition": true, "status": "success", "ack_ms": 3.077167086303234, "write_started_ms": 0.21162501070648432, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 14.06520907767117, "stable_since_ms": 14.06520907767117, "confirmed_ms": 68.09454201720655, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 4.77787503041327, "stable_since_ms": 4.77787503041327, "confirmed_ms": 147.3697500769049, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 14.06520907767117, "confirmed_ms": 147.3697500769049, "search_minus_ack_ms": 1.700707944110036 } ] }, { "run": "fb2_429e9a_023", "trial": 5, "backend": "pgvector", "scenario": "nq-100k", "status": "complete", "eligible": true, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "summary": { "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "wall_s": 30.001014166977257, "operations": { "insert": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 18.613749998621643, "p50": 29.98200000729412, "p95": 34.604297328041866, "p99": 36.37610270059668, "max": 37.24420804064721 }, "confirmed_ms": { "min": 74.31191601790488, "p50": 90.83866648143157, "p95": 96.78471490042284, "p99": 102.35200934112072, "max": 103.00050000660121 }, "ack_ms": { "min": 5.210708943195641, "p50": 18.86562502477318, "p95": 23.770862782839686, "p99": 25.90884794248268, "max": 26.203166926279664 }, "search_minus_ack_ms": { "min": 8.084749919362366, "p50": 11.432250496000051, "p95": 13.216880889376625, "p99": 15.646889642812313, "max": 15.94950002618134 }, "regressions": 0, "confirmed_within_100ms": 0.98, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "update": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 16.111499979160726, "p50": 27.327666466590017, "p95": 33.861366065684706, "p99": 34.92542133899407, "max": 37.06683299969882 }, "confirmed_ms": { "min": 72.96954095363617, "p50": 87.82177098328248, "p95": 94.33021912700497, "p99": 99.01737715932542, "max": 101.13787499722093 }, "ack_ms": { "min": 3.603791003115475, "p50": 15.747541503515095, "p95": 21.56914344523102, "p99": 24.454910950735226, "max": 27.185124927200377 }, "search_minus_ack_ms": { "min": 7.482375018298626, "p50": 11.869645968545228, "p95": 12.793543777661398, "p99": 13.368914828170091, "max": 13.438790920190513 }, "regressions": 0, "confirmed_within_100ms": 0.99, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "delete": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 12.017000000923872, "p50": 16.0689169424586, "p95": 31.244569044793025, "p99": 33.08105924399572, "max": 36.53954097535461 }, "confirmed_ms": { "min": 63.55870794504881, "p50": 79.97385447379202, "p95": 95.94862319063395, "p99": 100.87411834741943, "max": 102.12333302479237 }, "ack_ms": { "min": 1.2244170065969229, "p50": 2.35083291772753, "p95": 4.177454189630225, "p99": 5.480047812452541, "max": 5.5672499584034085 }, "search_minus_ack_ms": { "min": 3.428499912843108, "p50": 13.686562480870634, "p95": 27.933885372476652, "p99": 29.769567981129534, "max": 33.63729105331004 }, "regressions": 0, "confirmed_within_100ms": 0.98, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 } }, "total_operations": 300, "statuses": { "success": 300, "timeout": 0, "error": 0, "invalid_precondition": 0, "client_overload": 0 }, "query": { "count": 300, "success": 300, "ann_recall_at_10": 0.994, "ndcg_at_10": 0.7931268675492565, "latency_ms": { "min": 2.8424999909475446, "p50": 10.802228993270546, "p95": 15.70037849014625, "p99": 18.599904199363664, "max": 21.083833067677915 }, "dispatch_delay_ms": { "min": 0.11499994434416294, "p50": 0.9972504340112209, "p95": 1.792112522525713, "p99": 2.147816633805632, "max": 2.415291965007782 } }, "diagnostics": { "schedule_delay_ms": { "min": 0.005709007382392883, "p50": 0.3763544606041336, "p95": 0.7450981764116071, "p99": 0.7652516411613952, "max": 0.7731669582426992 }, "probe_dispatch_ms": { "min": 0.018582912161946297, "p50": 0.5085835000500083, "p95": 1.323214132571592, "p99": 1.6386267193593083, "max": 5.59916696511209 }, "probe_service_ms": { "min": 0.29562506824731827, "p50": 1.937312539666891, "p95": 12.462289229733864, "p99": 16.413377205608402, "max": 21.32604189682752 }, "write_service_ms": { "min": 1.0730830254033208, "p50": 14.693812525365502, "p95": 21.64398340391927, "p99": 24.970728757325553, "max": 26.850666967220604 } }, "actual_submission_rate": 9.99996976060032, "metadata": { "backend": "pgvector", "collection": "fb2_429e9a_023", "dataset": { "dataset": "nq", "source": "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/nq.zip", "source_sha256": "2553bf7bfbab47b1436ca00a34bce57320e18e611fd00999a2a3a1b4714be752", "source_md5": "d4d3d2e48787a744b6f6e691ff534307", "records": 124000, "queries": 100, "relevant_documents_retained": 120, "seed": 20261028, "license": "CC-BY-SA-3.0", "license_source": "https://raw.githubusercontent.com/google-research-datasets/natural-questions/master/nq_open/README.md", "attribution": "BEIR and original dataset authors; Wikipedia contributors for NQ, SciFact authors and cited article authors for SciFact.", "transformation": "200-word title+text prefix; seeded reservoir of documents with relevant documents for selected test queries retained at front. Simulated mutations; no historical replay.", "model": "sentence-transformers/all-MiniLM-L6-v2", "model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "dimensions": 384, "normalized": true, "max_tokens": 256, "device": "mps", "files": { "corpus.jsonl": "9dc487d785e6ee493eccaf764daa3bf8720a8f4fc8c406246065e1ad33a91f5d", "queries.json": "5603f52876ada5d36118b65dea490f1d43900bc159e1804e637796eb9115e999", "vectors.npy": "cb7421eb6ca5964231403f0a922843088f6fd0e01cbf67f2a1076cc7101005db", "query_vectors.npy": "5e666f8d966d8db6b0fd93d4963234d2dffa0ee3910b8fad069a8ee201e8b5d4", "UPSTREAM-LICENSE.md": "8b675521da3fa142ac6bda38c9bcfcc8cff0e2d6128be53a062ed55fb53f7b87" } }, "base_count": 100000, "profile": { "dataset": "nq", "base_count": 100000, "rate": 10, "query_rate": 10, "arrival": "steady" }, "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "preloaded_mutable": 200, "started_utc": "2026-10-29T03:07:19Z", "setup_s": 58.333049291046336, "backend_before": { "version": "0.8.7", "postgres": "PostgreSQL 16.15 (Debian 16.15-1.pgdg12+2) on aarch64-unknown-linux-gnu, compiled by gcc (Debian 12.2.0-14+deb12u1) 12.2.0, 64-bit", "ef_search": 128, "ack": "autocommit, READ COMMITTED", "transport": "psycopg, connection per worker", "query_plan": [ { "Plan": { "Node Type": "Limit", "Parallel Aware": false, "Async Capable": false, "Startup Cost": 1674.2, "Total Cost": 1691.32, "Plan Rows": 10, "Plan Width": 18, "Plans": [ { "Node Type": "Index Scan", "Parent Relationship": "Outer", "Parallel Aware": false, "Async Capable": false, "Scan Direction": "Forward", "Index Name": "fb2_429e9a_023_hnsw", "Relation Name": "fb2_429e9a_023", "Alias": "fb2_429e9a_023", "Startup Cost": 1674.2, "Total Cost": 172905.99, "Plan Rows": 99996, "Plan Width": 18, "Order By": "(embedding <=> '[-0.08946895,-0.094362676,0.092023626,-0.0040844884,0.043233823,0.00062519487,0.1075197,-0.035732996,0.013746996,-0.024103122,0.0021706754,-0.04720602,0.04787615,-0.02043222,0.058315646,0.081252135,-0.05375015,-0.030799072,-0.0077205584,0.035735283,-0.04474784,0.01614922,-0.06905586,0.04194401,-0.08832908,0.007873556,0.0044992934,0.021968655,0.010953364,0.07212316,0.023185596,0.001176476,0.04725166,-0.03580423,-0.032344446,0.004752835,-0.056179125,-0.0353235,-0.037251893,-0.005143068,-0.027599342,0.024403147,0.031338032,0.0056410627,0.008425455,-0.0087741045,-0.039870497,-0.011831836,0.042621233,0.048142947,0.01603842,0.06745542,-0.038018145,0.064212486,-0.009636566,0.01607475,-0.0027868534,-0.03761097,-0.011889431,-0.01084577,0.083403684,0.028946918,0.061870236,-0.045382615,0.03812586,0.060426526,-0.017003024,0.02328925,-0.093020774,-0.024072766,0.042720392,-0.022444837,-0.011597358,0.01209231,-0.005953629,0.029691145,-0.011711495,-0.053345133,-0.081757374,-0.049303155,0.032193717,-0.038507868,-0.0060101775,-0.031470332,-0.07362443,0.03575578,0.0245484,0.01985519,-0.15116754,0.050386053,-0.0389634,0.07548389,-0.09264168,-0.012602816,-0.0035539444,-0.046416666,0.015100493,0.100691915,-0.037529,0.04600518,-0.021859422,0.014675989,-0.097722545,-0.053200893,-0.017040461,-0.01629616,-0.012339222,0.06543302,0.07252074,-0.044301156,0.03054835,-0.038570553,0.019310065,-0.005527747,-0.029386865,-0.058248505,0.065443166,0.028506057,0.0878254,0.058387812,0.081721246,-0.0005962058,0.0569388,0.00960514,0.011667701,0.014456492,-0.07218306,-3.0444767e-33,0.06687367,-0.019213954,0.058202963,0.025171444,-0.028259544,-0.04171671,0.03127722,0.04865431,-0.0721459,-0.050143186,-0.029907746,0.06248994,-0.05096076,-0.065192886,0.010210447,-0.077256374,-0.043370593,-0.0901922,-0.066522665,0.026072454,-0.022315364,0.075375885,-0.029968103,-0.0050771856,-0.0046955296,0.0065650316,-0.06791897,0.07368851,0.056069646,-0.021026025,-0.011352883,0.038811427,-0.090847075,-0.056490287,-0.039881036,0.032885592,-0.029586185,0.04753608,0.006341778,-0.0034275919,0.03681623,0.09326265,-0.0532089,0.032396164,0.012605483,0.045778517,-0.0006472902,0.029721813,0.0734947,-0.037117425,-0.09356724,-0.0071634455,-0.032400057,0.010453378,0.06286513,0.006053158,0.060524322,0.091866635,0.063640274,0.055919155,0.053222507,-0.019174587,0.089296214,-0.058922265,0.013745364,-0.022907477,0.028347451,-0.040759467,-0.06820364,-0.061398294,-0.046857864,0.0060627847,0.010442015,-0.07662317,0.031950947,-0.107813306,-0.0016125626,-0.033413693,-0.09989722,-0.03580293,-0.036118146,-0.08276185,0.1280713,0.07596603,0.049225595,-0.026312606,0.0031930343,-0.0012469466,0.000872793,-0.040353082,-0.034516506,-0.036047976,0.004999689,-0.01570079,-0.09783203,1.4186912e-33,-0.0130903,0.04742377,0.02889633,0.09809088,0.043639127,-0.0747531,0.03402747,0.029147558,0.019848268,0.111865185,0.04845458,-0.016222177,0.025501896,0.09813583,-0.002833195,0.0038929228,0.048537854,0.014111116,0.036570966,-0.017512968,-0.079755045,0.043452248,0.10843892,0.061968718,-0.015678233,0.0060376576,0.010718569,-0.03907552,0.040307663,0.04553227,0.041731495,0.083179526,-0.0844024,-0.023447752,0.014485363,-0.00788756,-0.11410207,0.039309476,0.011220049,0.12889338,-0.07970701,0.010658111,-0.06657531,-0.022037188,-0.043459363,-0.030291978,0.05803613,0.06109021,-0.09415837,0.030727232,0.011858702,0.013526454,-0.049580045,-0.052486703,0.013218615,0.0050231596,0.07660412,-0.0046343356,-0.0692584,-0.03353194,0.022288544,0.022836983,0.01361394,-0.036331616,0.008194689,0.09226991,0.08229875,-0.047639843,-0.005033471,0.0066081914,-0.106666416,-0.06385503,0.006385333,0.06318358,0.03146985,-0.060764216,-0.043287244,-0.042385958,-0.07963737,-0.043478705,-0.031840656,0.060027804,0.027829666,0.03728562,-0.07014622,0.0649568,-0.052579004,-0.0034938955,-0.15390792,-0.04426162,0.028217599,0.05593145,0.023251446,0.029115746,-0.040321343,-1.8832651e-08,-0.037536032,0.0005056563,0.013919978,-0.038129315,0.061486673,0.005534046,-0.010768211,-0.16527015,0.061414365,0.019963786,-0.019666849,-0.059065994,0.00083181454,-0.016695915,0.01090875,0.02685501,-0.0008743374,-0.060967337,-0.049258173,0.012731965,-0.03973743,-0.0018210232,-0.03797001,0.00035104176,0.04477184,0.04185603,-0.03215796,-0.09103283,0.0467481,0.017810425,0.037437033,0.054782324,-0.0059442874,0.022557084,0.053393807,-0.03746794,-0.07080435,0.10548834,-0.04215435,0.042547524,0.05064807,0.07560324,0.04072184,-0.0011278372,0.00702018,-0.070807114,0.040035766,0.051417697,0.011316928,-0.028955372,0.04197099,-0.0026867494,-0.056857754,0.0761941,0.030621601,-0.052413527,-0.021492323,0.0957355,0.1013111,-0.082977295,-0.036080915,-0.03327806,0.0029404552,-0.03586191]'::vector)", "Filter": "(cohort = 'static'::text)" } ] } } ] }, "count_before": 100200, "idle_duration_s": 10.0, "idle_queries": { "count": 100, "ann_recall": 0.9940000000000001, "ndcg": 0.7931268675492565, "latency_ms": { "min": 4.297167062759399, "p50": 17.850000003818423, "p95": 20.900401787366718, "p99": 23.15179421333597, "max": 24.950542021542788 } }, "exact_ndcg": 0.7931268675492565, "resources_before": { "usage_usec": 427262090, "user_usec": 401481197, "system_usec": 25780893, "nice_usec": 0, "nr_periods": 5413, "nr_throttled": 1697, "throttled_usec": 178142032, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 1582358528, "pids": 23 }, "resources_after": { "usage_usec": 438325929, "user_usec": 408993305, "system_usec": 29332624, "nice_usec": 0, "nr_periods": 5714, "nr_throttled": 1697, "throttled_usec": 178142032, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 1602420736, "pids": 31 }, "count_after": 100200, "backend_after": { "version": "0.8.7", "postgres": "PostgreSQL 16.15 (Debian 16.15-1.pgdg12+2) on aarch64-unknown-linux-gnu, compiled by gcc (Debian 12.2.0-14+deb12u1) 12.2.0, 64-bit", "ef_search": 128, "ack": "autocommit, READ COMMITTED", "transport": "psycopg, connection per worker", "query_plan": [ { "Plan": { "Node Type": "Limit", "Parallel Aware": false, "Async Capable": false, "Startup Cost": 1675.71, "Total Cost": 1692.84, "Plan Rows": 10, "Plan Width": 18, "Plans": [ { "Node Type": "Index Scan", "Parent Relationship": "Outer", "Parallel Aware": false, "Async Capable": false, "Scan Direction": "Forward", "Index Name": "fb2_429e9a_023_hnsw", "Relation Name": "fb2_429e9a_023", "Alias": "fb2_429e9a_023", "Startup Cost": 1675.71, "Total Cost": 173192.67, "Plan Rows": 100115, "Plan Width": 18, "Order By": "(embedding <=> '[-0.08946895,-0.094362676,0.092023626,-0.0040844884,0.043233823,0.00062519487,0.1075197,-0.035732996,0.013746996,-0.024103122,0.0021706754,-0.04720602,0.04787615,-0.02043222,0.058315646,0.081252135,-0.05375015,-0.030799072,-0.0077205584,0.035735283,-0.04474784,0.01614922,-0.06905586,0.04194401,-0.08832908,0.007873556,0.0044992934,0.021968655,0.010953364,0.07212316,0.023185596,0.001176476,0.04725166,-0.03580423,-0.032344446,0.004752835,-0.056179125,-0.0353235,-0.037251893,-0.005143068,-0.027599342,0.024403147,0.031338032,0.0056410627,0.008425455,-0.0087741045,-0.039870497,-0.011831836,0.042621233,0.048142947,0.01603842,0.06745542,-0.038018145,0.064212486,-0.009636566,0.01607475,-0.0027868534,-0.03761097,-0.011889431,-0.01084577,0.083403684,0.028946918,0.061870236,-0.045382615,0.03812586,0.060426526,-0.017003024,0.02328925,-0.093020774,-0.024072766,0.042720392,-0.022444837,-0.011597358,0.01209231,-0.005953629,0.029691145,-0.011711495,-0.053345133,-0.081757374,-0.049303155,0.032193717,-0.038507868,-0.0060101775,-0.031470332,-0.07362443,0.03575578,0.0245484,0.01985519,-0.15116754,0.050386053,-0.0389634,0.07548389,-0.09264168,-0.012602816,-0.0035539444,-0.046416666,0.015100493,0.100691915,-0.037529,0.04600518,-0.021859422,0.014675989,-0.097722545,-0.053200893,-0.017040461,-0.01629616,-0.012339222,0.06543302,0.07252074,-0.044301156,0.03054835,-0.038570553,0.019310065,-0.005527747,-0.029386865,-0.058248505,0.065443166,0.028506057,0.0878254,0.058387812,0.081721246,-0.0005962058,0.0569388,0.00960514,0.011667701,0.014456492,-0.07218306,-3.0444767e-33,0.06687367,-0.019213954,0.058202963,0.025171444,-0.028259544,-0.04171671,0.03127722,0.04865431,-0.0721459,-0.050143186,-0.029907746,0.06248994,-0.05096076,-0.065192886,0.010210447,-0.077256374,-0.043370593,-0.0901922,-0.066522665,0.026072454,-0.022315364,0.075375885,-0.029968103,-0.0050771856,-0.0046955296,0.0065650316,-0.06791897,0.07368851,0.056069646,-0.021026025,-0.011352883,0.038811427,-0.090847075,-0.056490287,-0.039881036,0.032885592,-0.029586185,0.04753608,0.006341778,-0.0034275919,0.03681623,0.09326265,-0.0532089,0.032396164,0.012605483,0.045778517,-0.0006472902,0.029721813,0.0734947,-0.037117425,-0.09356724,-0.0071634455,-0.032400057,0.010453378,0.06286513,0.006053158,0.060524322,0.091866635,0.063640274,0.055919155,0.053222507,-0.019174587,0.089296214,-0.058922265,0.013745364,-0.022907477,0.028347451,-0.040759467,-0.06820364,-0.061398294,-0.046857864,0.0060627847,0.010442015,-0.07662317,0.031950947,-0.107813306,-0.0016125626,-0.033413693,-0.09989722,-0.03580293,-0.036118146,-0.08276185,0.1280713,0.07596603,0.049225595,-0.026312606,0.0031930343,-0.0012469466,0.000872793,-0.040353082,-0.034516506,-0.036047976,0.004999689,-0.01570079,-0.09783203,1.4186912e-33,-0.0130903,0.04742377,0.02889633,0.09809088,0.043639127,-0.0747531,0.03402747,0.029147558,0.019848268,0.111865185,0.04845458,-0.016222177,0.025501896,0.09813583,-0.002833195,0.0038929228,0.048537854,0.014111116,0.036570966,-0.017512968,-0.079755045,0.043452248,0.10843892,0.061968718,-0.015678233,0.0060376576,0.010718569,-0.03907552,0.040307663,0.04553227,0.041731495,0.083179526,-0.0844024,-0.023447752,0.014485363,-0.00788756,-0.11410207,0.039309476,0.011220049,0.12889338,-0.07970701,0.010658111,-0.06657531,-0.022037188,-0.043459363,-0.030291978,0.05803613,0.06109021,-0.09415837,0.030727232,0.011858702,0.013526454,-0.049580045,-0.052486703,0.013218615,0.0050231596,0.07660412,-0.0046343356,-0.0692584,-0.03353194,0.022288544,0.022836983,0.01361394,-0.036331616,0.008194689,0.09226991,0.08229875,-0.047639843,-0.005033471,0.0066081914,-0.106666416,-0.06385503,0.006385333,0.06318358,0.03146985,-0.060764216,-0.043287244,-0.042385958,-0.07963737,-0.043478705,-0.031840656,0.060027804,0.027829666,0.03728562,-0.07014622,0.0649568,-0.052579004,-0.0034938955,-0.15390792,-0.04426162,0.028217599,0.05593145,0.023251446,0.029115746,-0.040321343,-1.8832651e-08,-0.037536032,0.0005056563,0.013919978,-0.038129315,0.061486673,0.005534046,-0.010768211,-0.16527015,0.061414365,0.019963786,-0.019666849,-0.059065994,0.00083181454,-0.016695915,0.01090875,0.02685501,-0.0008743374,-0.060967337,-0.049258173,0.012731965,-0.03973743,-0.0018210232,-0.03797001,0.00035104176,0.04477184,0.04185603,-0.03215796,-0.09103283,0.0467481,0.017810425,0.037437033,0.054782324,-0.0059442874,0.022557084,0.053393807,-0.03746794,-0.07080435,0.10548834,-0.04215435,0.042547524,0.05064807,0.07560324,0.04072184,-0.0011278372,0.00702018,-0.070807114,0.040035766,0.051417697,0.011316928,-0.028955372,0.04197099,-0.0026867494,-0.056857754,0.0761941,0.030621601,-0.052413527,-0.021492323,0.0957355,0.1013111,-0.082977295,-0.036080915,-0.03327806,0.0029404552,-0.03586191]'::vector)", "Filter": "(cohort = 'static'::text)" } ] } } ] } }, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "eligible": true, "protocol": "mixed-mutations-v2" }, "examples": [ { "event": 0, "operation": "insert", "doc_id": "mutation-000000", "scheduled_s": 0.0, "submitted_s": 0.00023145892191678286, "schedule_delay_ms": 0.23145892191678286, "precondition": true, "status": "success", "ack_ms": 23.75987498089671, "write_started_ms": 1.0239580878987908, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 27.108874986879528, "stable_since_ms": 27.108874986879528, "confirmed_ms": 79.49733303394169, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 34.26550002768636, "stable_since_ms": 34.26550002768636, "confirmed_ms": 90.87700000964105, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 34.26550002768636, "confirmed_ms": 90.87700000964105, "search_minus_ack_ms": 10.505625046789646 }, { "event": 1, "operation": "update", "doc_id": "mutation-000001", "scheduled_s": 0.1, "submitted_s": 0.10065624990966171, "schedule_delay_ms": 0.6562499096617047, "precondition": true, "status": "success", "ack_ms": 9.23591700848192, "write_started_ms": 0.13995904009789228, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 13.399292016401887, "stable_since_ms": 13.399292016401887, "confirmed_ms": 66.86870905105025, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 21.872750017791986, "stable_since_ms": 21.872750017791986, "confirmed_ms": 83.37433403357863, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 21.872750017791986, "confirmed_ms": 83.37433403357863, "search_minus_ack_ms": 12.636833009310067 }, { "event": 2, "operation": "delete", "doc_id": "mutation-000002", "scheduled_s": 0.2, "submitted_s": 0.20059504196979105, "schedule_delay_ms": 0.5950419697910436, "precondition": true, "status": "success", "ack_ms": 3.1297500245273113, "write_started_ms": 0.22491696290671825, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 13.229000032879412, "stable_since_ms": 13.229000032879412, "confirmed_ms": 65.71995792910457, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 27.564582997001708, "stable_since_ms": 27.564582997001708, "confirmed_ms": 92.34220802318305, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 27.564582997001708, "confirmed_ms": 92.34220802318305, "search_minus_ack_ms": 24.434832972474396 } ] }, { "run": "fb2_429e9a_024", "trial": 2, "backend": "pgvector", "scenario": "nq-10k", "status": "complete", "eligible": true, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "summary": { "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "wall_s": 30.0010595000349, "operations": { "insert": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 15.692042070440948, "p50": 22.57885452127084, "p95": 25.324377499055117, "p99": 27.196365731069836, "max": 30.82954091951251 }, "confirmed_ms": { "min": 71.56745798420161, "p50": 86.97064546868205, "p95": 91.33284733397886, "p99": 92.73812915431336, "max": 93.03554194048047 }, "ack_ms": { "min": 4.661708022467792, "p50": 10.22672897670418, "p95": 12.353110115509478, "p99": 20.78053735080175, "max": 20.986290997825563 }, "search_minus_ack_ms": { "min": 6.381207960657775, "p50": 12.411229021381587, "p95": 14.318540290696546, "p99": 14.496769574470823, "max": 15.637208009138703 }, "regressions": 0, "confirmed_within_100ms": 1.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "update": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 15.797084080986679, "p50": 22.806854511145502, "p95": 25.879771169275045, "p99": 29.941756976768392, "max": 33.5022920044139 }, "confirmed_ms": { "min": 70.09920896962285, "p50": 87.36616699025035, "p95": 91.28019412746653, "p99": 92.42501640343107, "max": 92.48445800039917 }, "ack_ms": { "min": 4.297542036511004, "p50": 10.314625513274223, "p95": 12.816922995261846, "p99": 19.51307534123772, "max": 21.31075004581362 }, "search_minus_ack_ms": { "min": 9.465832961723208, "p50": 12.29837496066466, "p95": 14.780187554424629, "p99": 15.417821672745053, "max": 16.023083007894456 }, "regressions": 0, "confirmed_within_100ms": 1.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "delete": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 12.18062499538064, "p50": 14.170312497299165, "p95": 25.059871160192415, "p99": 26.825963628944017, "max": 28.18766701966524 }, "confirmed_ms": { "min": 63.55616601649672, "p50": 74.36287502059713, "p95": 89.99972859746777, "p99": 92.78905918006785, "max": 93.3559590484947 }, "ack_ms": { "min": 1.3221659464761615, "p50": 2.5760005228221416, "p95": 4.242569010239094, "p99": 4.866735436953623, "max": 8.540667011402547 }, "search_minus_ack_ms": { "min": 1.402208930812776, "p50": 5.280832992866635, "p95": 20.840941945789382, "p99": 22.958719028392814, "max": 24.160083034075797 }, "regressions": 0, "confirmed_within_100ms": 1.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 } }, "total_operations": 300, "statuses": { "success": 300, "timeout": 0, "error": 0, "invalid_precondition": 0, "client_overload": 0 }, "query": { "count": 300, "success": 300, "ann_recall_at_10": 0.999, "ndcg_at_10": 0.9481505328688746, "latency_ms": { "min": 2.0902500255033374, "p50": 5.8650210266932845, "p95": 7.885435351636262, "p99": 10.503048017853859, "max": 16.662917099893093 }, "dispatch_delay_ms": { "min": 0.09233388118445873, "p50": 0.9338544332422316, "p95": 1.9409354659728708, "p99": 2.1081840060651302, "max": 3.720874898135662 } }, "diagnostics": { "schedule_delay_ms": { "min": 0.006167008542945496, "p50": 0.31587547855416176, "p95": 0.6097646208932161, "p99": 0.632330713094964, "max": 0.6380839506163749 }, "probe_dispatch_ms": { "min": 0.015374971553683281, "p50": 0.412146036978811, "p95": 1.5825477545149613, "p99": 2.3483045375905927, "max": 3.9275409653782845 }, "probe_service_ms": { "min": 0.34545804373919964, "p50": 2.845895476639271, "p95": 7.629527244716882, "p99": 8.529336673673237, "max": 17.2793340170756 }, "write_service_ms": { "min": 1.0321249719709158, "p50": 8.948062546551228, "p95": 11.713531287387013, "p99": 19.215178412850943, "max": 20.403958042152226 } }, "actual_submission_rate": 9.999976142889807, "metadata": { "backend": "pgvector", "collection": "fb2_429e9a_024", "dataset": { "dataset": "nq", "source": "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/nq.zip", "source_sha256": "2553bf7bfbab47b1436ca00a34bce57320e18e611fd00999a2a3a1b4714be752", "source_md5": "d4d3d2e48787a744b6f6e691ff534307", "records": 124000, "queries": 100, "relevant_documents_retained": 120, "seed": 20261028, "license": "CC-BY-SA-3.0", "license_source": "https://raw.githubusercontent.com/google-research-datasets/natural-questions/master/nq_open/README.md", "attribution": "BEIR and original dataset authors; Wikipedia contributors for NQ, SciFact authors and cited article authors for SciFact.", "transformation": "200-word title+text prefix; seeded reservoir of documents with relevant documents for selected test queries retained at front. Simulated mutations; no historical replay.", "model": "sentence-transformers/all-MiniLM-L6-v2", "model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "dimensions": 384, "normalized": true, "max_tokens": 256, "device": "mps", "files": { "corpus.jsonl": "9dc487d785e6ee493eccaf764daa3bf8720a8f4fc8c406246065e1ad33a91f5d", "queries.json": "5603f52876ada5d36118b65dea490f1d43900bc159e1804e637796eb9115e999", "vectors.npy": "cb7421eb6ca5964231403f0a922843088f6fd0e01cbf67f2a1076cc7101005db", "query_vectors.npy": "5e666f8d966d8db6b0fd93d4963234d2dffa0ee3910b8fad069a8ee201e8b5d4", "UPSTREAM-LICENSE.md": "8b675521da3fa142ac6bda38c9bcfcc8cff0e2d6128be53a062ed55fb53f7b87" } }, "base_count": 10000, "profile": { "dataset": "nq", "base_count": 10000, "rate": 10, "query_rate": 10, "arrival": "steady" }, "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "preloaded_mutable": 200, "started_utc": "2026-10-29T03:09:02Z", "setup_s": 3.6053121249424294, "backend_before": { "version": "0.8.7", "postgres": "PostgreSQL 16.15 (Debian 16.15-1.pgdg12+2) on aarch64-unknown-linux-gnu, compiled by gcc (Debian 12.2.0-14+deb12u1) 12.2.0, 64-bit", "ef_search": 128, "ack": "autocommit, READ COMMITTED", "transport": "psycopg, connection per worker", "query_plan": [ { "Plan": { "Node Type": "Limit", "Parallel Aware": false, "Async Capable": false, "Startup Cost": 1340.05, "Total Cost": 1356.28, "Plan Rows": 10, "Plan Width": 18, "Plans": [ { "Node Type": "Index Scan", "Parent Relationship": "Outer", "Parallel Aware": false, "Async Capable": false, "Scan Direction": "Forward", "Index Name": "fb2_429e9a_024_hnsw", "Relation Name": "fb2_429e9a_024", "Alias": "fb2_429e9a_024", "Startup Cost": 1340.05, "Total Cost": 17569.0, "Plan Rows": 10000, "Plan Width": 18, "Order By": "(embedding <=> '[-0.08946895,-0.094362676,0.092023626,-0.0040844884,0.043233823,0.00062519487,0.1075197,-0.035732996,0.013746996,-0.024103122,0.0021706754,-0.04720602,0.04787615,-0.02043222,0.058315646,0.081252135,-0.05375015,-0.030799072,-0.0077205584,0.035735283,-0.04474784,0.01614922,-0.06905586,0.04194401,-0.08832908,0.007873556,0.0044992934,0.021968655,0.010953364,0.07212316,0.023185596,0.001176476,0.04725166,-0.03580423,-0.032344446,0.004752835,-0.056179125,-0.0353235,-0.037251893,-0.005143068,-0.027599342,0.024403147,0.031338032,0.0056410627,0.008425455,-0.0087741045,-0.039870497,-0.011831836,0.042621233,0.048142947,0.01603842,0.06745542,-0.038018145,0.064212486,-0.009636566,0.01607475,-0.0027868534,-0.03761097,-0.011889431,-0.01084577,0.083403684,0.028946918,0.061870236,-0.045382615,0.03812586,0.060426526,-0.017003024,0.02328925,-0.093020774,-0.024072766,0.042720392,-0.022444837,-0.011597358,0.01209231,-0.005953629,0.029691145,-0.011711495,-0.053345133,-0.081757374,-0.049303155,0.032193717,-0.038507868,-0.0060101775,-0.031470332,-0.07362443,0.03575578,0.0245484,0.01985519,-0.15116754,0.050386053,-0.0389634,0.07548389,-0.09264168,-0.012602816,-0.0035539444,-0.046416666,0.015100493,0.100691915,-0.037529,0.04600518,-0.021859422,0.014675989,-0.097722545,-0.053200893,-0.017040461,-0.01629616,-0.012339222,0.06543302,0.07252074,-0.044301156,0.03054835,-0.038570553,0.019310065,-0.005527747,-0.029386865,-0.058248505,0.065443166,0.028506057,0.0878254,0.058387812,0.081721246,-0.0005962058,0.0569388,0.00960514,0.011667701,0.014456492,-0.07218306,-3.0444767e-33,0.06687367,-0.019213954,0.058202963,0.025171444,-0.028259544,-0.04171671,0.03127722,0.04865431,-0.0721459,-0.050143186,-0.029907746,0.06248994,-0.05096076,-0.065192886,0.010210447,-0.077256374,-0.043370593,-0.0901922,-0.066522665,0.026072454,-0.022315364,0.075375885,-0.029968103,-0.0050771856,-0.0046955296,0.0065650316,-0.06791897,0.07368851,0.056069646,-0.021026025,-0.011352883,0.038811427,-0.090847075,-0.056490287,-0.039881036,0.032885592,-0.029586185,0.04753608,0.006341778,-0.0034275919,0.03681623,0.09326265,-0.0532089,0.032396164,0.012605483,0.045778517,-0.0006472902,0.029721813,0.0734947,-0.037117425,-0.09356724,-0.0071634455,-0.032400057,0.010453378,0.06286513,0.006053158,0.060524322,0.091866635,0.063640274,0.055919155,0.053222507,-0.019174587,0.089296214,-0.058922265,0.013745364,-0.022907477,0.028347451,-0.040759467,-0.06820364,-0.061398294,-0.046857864,0.0060627847,0.010442015,-0.07662317,0.031950947,-0.107813306,-0.0016125626,-0.033413693,-0.09989722,-0.03580293,-0.036118146,-0.08276185,0.1280713,0.07596603,0.049225595,-0.026312606,0.0031930343,-0.0012469466,0.000872793,-0.040353082,-0.034516506,-0.036047976,0.004999689,-0.01570079,-0.09783203,1.4186912e-33,-0.0130903,0.04742377,0.02889633,0.09809088,0.043639127,-0.0747531,0.03402747,0.029147558,0.019848268,0.111865185,0.04845458,-0.016222177,0.025501896,0.09813583,-0.002833195,0.0038929228,0.048537854,0.014111116,0.036570966,-0.017512968,-0.079755045,0.043452248,0.10843892,0.061968718,-0.015678233,0.0060376576,0.010718569,-0.03907552,0.040307663,0.04553227,0.041731495,0.083179526,-0.0844024,-0.023447752,0.014485363,-0.00788756,-0.11410207,0.039309476,0.011220049,0.12889338,-0.07970701,0.010658111,-0.06657531,-0.022037188,-0.043459363,-0.030291978,0.05803613,0.06109021,-0.09415837,0.030727232,0.011858702,0.013526454,-0.049580045,-0.052486703,0.013218615,0.0050231596,0.07660412,-0.0046343356,-0.0692584,-0.03353194,0.022288544,0.022836983,0.01361394,-0.036331616,0.008194689,0.09226991,0.08229875,-0.047639843,-0.005033471,0.0066081914,-0.106666416,-0.06385503,0.006385333,0.06318358,0.03146985,-0.060764216,-0.043287244,-0.042385958,-0.07963737,-0.043478705,-0.031840656,0.060027804,0.027829666,0.03728562,-0.07014622,0.0649568,-0.052579004,-0.0034938955,-0.15390792,-0.04426162,0.028217599,0.05593145,0.023251446,0.029115746,-0.040321343,-1.8832651e-08,-0.037536032,0.0005056563,0.013919978,-0.038129315,0.061486673,0.005534046,-0.010768211,-0.16527015,0.061414365,0.019963786,-0.019666849,-0.059065994,0.00083181454,-0.016695915,0.01090875,0.02685501,-0.0008743374,-0.060967337,-0.049258173,0.012731965,-0.03973743,-0.0018210232,-0.03797001,0.00035104176,0.04477184,0.04185603,-0.03215796,-0.09103283,0.0467481,0.017810425,0.037437033,0.054782324,-0.0059442874,0.022557084,0.053393807,-0.03746794,-0.07080435,0.10548834,-0.04215435,0.042547524,0.05064807,0.07560324,0.04072184,-0.0011278372,0.00702018,-0.070807114,0.040035766,0.051417697,0.011316928,-0.028955372,0.04197099,-0.0026867494,-0.056857754,0.0761941,0.030621601,-0.052413527,-0.021492323,0.0957355,0.1013111,-0.082977295,-0.036080915,-0.03327806,0.0029404552,-0.03586191]'::vector)", "Filter": "(cohort = 'static'::text)" } ] } } ] }, "count_before": 10200, "idle_duration_s": 10.0, "idle_queries": { "count": 100, "ann_recall": 0.9990000000000001, "ndcg": 0.9481505328688745, "latency_ms": { "min": 2.4842499988153577, "p50": 7.324645994231105, "p95": 8.941639313707128, "p99": 9.629571203840966, "max": 22.877875017002225 } }, "exact_ndcg": 0.9481505328688745, "resources_before": { "usage_usec": 443636838, "user_usec": 413959993, "system_usec": 29676845, "nice_usec": 0, "nr_periods": 5874, "nr_throttled": 1716, "throttled_usec": 180001427, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 1168064512, "pids": 23 }, "resources_after": { "usage_usec": 451364572, "user_usec": 421242233, "system_usec": 30122339, "nice_usec": 0, "nr_periods": 6175, "nr_throttled": 1716, "throttled_usec": 180001427, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 1178894336, "pids": 27 }, "count_after": 10200, "backend_after": { "version": "0.8.7", "postgres": "PostgreSQL 16.15 (Debian 16.15-1.pgdg12+2) on aarch64-unknown-linux-gnu, compiled by gcc (Debian 12.2.0-14+deb12u1) 12.2.0, 64-bit", "ef_search": 128, "ack": "autocommit, READ COMMITTED", "transport": "psycopg, connection per worker", "query_plan": [ { "Plan": { "Node Type": "Limit", "Parallel Aware": false, "Async Capable": false, "Startup Cost": 1351.31, "Total Cost": 1367.61, "Plan Rows": 10, "Plan Width": 18, "Plans": [ { "Node Type": "Index Scan", "Parent Relationship": "Outer", "Parallel Aware": false, "Async Capable": false, "Scan Direction": "Forward", "Index Name": "fb2_429e9a_024_hnsw", "Relation Name": "fb2_429e9a_024", "Alias": "fb2_429e9a_024", "Startup Cost": 1351.31, "Total Cost": 17859.83, "Plan Rows": 10124, "Plan Width": 18, "Order By": "(embedding <=> '[-0.08946895,-0.094362676,0.092023626,-0.0040844884,0.043233823,0.00062519487,0.1075197,-0.035732996,0.013746996,-0.024103122,0.0021706754,-0.04720602,0.04787615,-0.02043222,0.058315646,0.081252135,-0.05375015,-0.030799072,-0.0077205584,0.035735283,-0.04474784,0.01614922,-0.06905586,0.04194401,-0.08832908,0.007873556,0.0044992934,0.021968655,0.010953364,0.07212316,0.023185596,0.001176476,0.04725166,-0.03580423,-0.032344446,0.004752835,-0.056179125,-0.0353235,-0.037251893,-0.005143068,-0.027599342,0.024403147,0.031338032,0.0056410627,0.008425455,-0.0087741045,-0.039870497,-0.011831836,0.042621233,0.048142947,0.01603842,0.06745542,-0.038018145,0.064212486,-0.009636566,0.01607475,-0.0027868534,-0.03761097,-0.011889431,-0.01084577,0.083403684,0.028946918,0.061870236,-0.045382615,0.03812586,0.060426526,-0.017003024,0.02328925,-0.093020774,-0.024072766,0.042720392,-0.022444837,-0.011597358,0.01209231,-0.005953629,0.029691145,-0.011711495,-0.053345133,-0.081757374,-0.049303155,0.032193717,-0.038507868,-0.0060101775,-0.031470332,-0.07362443,0.03575578,0.0245484,0.01985519,-0.15116754,0.050386053,-0.0389634,0.07548389,-0.09264168,-0.012602816,-0.0035539444,-0.046416666,0.015100493,0.100691915,-0.037529,0.04600518,-0.021859422,0.014675989,-0.097722545,-0.053200893,-0.017040461,-0.01629616,-0.012339222,0.06543302,0.07252074,-0.044301156,0.03054835,-0.038570553,0.019310065,-0.005527747,-0.029386865,-0.058248505,0.065443166,0.028506057,0.0878254,0.058387812,0.081721246,-0.0005962058,0.0569388,0.00960514,0.011667701,0.014456492,-0.07218306,-3.0444767e-33,0.06687367,-0.019213954,0.058202963,0.025171444,-0.028259544,-0.04171671,0.03127722,0.04865431,-0.0721459,-0.050143186,-0.029907746,0.06248994,-0.05096076,-0.065192886,0.010210447,-0.077256374,-0.043370593,-0.0901922,-0.066522665,0.026072454,-0.022315364,0.075375885,-0.029968103,-0.0050771856,-0.0046955296,0.0065650316,-0.06791897,0.07368851,0.056069646,-0.021026025,-0.011352883,0.038811427,-0.090847075,-0.056490287,-0.039881036,0.032885592,-0.029586185,0.04753608,0.006341778,-0.0034275919,0.03681623,0.09326265,-0.0532089,0.032396164,0.012605483,0.045778517,-0.0006472902,0.029721813,0.0734947,-0.037117425,-0.09356724,-0.0071634455,-0.032400057,0.010453378,0.06286513,0.006053158,0.060524322,0.091866635,0.063640274,0.055919155,0.053222507,-0.019174587,0.089296214,-0.058922265,0.013745364,-0.022907477,0.028347451,-0.040759467,-0.06820364,-0.061398294,-0.046857864,0.0060627847,0.010442015,-0.07662317,0.031950947,-0.107813306,-0.0016125626,-0.033413693,-0.09989722,-0.03580293,-0.036118146,-0.08276185,0.1280713,0.07596603,0.049225595,-0.026312606,0.0031930343,-0.0012469466,0.000872793,-0.040353082,-0.034516506,-0.036047976,0.004999689,-0.01570079,-0.09783203,1.4186912e-33,-0.0130903,0.04742377,0.02889633,0.09809088,0.043639127,-0.0747531,0.03402747,0.029147558,0.019848268,0.111865185,0.04845458,-0.016222177,0.025501896,0.09813583,-0.002833195,0.0038929228,0.048537854,0.014111116,0.036570966,-0.017512968,-0.079755045,0.043452248,0.10843892,0.061968718,-0.015678233,0.0060376576,0.010718569,-0.03907552,0.040307663,0.04553227,0.041731495,0.083179526,-0.0844024,-0.023447752,0.014485363,-0.00788756,-0.11410207,0.039309476,0.011220049,0.12889338,-0.07970701,0.010658111,-0.06657531,-0.022037188,-0.043459363,-0.030291978,0.05803613,0.06109021,-0.09415837,0.030727232,0.011858702,0.013526454,-0.049580045,-0.052486703,0.013218615,0.0050231596,0.07660412,-0.0046343356,-0.0692584,-0.03353194,0.022288544,0.022836983,0.01361394,-0.036331616,0.008194689,0.09226991,0.08229875,-0.047639843,-0.005033471,0.0066081914,-0.106666416,-0.06385503,0.006385333,0.06318358,0.03146985,-0.060764216,-0.043287244,-0.042385958,-0.07963737,-0.043478705,-0.031840656,0.060027804,0.027829666,0.03728562,-0.07014622,0.0649568,-0.052579004,-0.0034938955,-0.15390792,-0.04426162,0.028217599,0.05593145,0.023251446,0.029115746,-0.040321343,-1.8832651e-08,-0.037536032,0.0005056563,0.013919978,-0.038129315,0.061486673,0.005534046,-0.010768211,-0.16527015,0.061414365,0.019963786,-0.019666849,-0.059065994,0.00083181454,-0.016695915,0.01090875,0.02685501,-0.0008743374,-0.060967337,-0.049258173,0.012731965,-0.03973743,-0.0018210232,-0.03797001,0.00035104176,0.04477184,0.04185603,-0.03215796,-0.09103283,0.0467481,0.017810425,0.037437033,0.054782324,-0.0059442874,0.022557084,0.053393807,-0.03746794,-0.07080435,0.10548834,-0.04215435,0.042547524,0.05064807,0.07560324,0.04072184,-0.0011278372,0.00702018,-0.070807114,0.040035766,0.051417697,0.011316928,-0.028955372,0.04197099,-0.0026867494,-0.056857754,0.0761941,0.030621601,-0.052413527,-0.021492323,0.0957355,0.1013111,-0.082977295,-0.036080915,-0.03327806,0.0029404552,-0.03586191]'::vector)", "Filter": "(cohort = 'static'::text)" } ] } } ] } }, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "eligible": true, "protocol": "mixed-mutations-v2" }, "examples": [ { "event": 0, "operation": "insert", "doc_id": "mutation-000000", "scheduled_s": 0.0, "submitted_s": 0.0005194590194150805, "schedule_delay_ms": 0.5194590194150805, "precondition": true, "status": "success", "ack_ms": 20.986290997825563, "write_started_ms": 1.2213329318910837, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 26.551582966931164, "stable_since_ms": 26.551582966931164, "confirmed_ms": 77.80170801561326, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 30.82954091951251, "stable_since_ms": 30.82954091951251, "confirmed_ms": 85.15908301342279, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 30.82954091951251, "confirmed_ms": 85.15908301342279, "search_minus_ack_ms": 9.843249921686947 }, { "event": 1, "operation": "update", "doc_id": "mutation-000001", "scheduled_s": 0.1, "submitted_s": 0.10019937495235354, "schedule_delay_ms": 0.19937495235353153, "precondition": true, "status": "success", "ack_ms": 6.9706670474261045, "write_started_ms": 0.22045907098799944, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 12.580209062434733, "stable_since_ms": 12.580209062434733, "confirmed_ms": 64.67441702261567, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 19.242624985054135, "stable_since_ms": 19.242624985054135, "confirmed_ms": 76.3565000379458, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 19.242624985054135, "confirmed_ms": 76.3565000379458, "search_minus_ack_ms": 12.27195793762803 }, { "event": 2, "operation": "delete", "doc_id": "mutation-000002", "scheduled_s": 0.2, "submitted_s": 0.2002910419832915, "schedule_delay_ms": 0.29104198329149567, "precondition": true, "status": "success", "ack_ms": 1.8316250061616302, "write_started_ms": 0.12487499043345451, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 12.286667013540864, "stable_since_ms": 12.286667013540864, "confirmed_ms": 64.47087496053427, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 4.012000048533082, "stable_since_ms": 4.012000048533082, "confirmed_ms": 66.32729200646281, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 12.286667013540864, "confirmed_ms": 66.32729200646281, "search_minus_ack_ms": 2.180375042371452 } ] }, { "run": "fb2_429e9a_025", "trial": 1, "backend": "qdrant", "scenario": "nq-10k", "status": "complete", "eligible": true, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "summary": { "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "wall_s": 30.053141999989748, "operations": { "insert": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 19.238666049204767, "p50": 66.70652102911845, "p95": 80.69105662871146, "p99": 113.88499070191762, "max": 123.54479206260294 }, "confirmed_ms": { "min": 111.99208302423358, "p50": 211.53077099006623, "p95": 216.64947674726136, "p99": 258.2336916355417, "max": 265.19504201132804 }, "ack_ms": { "min": 4.108334076590836, "p50": 9.326688013970852, "p95": 14.081193495076146, "p99": 28.11504118726592, "max": 34.20345904305577 }, "search_minus_ack_ms": { "min": 0.7804159540683031, "p50": 56.995854480192065, "p95": 63.30159548087969, "p99": 96.580449026078, "max": 98.5759999603033 }, "regressions": 0, "confirmed_within_100ms": 0.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "update": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 17.17212493531406, "p50": 66.46104156970978, "p95": 69.41917531075887, "p99": 71.75155517761597, "max": 74.57445794716477 }, "confirmed_ms": { "min": 109.6631659893319, "p50": 211.2656455137767, "p95": 216.19951814063825, "p99": 219.98086123960093, "max": 222.00899990275502 }, "ack_ms": { "min": 5.406041978858411, "p50": 9.188249474391341, "p95": 12.046485411701722, "p99": 14.25467228866185, "max": 19.440416945144534 }, "search_minus_ack_ms": { "min": 1.1647919891402125, "p50": 56.651062972377986, "p95": 58.79922858439386, "p99": 60.154294306412346, "max": 61.55300000682473 }, "regressions": 0, "confirmed_within_100ms": 0.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "delete": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 7.0946249179542065, "p50": 18.440229003317654, "p95": 59.267306269612156, "p99": 65.33980867476204, "max": 67.37095804419369 }, "confirmed_ms": { "min": 104.9001250648871, "p50": 151.77833300549537, "p95": 203.17447943380103, "p99": 211.25313914613798, "max": 212.78904099017382 }, "ack_ms": { "min": 3.893457935191691, "p50": 7.741125009488314, "p95": 10.97663542605005, "p99": 12.033147972542805, "max": 14.473292045295238 }, "search_minus_ack_ms": { "min": -2.860250067897141, "p50": 0.17539598047733307, "p95": 52.85351878264918, "p99": 57.46230160584673, "max": 57.54166701808572 }, "regressions": 0, "confirmed_within_100ms": 0.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 } }, "total_operations": 300, "statuses": { "success": 300, "timeout": 0, "error": 0, "invalid_precondition": 0, "client_overload": 0 }, "query": { "count": 300, "success": 300, "ann_recall_at_10": 1.0, "ndcg_at_10": 0.9481505328688746, "latency_ms": { "min": 4.214332904666662, "p50": 8.313624479342252, "p95": 11.342126945964994, "p99": 47.90172750828787, "max": 57.72354197688401 }, "dispatch_delay_ms": { "min": 0.1330419909209013, "p50": 1.3370420201681554, "p95": 2.01071102055721, "p99": 2.4918429646641003, "max": 2.8658340452238917 } }, "diagnostics": { "schedule_delay_ms": { "min": 0.007666973395714649, "p50": 0.3151670098308357, "p95": 0.637472954113083, "p99": 0.8710643970414704, "max": 1.420042011886835 }, "probe_dispatch_ms": { "min": 0.025084009394049644, "p50": 0.39562495658174157, "p95": 1.6006811405532058, "p99": 2.0546695520170033, "max": 3.559666918590665 }, "probe_service_ms": { "min": 1.012749969959259, "p50": 6.130603956989944, "p95": 47.87676685373299, "p99": 48.96941509330646, "max": 56.30829092115164 }, "write_service_ms": { "min": 3.598166978918016, "p50": 8.221270516514778, "p95": 11.534071172354745, "p99": 18.954240652965336, "max": 33.451874973252416 } }, "actual_submission_rate": 10.000030295438897, "metadata": { "backend": "qdrant", "collection": "fb2_429e9a_025", "dataset": { "dataset": "nq", "source": "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/nq.zip", "source_sha256": "2553bf7bfbab47b1436ca00a34bce57320e18e611fd00999a2a3a1b4714be752", "source_md5": "d4d3d2e48787a744b6f6e691ff534307", "records": 124000, "queries": 100, "relevant_documents_retained": 120, "seed": 20261028, "license": "CC-BY-SA-3.0", "license_source": "https://raw.githubusercontent.com/google-research-datasets/natural-questions/master/nq_open/README.md", "attribution": "BEIR and original dataset authors; Wikipedia contributors for NQ, SciFact authors and cited article authors for SciFact.", "transformation": "200-word title+text prefix; seeded reservoir of documents with relevant documents for selected test queries retained at front. Simulated mutations; no historical replay.", "model": "sentence-transformers/all-MiniLM-L6-v2", "model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "dimensions": 384, "normalized": true, "max_tokens": 256, "device": "mps", "files": { "corpus.jsonl": "9dc487d785e6ee493eccaf764daa3bf8720a8f4fc8c406246065e1ad33a91f5d", "queries.json": "5603f52876ada5d36118b65dea490f1d43900bc159e1804e637796eb9115e999", "vectors.npy": "cb7421eb6ca5964231403f0a922843088f6fd0e01cbf67f2a1076cc7101005db", "query_vectors.npy": "5e666f8d966d8db6b0fd93d4963234d2dffa0ee3910b8fad069a8ee201e8b5d4", "UPSTREAM-LICENSE.md": "8b675521da3fa142ac6bda38c9bcfcc8cff0e2d6128be53a062ed55fb53f7b87" } }, "base_count": 10000, "profile": { "dataset": "nq", "base_count": 10000, "rate": 10, "query_rate": 10, "arrival": "steady" }, "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "preloaded_mutable": 200, "started_utc": "2026-10-29T03:09:50Z", "setup_s": 3.4781102909473702, "backend_before": { "version": "1.20.0", "collection": { "status": "green", "optimizer_status": "ok", "indexed_vectors_count": 10200, "points_count": 10200, "segments_count": 2, "config": { "params": { "vectors": { "size": 384, "distance": "Cosine" }, "shard_number": 1, "replication_factor": 1, "write_consistency_factor": 1, "on_disk_payload": true }, "hnsw_config": { "m": 16, "ef_construct": 128, "full_scan_threshold": 10, "max_indexing_threads": 0, "on_disk": false }, "optimizer_config": { "deleted_threshold": 0.2, "vacuum_min_vector_number": 1000, "default_segment_number": 0, "max_segment_size": null, "memmap_threshold": null, "indexing_threshold": 1024, "flush_interval_sec": 5, "max_optimization_threads": null, "prevent_unoptimized": null }, "wal_config": { "wal_capacity_mb": 32, "wal_segments_ahead": 0, "wal_retain_closed": 1 }, "quantization_config": null }, "payload_schema": { "cohort": { "data_type": "keyword", "points": 10200 } }, "update_queue": { "length": 0 }, "created_at": "2026-10-29T03:09:50.264Z" }, "ack": "wait=true", "transport": "HTTP REST, pooled httpx" }, "count_before": 10200, "idle_duration_s": 10.0, "idle_queries": { "count": 100, "ann_recall": 1.0, "ndcg": 0.9481505328688745, "latency_ms": { "min": 2.38508393522352, "p50": 7.354250003118068, "p95": 10.163914895383641, "p99": 18.209505011327703, "max": 45.00187502708286 } }, "exact_ndcg": 0.9481505328688745, "resources_before": { "usage_usec": 241254326, "user_usec": 228233388, "system_usec": 13020938, "nice_usec": 174169000, "nr_periods": 4763, "nr_throttled": 382, "throttled_usec": 2082403, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 408801280, "pids": 31 }, "resources_after": { "usage_usec": 247334070, "user_usec": 233596660, "system_usec": 13737409, "nice_usec": 174169000, "nr_periods": 5064, "nr_throttled": 382, "throttled_usec": 2082403, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 396972032, "pids": 33 }, "count_after": 10200, "backend_after": { "version": "1.20.0", "collection": { "status": "green", "optimizer_status": "ok", "indexed_vectors_count": 10200, "points_count": 10200, "segments_count": 2, "config": { "params": { "vectors": { "size": 384, "distance": "Cosine" }, "shard_number": 1, "replication_factor": 1, "write_consistency_factor": 1, "on_disk_payload": true }, "hnsw_config": { "m": 16, "ef_construct": 128, "full_scan_threshold": 10, "max_indexing_threads": 0, "on_disk": false }, "optimizer_config": { "deleted_threshold": 0.2, "vacuum_min_vector_number": 1000, "default_segment_number": 0, "max_segment_size": null, "memmap_threshold": null, "indexing_threshold": 1024, "flush_interval_sec": 5, "max_optimization_threads": null, "prevent_unoptimized": null }, "wal_config": { "wal_capacity_mb": 32, "wal_segments_ahead": 0, "wal_retain_closed": 1 }, "quantization_config": null }, "payload_schema": { "cohort": { "data_type": "keyword", "points": 10200 } }, "update_queue": { "length": 0 }, "created_at": "2026-10-29T03:09:50.264Z" }, "ack": "wait=true", "transport": "HTTP REST, pooled httpx" } }, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "eligible": true, "protocol": "mixed-mutations-v2" }, "examples": [ { "event": 0, "operation": "insert", "doc_id": "mutation-000000", "scheduled_s": 0.0, "submitted_s": 0.0002292500576004386, "schedule_delay_ms": 0.2292500576004386, "precondition": true, "status": "success", "ack_ms": 6.8058749893680215, "write_started_ms": 1.3563749380409718, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 15.043166931718588, "stable_since_ms": 15.043166931718588, "confirmed_ms": 69.02387493755668, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 59.302959009073675, "stable_since_ms": 59.302959009073675, "confirmed_ms": 203.1749589368701, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 59.302959009073675, "confirmed_ms": 203.1749589368701, "search_minus_ack_ms": 52.49708401970565 }, { "event": 1, "operation": "update", "doc_id": "mutation-000001", "scheduled_s": 0.1, "submitted_s": 0.10004329204093665, "schedule_delay_ms": 0.043292040936643295, "precondition": true, "status": "success", "ack_ms": 9.46658302564174, "write_started_ms": 0.7141249952837825, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 17.42495794314891, "stable_since_ms": 17.42495794314891, "confirmed_ms": 73.06841702666134, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 63.99887497536838, "stable_since_ms": 63.99887497536838, "confirmed_ms": 207.78712502215058, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 63.99887497536838, "confirmed_ms": 207.78712502215058, "search_minus_ack_ms": 54.53229194972664 }, { "event": 2, "operation": "delete", "doc_id": "mutation-000002", "scheduled_s": 0.2, "submitted_s": 0.20002058404497802, "schedule_delay_ms": 0.020584044978011473, "precondition": true, "status": "success", "ack_ms": 7.938290946185589, "write_started_ms": 0.5587909836322069, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 17.711499938741326, "stable_since_ms": 17.711499938741326, "confirmed_ms": 72.62925000395626, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 7.431083009578288, "stable_since_ms": 7.431083009578288, "confirmed_ms": 109.54787500668317, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 17.711499938741326, "confirmed_ms": 109.54787500668317, "search_minus_ack_ms": -0.5072079366073012 } ] }, { "run": "fb2_429e9a_026", "trial": 1, "backend": "weaviate", "scenario": "nq-100k", "status": "complete", "eligible": true, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "summary": { "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "wall_s": 30.00124999997206, "operations": { "insert": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 14.223707956261933, "p50": 17.67370803281665, "p95": 33.22017741156742, "p99": 34.10216313088313, "max": 34.11424998193979 }, "confirmed_ms": { "min": 67.47466593515128, "p50": 84.41902149934322, "p95": 105.6592271081172, "p99": 106.38638086384162, "max": 107.48833301477134 }, "ack_ms": { "min": 4.840499954298139, "p50": 10.633520490955561, "p95": 14.86751438351348, "p99": 15.531867402605714, "max": 15.960084041580558 }, "search_minus_ack_ms": { "min": -0.26416603941470385, "p50": 4.218166461214423, "p95": 19.24542881315574, "p99": 19.97394839883782, "max": 20.232875016517937 }, "regressions": 0, "confirmed_within_100ms": 0.83, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "update": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 5.990291014313698, "p50": 17.50331255607307, "p95": 30.47465423005633, "p99": 33.95140045206063, "max": 35.20379099063575 }, "confirmed_ms": { "min": 65.82162505947053, "p50": 84.40966700436547, "p95": 103.45267889788374, "p99": 106.60174466436732, "max": 113.31262497697026 }, "ack_ms": { "min": 5.277457996271551, "p50": 9.858499513939023, "p95": 14.190658065490425, "p99": 16.727762413211206, "max": 24.129250086843967 }, "search_minus_ack_ms": { "min": 0.10983296670019627, "p50": 3.75016697216779, "p95": 17.805712099652734, "p99": 20.145734257530425, "max": 20.767083042301238 }, "regressions": 0, "confirmed_within_100ms": 0.88, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "delete": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 8.200000040233135, "p50": 16.336041502654552, "p95": 20.25102086481638, "p99": 24.100982920499632, "max": 46.26291699241847 }, "confirmed_ms": { "min": 68.11104097869247, "p50": 82.10318750934675, "p95": 87.76533509371802, "p99": 94.53533423831696, "max": 114.71908399835229 }, "ack_ms": { "min": 3.1489580869674683, "p50": 5.110249971039593, "p95": 8.84473510668613, "p99": 12.278442776296414, "max": 36.56074998434633 }, "search_minus_ack_ms": { "min": 0.12737500946968794, "p50": 6.160937016829848, "p95": 7.184647908434271, "p99": 13.257204020628713, "max": 14.481833088211715 }, "regressions": 0, "confirmed_within_100ms": 0.99, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 } }, "total_operations": 300, "statuses": { "success": 300, "timeout": 0, "error": 0, "invalid_precondition": 0, "client_overload": 0 }, "query": { "count": 300, "success": 300, "ann_recall_at_10": 0.991, "ndcg_at_10": 0.7931268675492565, "latency_ms": { "min": 4.207000019960105, "p50": 10.847416531760246, "p95": 14.928012469317766, "p99": 18.667150521650903, "max": 41.74095904454589 }, "dispatch_delay_ms": { "min": 0.1342919422313571, "p50": 1.525041472632438, "p95": 2.322496136184782, "p99": 2.861216167220845, "max": 3.2673749374225736 } }, "diagnostics": { "schedule_delay_ms": { "min": 0.007542036473573432, "p50": 0.3131039789874279, "p95": 0.6238749506882968, "p99": 0.6451000235975668, "max": 0.705041992477895 }, "probe_dispatch_ms": { "min": 0.040708924643695354, "p50": 0.39035402005538344, "p95": 2.4627521983347833, "p99": 2.759848198620602, "max": 3.492125077173114 }, "probe_service_ms": { "min": 0.8821249939501286, "p50": 4.861979512497783, "p95": 11.8858773377724, "p99": 14.429189495276656, "max": 40.77399999368936 }, "write_service_ms": { "min": 2.9075839556753635, "p50": 8.654145989567041, "p95": 13.452681270428002, "p99": 15.324217397719616, "max": 35.84687504917383 } }, "actual_submission_rate": 10.000041820249521, "metadata": { "backend": "weaviate", "collection": "fb2_429e9a_026", "dataset": { "dataset": "nq", "source": "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/nq.zip", "source_sha256": "2553bf7bfbab47b1436ca00a34bce57320e18e611fd00999a2a3a1b4714be752", "source_md5": "d4d3d2e48787a744b6f6e691ff534307", "records": 124000, "queries": 100, "relevant_documents_retained": 120, "seed": 20261028, "license": "CC-BY-SA-3.0", "license_source": "https://raw.githubusercontent.com/google-research-datasets/natural-questions/master/nq_open/README.md", "attribution": "BEIR and original dataset authors; Wikipedia contributors for NQ, SciFact authors and cited article authors for SciFact.", "transformation": "200-word title+text prefix; seeded reservoir of documents with relevant documents for selected test queries retained at front. Simulated mutations; no historical replay.", "model": "sentence-transformers/all-MiniLM-L6-v2", "model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "dimensions": 384, "normalized": true, "max_tokens": 256, "device": "mps", "files": { "corpus.jsonl": "9dc487d785e6ee493eccaf764daa3bf8720a8f4fc8c406246065e1ad33a91f5d", "queries.json": "5603f52876ada5d36118b65dea490f1d43900bc159e1804e637796eb9115e999", "vectors.npy": "cb7421eb6ca5964231403f0a922843088f6fd0e01cbf67f2a1076cc7101005db", "query_vectors.npy": "5e666f8d966d8db6b0fd93d4963234d2dffa0ee3910b8fad069a8ee201e8b5d4", "UPSTREAM-LICENSE.md": "8b675521da3fa142ac6bda38c9bcfcc8cff0e2d6128be53a062ed55fb53f7b87" } }, "base_count": 100000, "profile": { "dataset": "nq", "base_count": 100000, "rate": 10, "query_rate": 10, "arrival": "steady" }, "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "preloaded_mutable": 200, "started_utc": "2026-10-29T03:10:39Z", "setup_s": 35.94634058396332, "backend_before": { "version": "1.40.3", "schema": { "class": "Ffb2429e9a026", "invertedIndexConfig": { "bm25": { "b": 0.75, "k1": 1.2 }, "cleanupIntervalSeconds": 60, "stopwords": { "additions": null, "preset": "en", "removals": null }, "usingBlockMaxWAND": true }, "multiTenancyConfig": { "autoTenantActivation": false, "autoTenantCreation": false, "enabled": false }, "properties": [ { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "doc_id", "tokenization": "field" }, { "dataType": [ "int" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": false, "name": "version" }, { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "cohort", "tokenization": "field" }, { "dataType": [ "text" ], "indexFilterable": false, "indexRangeFilters": false, "indexSearchable": false, "name": "text", "tokenization": "word" } ], "shardingConfig": { "actualCount": 1, "actualVirtualCount": 128, "desiredCount": 1, "desiredVirtualCount": 128, "function": "murmur3", "key": "_id", "strategy": "hash", "virtualPerPhysical": 128 }, "vectorIndexConfig": { "bq": { "enabled": false }, "cleanupIntervalSeconds": 300, "distance": "cosine", "dynamicEfFactor": 8, "dynamicEfMax": 500, "dynamicEfMin": 100, "ef": 128, "efConstruction": 128, "filterStrategy": "acorn", "flatSearchCutoff": 0, "maxConnections": 16, "multivector": { "aggregation": "maxSim", "enabled": false, "muvera": { "dprojections": 16, "enabled": false, "ksim": 4, "repetitions": 10 } }, "pq": { "bitCompression": false, "centroids": 256, "enabled": false, "encoder": { "distribution": "log-normal", "type": "kmeans" }, "segments": 0, "trainingLimit": 100000 }, "rq": { "bits": 8, "centering": false, "enabled": false, "rescoreLimit": 20, "trainingLimit": 10000 }, "skip": false, "skipDefaultQuantization": false, "sq": { "enabled": false, "rescoreLimit": 20, "trainingLimit": 100000 }, "trackDefaultQuantization": false, "vectorCacheMaxObjects": 1000000000000 }, "vectorIndexType": "hnsw", "vectorizer": "none", "replicationConfig": { "deletionStrategy": "TimeBasedResolution", "factor": 1, "asyncEnabled": false } }, "shards": [ { "name": "weBQv7u8yH8F", "per_node_status": { "node1": "READY" }, "status": "READY", "vectorQueueSize": 0 } ], "ack": "synchronous indexing, object PUT/POST/DELETE", "transport": "HTTP REST writes + GraphQL search, pooled httpx" }, "count_before": 100200, "idle_duration_s": 10.0, "idle_queries": { "count": 100, "ann_recall": 0.991, "ndcg": 0.7931268675492565, "latency_ms": { "min": 5.196916055865586, "p50": 12.145583983510733, "p95": 13.368631224147975, "p99": 13.658125993097203, "max": 13.971665990538895 } }, "exact_ndcg": 0.7931268675492565, "resources_before": { "usage_usec": 280544616, "user_usec": 274250819, "system_usec": 6293797, "nice_usec": 0, "nr_periods": 7276, "nr_throttled": 379, "throttled_usec": 1333616, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 1330950144, "pids": 11 }, "resources_after": { "usage_usec": 290162275, "user_usec": 283462856, "system_usec": 6699419, "nice_usec": 0, "nr_periods": 7578, "nr_throttled": 379, "throttled_usec": 1333616, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 1215512576, "pids": 11 }, "count_after": 100200, "backend_after": { "version": "1.40.3", "schema": { "class": "Ffb2429e9a026", "invertedIndexConfig": { "bm25": { "b": 0.75, "k1": 1.2 }, "cleanupIntervalSeconds": 60, "stopwords": { "additions": null, "preset": "en", "removals": null }, "usingBlockMaxWAND": true }, "multiTenancyConfig": { "autoTenantActivation": false, "autoTenantCreation": false, "enabled": false }, "properties": [ { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "doc_id", "tokenization": "field" }, { "dataType": [ "int" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": false, "name": "version" }, { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "cohort", "tokenization": "field" }, { "dataType": [ "text" ], "indexFilterable": false, "indexRangeFilters": false, "indexSearchable": false, "name": "text", "tokenization": "word" } ], "shardingConfig": { "actualCount": 1, "actualVirtualCount": 128, "desiredCount": 1, "desiredVirtualCount": 128, "function": "murmur3", "key": "_id", "strategy": "hash", "virtualPerPhysical": 128 }, "vectorIndexConfig": { "bq": { "enabled": false }, "cleanupIntervalSeconds": 300, "distance": "cosine", "dynamicEfFactor": 8, "dynamicEfMax": 500, "dynamicEfMin": 100, "ef": 128, "efConstruction": 128, "filterStrategy": "acorn", "flatSearchCutoff": 0, "maxConnections": 16, "multivector": { "aggregation": "maxSim", "enabled": false, "muvera": { "dprojections": 16, "enabled": false, "ksim": 4, "repetitions": 10 } }, "pq": { "bitCompression": false, "centroids": 256, "enabled": false, "encoder": { "distribution": "log-normal", "type": "kmeans" }, "segments": 0, "trainingLimit": 100000 }, "rq": { "bits": 8, "centering": false, "enabled": false, "rescoreLimit": 20, "trainingLimit": 10000 }, "skip": false, "skipDefaultQuantization": false, "sq": { "enabled": false, "rescoreLimit": 20, "trainingLimit": 100000 }, "trackDefaultQuantization": false, "vectorCacheMaxObjects": 1000000000000 }, "vectorIndexType": "hnsw", "vectorizer": "none", "replicationConfig": { "deletionStrategy": "TimeBasedResolution", "factor": 1, "asyncEnabled": false } }, "shards": [ { "name": "weBQv7u8yH8F", "per_node_status": { "node1": "READY" }, "status": "READY", "vectorQueueSize": 0 } ], "ack": "synchronous indexing, object PUT/POST/DELETE", "transport": "HTTP REST writes + GraphQL search, pooled httpx" } }, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "eligible": true, "protocol": "mixed-mutations-v2" }, "examples": [ { "event": 0, "operation": "insert", "doc_id": "mutation-000000", "scheduled_s": 0.0, "submitted_s": 0.00019279203843325377, "schedule_delay_ms": 0.19279203843325377, "precondition": true, "status": "success", "ack_ms": 5.582207930274308, "write_started_ms": 1.2859159614890814, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 14.223707956261933, "stable_since_ms": 14.223707956261933, "confirmed_ms": 67.47466593515128, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 7.407999946735799, "stable_since_ms": 7.407999946735799, "confirmed_ms": 65.71420794352889, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 14.223707956261933, "confirmed_ms": 67.47466593515128, "search_minus_ack_ms": 1.8257920164614916 }, { "event": 1, "operation": "update", "doc_id": "mutation-000001", "scheduled_s": 0.1, "submitted_s": 0.10036974994000047, "schedule_delay_ms": 0.3697499400004689, "precondition": true, "status": "success", "ack_ms": 11.843583080917597, "write_started_ms": 0.29612507205456495, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 9.85220808070153, "stable_since_ms": 9.85220808070153, "confirmed_ms": 64.6444580052048, "consecutive": 3, "regressions": 0, "requests": 3 }, "search": { "first_ms": 25.61220806092024, "stable_since_ms": 25.61220806092024, "confirmed_ms": 93.64316705614328, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 25.61220806092024, "confirmed_ms": 93.64316705614328, "search_minus_ack_ms": 13.768624980002642 }, { "event": 2, "operation": "delete", "doc_id": "mutation-000002", "scheduled_s": 0.2, "submitted_s": 0.20032933296170086, "schedule_delay_ms": 0.3293329617008456, "precondition": true, "status": "success", "ack_ms": 4.499792004935443, "write_started_ms": 0.3403340233489871, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 15.942084020934999, "stable_since_ms": 15.942084020934999, "confirmed_ms": 70.50308398902416, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 11.03145896922797, "stable_since_ms": 11.03145896922797, "confirmed_ms": 81.1260420596227, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 15.942084020934999, "confirmed_ms": 81.1260420596227, "search_minus_ack_ms": 6.531666964292526 } ] }, { "run": "fb2_429e9a_027", "trial": 5, "backend": "qdrant", "scenario": "scifact-small", "status": "complete", "eligible": true, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "summary": { "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "wall_s": 30.05199920805171, "operations": { "insert": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 18.613292020745575, "p50": 65.13185449875891, "p95": 104.41014791140333, "p99": 107.38192265038379, "max": 107.92708292137831 }, "confirmed_ms": { "min": 111.70429200865328, "p50": 209.10212449962273, "p95": 216.07786495587789, "p99": 249.14042622898708, "max": 252.1031249780208 }, "ack_ms": { "min": 3.7632089806720614, "p50": 9.32631257455796, "p95": 10.953242238610983, "p99": 13.210106373298949, "max": 20.03520797006786 }, "search_minus_ack_ms": { "min": 0.1669999910518527, "p50": 55.6714374688454, "p95": 96.28597489790991, "p99": 98.33632934256457, "max": 99.11974996794015 }, "regressions": 0, "confirmed_within_100ms": 0.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "update": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 58.222000021487474, "p50": 64.69716649735346, "p95": 67.1088270668406, "p99": 84.63931811391397, "max": 125.07933401502669 }, "confirmed_ms": { "min": 196.9225419452414, "p50": 209.04262497788295, "p95": 212.79386245878413, "p99": 227.9453784157524, "max": 268.8368749804795 }, "ack_ms": { "min": 4.64737496804446, "p50": 8.697541547007859, "p95": 10.499040136346593, "p99": 30.931365763535727, "max": 31.672958051785827 }, "search_minus_ack_ms": { "min": 50.613459083251655, "p50": 55.80420856131241, "p95": 58.20842052344233, "p99": 59.933839922305374, "max": 94.15545908268541 }, "regressions": 0, "confirmed_within_100ms": 0.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "delete": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 14.827542006969452, "p50": 18.677125044632703, "p95": 58.19412565906532, "p99": 64.71690577454865, "max": 65.45008299872279 }, "confirmed_ms": { "min": 105.50387494731694, "p50": 149.8047505156137, "p95": 198.76028327271342, "p99": 209.8023503238801, "max": 211.44579199608415 }, "ack_ms": { "min": 3.7847920320928097, "p50": 7.727437536232173, "p95": 9.18498927494511, "p99": 10.209621307440102, "max": 10.283500072546303 }, "search_minus_ack_ms": { "min": -2.0871669985353947, "p50": 0.21156301954761147, "p95": 53.623856318881735, "p99": 56.08530241064727, "max": 56.432834011502564 }, "regressions": 0, "confirmed_within_100ms": 0.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 } }, "total_operations": 300, "statuses": { "success": 300, "timeout": 0, "error": 0, "invalid_precondition": 0, "client_overload": 0 }, "query": { "count": 300, "success": 300, "ann_recall_at_10": 1.0, "ndcg_at_10": 0.6567157762068453, "latency_ms": { "min": 3.4129590494558215, "p50": 7.197729020845145, "p95": 9.489391755778342, "p99": 46.921645326074206, "max": 48.9113749936223 }, "dispatch_delay_ms": { "min": 0.0825830502435565, "p50": 1.5097080613486469, "p95": 2.292888954980299, "p99": 3.1636930524837217, "max": 3.680541063658893 } }, "diagnostics": { "schedule_delay_ms": { "min": 0.007166038265893349, "p50": 0.3196035278965059, "p95": 0.6232430937240353, "p99": 0.6473866479922918, "max": 0.6545830518014384 }, "probe_dispatch_ms": { "min": 0.0274579506367445, "p50": 0.4007290117442608, "p95": 2.3262285452801734, "p99": 2.69537696498446, "max": 27.232250082306564 }, "probe_service_ms": { "min": 0.8487079758197069, "p50": 5.61416702112183, "p95": 47.579384729033336, "p99": 48.62152707064524, "max": 55.5643750121817 }, "write_service_ms": { "min": 3.4782920265570283, "p50": 7.980749476701021, "p95": 9.977970860200003, "p99": 12.491789375198938, "max": 30.966707970947027 } }, "actual_submission_rate": 9.999941750854525, "metadata": { "backend": "qdrant", "collection": "fb2_429e9a_027", "dataset": { "dataset": "scifact", "source": "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/scifact.zip", "source_sha256": "536e14446a0ba56ed1398ab1055f39fe852686ecad24a6306c80c490fa8e0165", "source_md5": "5f7d1de60b170fc8027bb7898e2efca1", "records": 5183, "queries": 100, "relevant_documents_retained": 107, "seed": 20261028, "license": "Corpus: ODC-By-1.0; claims/evidence: CC-BY-4.0", "license_source": "https://raw.githubusercontent.com/allenai/scifact/master/LICENSE.md", "attribution": "BEIR and original dataset authors; Wikipedia contributors for NQ, SciFact authors and cited article authors for SciFact.", "transformation": "200-word title+text prefix; seeded reservoir of documents with relevant documents for selected test queries retained at front. Simulated mutations; no historical replay.", "model": "sentence-transformers/all-MiniLM-L6-v2", "model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "dimensions": 384, "normalized": true, "max_tokens": 256, "device": "mps", "files": { "corpus.jsonl": "189d58f273468352c9f73c6333c5120dcfb7ba1695984f073f0ebb6c100ed0cd", "queries.json": "c8b333c73353a8c4aeb7d6b63067c9a241aa5d39a0314f0c315ec63019ee9558", "vectors.npy": "b6b21404977151ce565e7d6d57d9931a4679e1ecf4aa44fcd9b29eb54955d09e", "query_vectors.npy": "02324eb7c3b3d2e2f0ec3208453e2ec0efd5d61d9fc845b1f8b7950e913f2730", "UPSTREAM-LICENSE.md": "56ee69bc639cbab8f0b2e4b67091ec5e9390aa99a1d65f0a35e4274686c9a806" } }, "base_count": 4000, "profile": { "dataset": "scifact", "base_count": 4000, "rate": 10, "query_rate": 10, "arrival": "steady" }, "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "preloaded_mutable": 200, "started_utc": "2026-10-29T03:12:00Z", "setup_s": 1.701082415995188, "backend_before": { "version": "1.20.0", "collection": { "status": "green", "optimizer_status": "ok", "indexed_vectors_count": 4200, "points_count": 4200, "segments_count": 2, "config": { "params": { "vectors": { "size": 384, "distance": "Cosine" }, "shard_number": 1, "replication_factor": 1, "write_consistency_factor": 1, "on_disk_payload": true }, "hnsw_config": { "m": 16, "ef_construct": 128, "full_scan_threshold": 10, "max_indexing_threads": 0, "on_disk": false }, "optimizer_config": { "deleted_threshold": 0.2, "vacuum_min_vector_number": 1000, "default_segment_number": 0, "max_segment_size": null, "memmap_threshold": null, "indexing_threshold": 1024, "flush_interval_sec": 5, "max_optimization_threads": null, "prevent_unoptimized": null }, "wal_config": { "wal_capacity_mb": 32, "wal_segments_ahead": 0, "wal_retain_closed": 1 }, "quantization_config": null }, "payload_schema": { "cohort": { "data_type": "keyword", "points": 4200 } }, "update_queue": { "length": 0 }, "created_at": "2026-10-29T03:12:00.144Z" }, "ack": "wait=true", "transport": "HTTP REST, pooled httpx" }, "count_before": 4200, "idle_duration_s": 10.0, "idle_queries": { "count": 100, "ann_recall": 1.0, "ndcg": 0.6567157762068453, "latency_ms": { "min": 3.064415999688208, "p50": 7.194166479166597, "p95": 8.756419009296224, "p99": 9.851516161579847, "max": 50.17087503802031 } }, "exact_ndcg": 0.6567157762068453, "resources_before": { "usage_usec": 248982896, "user_usec": 234983817, "system_usec": 13999078, "nice_usec": 174583000, "nr_periods": 5239, "nr_throttled": 382, "throttled_usec": 2082403, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 364085248, "pids": 32 }, "resources_after": { "usage_usec": 253975725, "user_usec": 239290267, "system_usec": 14685457, "nice_usec": 174583000, "nr_periods": 5541, "nr_throttled": 382, "throttled_usec": 2082403, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 359239680, "pids": 32 }, "count_after": 4200, "backend_after": { "version": "1.20.0", "collection": { "status": "green", "optimizer_status": "ok", "indexed_vectors_count": 4200, "points_count": 4200, "segments_count": 2, "config": { "params": { "vectors": { "size": 384, "distance": "Cosine" }, "shard_number": 1, "replication_factor": 1, "write_consistency_factor": 1, "on_disk_payload": true }, "hnsw_config": { "m": 16, "ef_construct": 128, "full_scan_threshold": 10, "max_indexing_threads": 0, "on_disk": false }, "optimizer_config": { "deleted_threshold": 0.2, "vacuum_min_vector_number": 1000, "default_segment_number": 0, "max_segment_size": null, "memmap_threshold": null, "indexing_threshold": 1024, "flush_interval_sec": 5, "max_optimization_threads": null, "prevent_unoptimized": null }, "wal_config": { "wal_capacity_mb": 32, "wal_segments_ahead": 0, "wal_retain_closed": 1 }, "quantization_config": null }, "payload_schema": { "cohort": { "data_type": "keyword", "points": 4200 } }, "update_queue": { "length": 0 }, "created_at": "2026-10-29T03:12:00.144Z" }, "ack": "wait=true", "transport": "HTTP REST, pooled httpx" } }, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "eligible": true, "protocol": "mixed-mutations-v2" }, "examples": [ { "event": 0, "operation": "insert", "doc_id": "mutation-000000", "scheduled_s": 0.0, "submitted_s": 0.0001727500930428505, "schedule_delay_ms": 0.1727500930428505, "precondition": true, "status": "success", "ack_ms": 5.174582940526307, "write_started_ms": 1.0375409619882703, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 13.4181659668684, "stable_since_ms": 13.4181659668684, "confirmed_ms": 66.7701659258455, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 58.90099995303899, "stable_since_ms": 58.90099995303899, "confirmed_ms": 197.91191595140845, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 58.90099995303899, "confirmed_ms": 197.91191595140845, "search_minus_ack_ms": 53.726417012512684 }, { "event": 1, "operation": "update", "doc_id": "mutation-000001", "scheduled_s": 0.1, "submitted_s": 0.10001354105770588, "schedule_delay_ms": 0.01354105770587366, "precondition": true, "status": "success", "ack_ms": 6.34329195600003, "write_started_ms": 0.44758396688848734, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 16.581792035140097, "stable_since_ms": 16.581792035140097, "confirmed_ms": 71.89433393068612, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 59.01737499516457, "stable_since_ms": 59.01737499516457, "confirmed_ms": 196.9225419452414, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 59.01737499516457, "confirmed_ms": 196.9225419452414, "search_minus_ack_ms": 52.67408303916454 }, { "event": 2, "operation": "delete", "doc_id": "mutation-000002", "scheduled_s": 0.2, "submitted_s": 0.20056854106951505, "schedule_delay_ms": 0.5685410695150384, "precondition": true, "status": "success", "ack_ms": 4.135583993047476, "write_started_ms": 0.3010840155184269, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 15.31883399002254, "stable_since_ms": 15.31883399002254, "confirmed_ms": 69.73362492863089, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 58.141459012404084, "stable_since_ms": 58.141459012404084, "confirmed_ms": 198.7303749192506, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 58.141459012404084, "confirmed_ms": 198.7303749192506, "search_minus_ack_ms": 54.00587501935661 } ] }, { "run": "fb2_429e9a_028", "trial": 2, "backend": "weaviate", "scenario": "nq-100k-burst", "status": "complete", "eligible": true, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "summary": { "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "burst", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "wall_s": 30.000929249916226, "operations": { "insert": { "count": 130, "success": 130, "success_rate": 1.0, "first_observed_ms": { "min": 7.271000067703426, "p50": 17.31556298909709, "p95": 31.26471490249969, "p99": 34.5388114999514, "max": 36.302249995060265 }, "confirmed_ms": { "min": 64.14029200095683, "p50": 82.17937545850873, "p95": 103.5491458082106, "p99": 106.32532957126385, "max": 116.62579199764878 }, "ack_ms": { "min": 2.1301249507814646, "p50": 9.652958484366536, "p95": 13.482012547319755, "p99": 14.95980501640588, "max": 15.181125025264919 }, "search_minus_ack_ms": { "min": 0.07995904888957739, "p50": 2.3293959675356746, "p95": 18.98793110740371, "p99": 20.852283064741634, "max": 21.487083984538913 }, "regressions": 0, "confirmed_within_100ms": 0.9076923076923077, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "update": { "count": 130, "success": 130, "success_rate": 1.0, "first_observed_ms": { "min": 3.21712507866323, "p50": 17.005729489028454, "p95": 32.93857527896762, "p99": 37.24643256398849, "max": 38.43237494584173 }, "confirmed_ms": { "min": 60.18587504513562, "p50": 81.10060397302732, "p95": 103.3983581524808, "p99": 106.06704252655618, "max": 107.09208308253437 }, "ack_ms": { "min": 2.5943750515580177, "p50": 8.955625002272427, "p95": 13.900799816474319, "p99": 16.289599613519385, "max": 17.345292028039694 }, "search_minus_ack_ms": { "min": 0.11545803863555193, "p50": 2.0103335264138877, "p95": 19.566360575845465, "p99": 24.61290761362761, "max": 25.535749970003963 }, "regressions": 0, "confirmed_within_100ms": 0.9153846153846154, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "delete": { "count": 130, "success": 130, "success_rate": 1.0, "first_observed_ms": { "min": 2.95691704377532, "p50": 15.585332992486656, "p95": 18.518729164497927, "p99": 20.682390216970816, "max": 27.350250049494207 }, "confirmed_ms": { "min": 58.289125096052885, "p50": 77.90037547238171, "p95": 86.66508560418151, "p99": 88.75637506484053, "max": 95.19095800351351 }, "ack_ms": { "min": 1.1747090611606836, "p50": 4.529812547843903, "p95": 8.173402457032353, "p99": 9.46918660425581, "max": 13.170542079024017 }, "search_minus_ack_ms": { "min": 0.08229096420109272, "p50": 4.263895971234888, "p95": 7.189210772048682, "p99": 8.937035327544427, "max": 17.963542020879686 }, "regressions": 0, "confirmed_within_100ms": 1.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 } }, "total_operations": 390, "statuses": { "success": 390, "timeout": 0, "error": 0, "invalid_precondition": 0, "client_overload": 0 }, "query": { "count": 300, "success": 300, "ann_recall_at_10": 0.994, "ndcg_at_10": 0.7931268675492565, "latency_ms": { "min": 2.729667001403868, "p50": 10.345478949602693, "p95": 14.804920769529417, "p99": 16.361587560968466, "max": 23.07075005955994 }, "dispatch_delay_ms": { "min": 0.09308289736509323, "p50": 1.4482703991234303, "p95": 2.2563847014680505, "p99": 2.970156604424119, "max": 7.655624998733401 } }, "diagnostics": { "schedule_delay_ms": { "min": 0.0034999335181140623, "p50": 0.3368329373190093, "p95": 0.6281373486846319, "p99": 0.6436772281304792, "max": 4.278290900401771 }, "probe_dispatch_ms": { "min": 0.01391698606312275, "p50": 0.3973750863224268, "p95": 2.2109170095063746, "p99": 2.6884206919930875, "max": 5.810333997942507 }, "probe_service_ms": { "min": 0.4896249156445265, "p50": 4.019333049654961, "p95": 11.485791474115103, "p99": 13.572804385330526, "max": 20.587125094607472 }, "write_service_ms": { "min": 1.081791939213872, "p50": 5.982916511129588, "p95": 12.287883745739233, "p99": 14.738485300913458, "max": 16.708999988622963 } }, "actual_submission_rate": 13.010003675421945, "metadata": { "backend": "weaviate", "collection": "fb2_429e9a_028", "dataset": { "dataset": "nq", "source": "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/nq.zip", "source_sha256": "2553bf7bfbab47b1436ca00a34bce57320e18e611fd00999a2a3a1b4714be752", "source_md5": "d4d3d2e48787a744b6f6e691ff534307", "records": 124000, "queries": 100, "relevant_documents_retained": 120, "seed": 20261028, "license": "CC-BY-SA-3.0", "license_source": "https://raw.githubusercontent.com/google-research-datasets/natural-questions/master/nq_open/README.md", "attribution": "BEIR and original dataset authors; Wikipedia contributors for NQ, SciFact authors and cited article authors for SciFact.", "transformation": "200-word title+text prefix; seeded reservoir of documents with relevant documents for selected test queries retained at front. Simulated mutations; no historical replay.", "model": "sentence-transformers/all-MiniLM-L6-v2", "model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "dimensions": 384, "normalized": true, "max_tokens": 256, "device": "mps", "files": { "corpus.jsonl": "9dc487d785e6ee493eccaf764daa3bf8720a8f4fc8c406246065e1ad33a91f5d", "queries.json": "5603f52876ada5d36118b65dea490f1d43900bc159e1804e637796eb9115e999", "vectors.npy": "cb7421eb6ca5964231403f0a922843088f6fd0e01cbf67f2a1076cc7101005db", "query_vectors.npy": "5e666f8d966d8db6b0fd93d4963234d2dffa0ee3910b8fad069a8ee201e8b5d4", "UPSTREAM-LICENSE.md": "8b675521da3fa142ac6bda38c9bcfcc8cff0e2d6128be53a062ed55fb53f7b87" } }, "base_count": 100000, "profile": { "dataset": "nq", "base_count": 100000, "rate": 10, "query_rate": 10, "arrival": "burst" }, "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "burst", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "preloaded_mutable": 260, "started_utc": "2026-10-29T03:12:47Z", "setup_s": 36.000539000029676, "backend_before": { "version": "1.40.3", "schema": { "class": "Ffb2429e9a028", "invertedIndexConfig": { "bm25": { "b": 0.75, "k1": 1.2 }, "cleanupIntervalSeconds": 60, "stopwords": { "additions": null, "preset": "en", "removals": null }, "usingBlockMaxWAND": true }, "multiTenancyConfig": { "autoTenantActivation": false, "autoTenantCreation": false, "enabled": false }, "properties": [ { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "doc_id", "tokenization": "field" }, { "dataType": [ "int" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": false, "name": "version" }, { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "cohort", "tokenization": "field" }, { "dataType": [ "text" ], "indexFilterable": false, "indexRangeFilters": false, "indexSearchable": false, "name": "text", "tokenization": "word" } ], "shardingConfig": { "actualCount": 1, "actualVirtualCount": 128, "desiredCount": 1, "desiredVirtualCount": 128, "function": "murmur3", "key": "_id", "strategy": "hash", "virtualPerPhysical": 128 }, "vectorIndexConfig": { "bq": { "enabled": false }, "cleanupIntervalSeconds": 300, "distance": "cosine", "dynamicEfFactor": 8, "dynamicEfMax": 500, "dynamicEfMin": 100, "ef": 128, "efConstruction": 128, "filterStrategy": "acorn", "flatSearchCutoff": 0, "maxConnections": 16, "multivector": { "aggregation": "maxSim", "enabled": false, "muvera": { "dprojections": 16, "enabled": false, "ksim": 4, "repetitions": 10 } }, "pq": { "bitCompression": false, "centroids": 256, "enabled": false, "encoder": { "distribution": "log-normal", "type": "kmeans" }, "segments": 0, "trainingLimit": 100000 }, "rq": { "bits": 8, "centering": false, "enabled": false, "rescoreLimit": 20, "trainingLimit": 10000 }, "skip": false, "skipDefaultQuantization": false, "sq": { "enabled": false, "rescoreLimit": 20, "trainingLimit": 100000 }, "trackDefaultQuantization": false, "vectorCacheMaxObjects": 1000000000000 }, "vectorIndexType": "hnsw", "vectorizer": "none", "replicationConfig": { "deletionStrategy": "TimeBasedResolution", "factor": 1, "asyncEnabled": false } }, "shards": [ { "name": "3GjWaXTkjoCH", "per_node_status": { "node1": "READY" }, "status": "READY", "vectorQueueSize": 0 } ], "ack": "synchronous indexing, object PUT/POST/DELETE", "transport": "HTTP REST writes + GraphQL search, pooled httpx" }, "count_before": 100260, "idle_duration_s": 10.0, "idle_queries": { "count": 100, "ann_recall": 0.9940000000000001, "ndcg": 0.7931268675492565, "latency_ms": { "min": 3.6550420336425304, "p50": 11.719853966496885, "p95": 13.10447717551142, "p99": 13.497463320381941, "max": 13.799041975289583 } }, "exact_ndcg": 0.7931268675492565, "resources_before": { "usage_usec": 363508498, "user_usec": 355604012, "system_usec": 7904486, "nice_usec": 0, "nr_periods": 8196, "nr_throttled": 486, "throttled_usec": 1571808, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 1420341248, "pids": 11 }, "resources_after": { "usage_usec": 373494498, "user_usec": 365185555, "system_usec": 8308942, "nice_usec": 0, "nr_periods": 8497, "nr_throttled": 486, "throttled_usec": 1571808, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 1302528000, "pids": 11 }, "count_after": 100260, "backend_after": { "version": "1.40.3", "schema": { "class": "Ffb2429e9a028", "invertedIndexConfig": { "bm25": { "b": 0.75, "k1": 1.2 }, "cleanupIntervalSeconds": 60, "stopwords": { "additions": null, "preset": "en", "removals": null }, "usingBlockMaxWAND": true }, "multiTenancyConfig": { "autoTenantActivation": false, "autoTenantCreation": false, "enabled": false }, "properties": [ { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "doc_id", "tokenization": "field" }, { "dataType": [ "int" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": false, "name": "version" }, { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "cohort", "tokenization": "field" }, { "dataType": [ "text" ], "indexFilterable": false, "indexRangeFilters": false, "indexSearchable": false, "name": "text", "tokenization": "word" } ], "shardingConfig": { "actualCount": 1, "actualVirtualCount": 128, "desiredCount": 1, "desiredVirtualCount": 128, "function": "murmur3", "key": "_id", "strategy": "hash", "virtualPerPhysical": 128 }, "vectorIndexConfig": { "bq": { "enabled": false }, "cleanupIntervalSeconds": 300, "distance": "cosine", "dynamicEfFactor": 8, "dynamicEfMax": 500, "dynamicEfMin": 100, "ef": 128, "efConstruction": 128, "filterStrategy": "acorn", "flatSearchCutoff": 0, "maxConnections": 16, "multivector": { "aggregation": "maxSim", "enabled": false, "muvera": { "dprojections": 16, "enabled": false, "ksim": 4, "repetitions": 10 } }, "pq": { "bitCompression": false, "centroids": 256, "enabled": false, "encoder": { "distribution": "log-normal", "type": "kmeans" }, "segments": 0, "trainingLimit": 100000 }, "rq": { "bits": 8, "centering": false, "enabled": false, "rescoreLimit": 20, "trainingLimit": 10000 }, "skip": false, "skipDefaultQuantization": false, "sq": { "enabled": false, "rescoreLimit": 20, "trainingLimit": 100000 }, "trackDefaultQuantization": false, "vectorCacheMaxObjects": 1000000000000 }, "vectorIndexType": "hnsw", "vectorizer": "none", "replicationConfig": { "deletionStrategy": "TimeBasedResolution", "factor": 1, "asyncEnabled": false } }, "shards": [ { "name": "3GjWaXTkjoCH", "per_node_status": { "node1": "READY" }, "status": "READY", "vectorQueueSize": 0 } ], "ack": "synchronous indexing, object PUT/POST/DELETE", "transport": "HTTP REST writes + GraphQL search, pooled httpx" } }, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "eligible": true, "protocol": "mixed-mutations-v2" }, "examples": [ { "event": 0, "operation": "insert", "doc_id": "mutation-000000", "scheduled_s": 0.0, "submitted_s": 0.00019429100211709738, "schedule_delay_ms": 0.19429100211709738, "precondition": true, "status": "success", "ack_ms": 9.056749986484647, "write_started_ms": 1.3671249616891146, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 14.965208945795894, "stable_since_ms": 14.965208945795894, "confirmed_ms": 67.82624998595566, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 22.130750003270805, "stable_since_ms": 22.130750003270805, "confirmed_ms": 82.06383395008743, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 22.130750003270805, "confirmed_ms": 82.06383395008743, "search_minus_ack_ms": 13.074000016786158 }, { "event": 1, "operation": "update", "doc_id": "mutation-000001", "scheduled_s": 0.025, "submitted_s": 0.025483625009655952, "schedule_delay_ms": 0.48362500965595107, "precondition": true, "status": "success", "ack_ms": 4.069332964718342, "write_started_ms": 0.040665967389941216, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 14.129665913060308, "stable_since_ms": 14.129665913060308, "confirmed_ms": 66.61966594401747, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 4.790040897205472, "stable_since_ms": 4.790040897205472, "confirmed_ms": 63.59375000465661, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 14.129665913060308, "confirmed_ms": 66.61966594401747, "search_minus_ack_ms": 0.7207079324871302 }, { "event": 2, "operation": "delete", "doc_id": "mutation-000002", "scheduled_s": 0.05, "submitted_s": 0.050324165960773826, "schedule_delay_ms": 0.32416596077382287, "precondition": true, "status": "success", "ack_ms": 2.7231250423938036, "write_started_ms": 1.1421249946579337, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 13.816250022500753, "stable_since_ms": 13.816250022500753, "confirmed_ms": 79.60991701111197, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 4.955208976753056, "stable_since_ms": 4.955208976753056, "confirmed_ms": 63.50725004449487, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 13.816250022500753, "confirmed_ms": 79.60991701111197, "search_minus_ack_ms": 2.2320839343592525 } ] }, { "run": "fb2_429e9a_029", "trial": 5, "backend": "pgvector", "scenario": "nq-100k-burst", "status": "complete", "eligible": true, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "summary": { "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "burst", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "wall_s": 30.000890415976755, "operations": { "insert": { "count": 130, "success": 130, "success_rate": 1.0, "first_observed_ms": { "min": 16.078541986644268, "p50": 27.57856243988499, "p95": 34.707798273302615, "p99": 36.913888496346765, "max": 37.35120804049075 }, "confirmed_ms": { "min": 70.72158402297646, "p50": 87.06200052984059, "p95": 95.13019393780269, "p99": 98.2311779668089, "max": 98.94195792730898 }, "ack_ms": { "min": 5.037166993133724, "p50": 16.486437525600195, "p95": 23.144818731816486, "p99": 24.997118604369458, "max": 25.624083005823195 }, "search_minus_ack_ms": { "min": 6.942875101231039, "p50": 11.325417028274387, "p95": 12.887445924570784, "p99": 13.619392509572208, "max": 14.12795891519636 }, "regressions": 0, "confirmed_within_100ms": 1.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "update": { "count": 130, "success": 130, "success_rate": 1.0, "first_observed_ms": { "min": 14.364292030222714, "p50": 25.565353978890926, "p95": 31.420997926034033, "p99": 32.69961899495683, "max": 36.031041061505675 }, "confirmed_ms": { "min": 69.5033750962466, "p50": 85.72468743659556, "p95": 92.13067924138159, "p99": 95.3056347509846, "max": 96.70720796566457 }, "ack_ms": { "min": 4.211209015920758, "p50": 14.579458453226835, "p95": 19.893616798799485, "p99": 23.048393747303617, "max": 25.675041018985212 }, "search_minus_ack_ms": { "min": 7.615249953232706, "p50": 11.380229028873146, "p95": 12.96532292617485, "p99": 13.631053474964572, "max": 14.012624975293875 }, "regressions": 0, "confirmed_within_100ms": 1.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "delete": { "count": 130, "success": 130, "success_rate": 1.0, "first_observed_ms": { "min": 10.101834079250693, "p50": 12.986082991119474, "p95": 28.83260215166956, "p99": 29.707195329247043, "max": 31.23500000219792 }, "confirmed_ms": { "min": 62.25699989590794, "p50": 75.61002095462754, "p95": 91.41639804583974, "p99": 95.1644265046343, "max": 97.01095800846815 }, "ack_ms": { "min": 0.569332973100245, "p50": 2.0466454443521798, "p95": 4.545242135645822, "p99": 5.7609252352267495, "max": 7.746750023216009 }, "search_minus_ack_ms": { "min": 2.7515419060364366, "p50": 10.136520431842655, "p95": 25.304360402515158, "p99": 27.01980337500572, "max": 28.432417078875005 }, "regressions": 0, "confirmed_within_100ms": 1.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 } }, "total_operations": 390, "statuses": { "success": 390, "timeout": 0, "error": 0, "invalid_precondition": 0, "client_overload": 0 }, "query": { "count": 300, "success": 300, "ann_recall_at_10": 0.995, "ndcg_at_10": 0.7931268675492565, "latency_ms": { "min": 3.4637500066310167, "p50": 9.950312494765967, "p95": 14.536312769632787, "p99": 18.487835408886895, "max": 19.83683300204575 }, "dispatch_delay_ms": { "min": 0.10658300016075373, "p50": 0.9675414767116308, "p95": 1.6017516900319608, "p99": 1.9420468353200686, "max": 8.766040904447436 } }, "diagnostics": { "schedule_delay_ms": { "min": 0.00241592060845619, "p50": 0.34583295928403146, "p95": 0.6264999974519014, "p99": 0.6668324526393832, "max": 0.7651659892872198 }, "probe_dispatch_ms": { "min": 0.012499978765845299, "p50": 0.3920624731108546, "p95": 1.3145748816896226, "p99": 1.652543968521059, "max": 9.087749989703298 }, "probe_service_ms": { "min": 0.19741698633879423, "p50": 1.9842705223709345, "p95": 10.702795645920553, "p99": 14.872324966127062, "max": 20.03000001423061 }, "write_service_ms": { "min": 0.5040830001235008, "p50": 8.550104452297091, "p95": 20.846114901360124, "p99": 23.875957367708914, "max": 25.411874987185 } }, "actual_submission_rate": 13.010080800632766, "metadata": { "backend": "pgvector", "collection": "fb2_429e9a_029", "dataset": { "dataset": "nq", "source": "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/nq.zip", "source_sha256": "2553bf7bfbab47b1436ca00a34bce57320e18e611fd00999a2a3a1b4714be752", "source_md5": "d4d3d2e48787a744b6f6e691ff534307", "records": 124000, "queries": 100, "relevant_documents_retained": 120, "seed": 20261028, "license": "CC-BY-SA-3.0", "license_source": "https://raw.githubusercontent.com/google-research-datasets/natural-questions/master/nq_open/README.md", "attribution": "BEIR and original dataset authors; Wikipedia contributors for NQ, SciFact authors and cited article authors for SciFact.", "transformation": "200-word title+text prefix; seeded reservoir of documents with relevant documents for selected test queries retained at front. Simulated mutations; no historical replay.", "model": "sentence-transformers/all-MiniLM-L6-v2", "model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "dimensions": 384, "normalized": true, "max_tokens": 256, "device": "mps", "files": { "corpus.jsonl": "9dc487d785e6ee493eccaf764daa3bf8720a8f4fc8c406246065e1ad33a91f5d", "queries.json": "5603f52876ada5d36118b65dea490f1d43900bc159e1804e637796eb9115e999", "vectors.npy": "cb7421eb6ca5964231403f0a922843088f6fd0e01cbf67f2a1076cc7101005db", "query_vectors.npy": "5e666f8d966d8db6b0fd93d4963234d2dffa0ee3910b8fad069a8ee201e8b5d4", "UPSTREAM-LICENSE.md": "8b675521da3fa142ac6bda38c9bcfcc8cff0e2d6128be53a062ed55fb53f7b87" } }, "base_count": 100000, "profile": { "dataset": "nq", "base_count": 100000, "rate": 10, "query_rate": 10, "arrival": "burst" }, "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "burst", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "preloaded_mutable": 260, "started_utc": "2026-10-29T03:14:08Z", "setup_s": 58.61869816691615, "backend_before": { "version": "0.8.7", "postgres": "PostgreSQL 16.15 (Debian 16.15-1.pgdg12+2) on aarch64-unknown-linux-gnu, compiled by gcc (Debian 12.2.0-14+deb12u1) 12.2.0, 64-bit", "ef_search": 128, "ack": "autocommit, READ COMMITTED", "transport": "psycopg, connection per worker", "query_plan": [ { "Plan": { "Node Type": "Limit", "Parallel Aware": false, "Async Capable": false, "Startup Cost": 1674.28, "Total Cost": 1691.41, "Plan Rows": 10, "Plan Width": 18, "Plans": [ { "Node Type": "Index Scan", "Parent Relationship": "Outer", "Parallel Aware": false, "Async Capable": false, "Scan Direction": "Forward", "Index Name": "fb2_429e9a_029_hnsw", "Relation Name": "fb2_429e9a_029", "Alias": "fb2_429e9a_029", "Startup Cost": 1674.28, "Total Cost": 172999.29, "Plan Rows": 100036, "Plan Width": 18, "Order By": "(embedding <=> '[-0.08946895,-0.094362676,0.092023626,-0.0040844884,0.043233823,0.00062519487,0.1075197,-0.035732996,0.013746996,-0.024103122,0.0021706754,-0.04720602,0.04787615,-0.02043222,0.058315646,0.081252135,-0.05375015,-0.030799072,-0.0077205584,0.035735283,-0.04474784,0.01614922,-0.06905586,0.04194401,-0.08832908,0.007873556,0.0044992934,0.021968655,0.010953364,0.07212316,0.023185596,0.001176476,0.04725166,-0.03580423,-0.032344446,0.004752835,-0.056179125,-0.0353235,-0.037251893,-0.005143068,-0.027599342,0.024403147,0.031338032,0.0056410627,0.008425455,-0.0087741045,-0.039870497,-0.011831836,0.042621233,0.048142947,0.01603842,0.06745542,-0.038018145,0.064212486,-0.009636566,0.01607475,-0.0027868534,-0.03761097,-0.011889431,-0.01084577,0.083403684,0.028946918,0.061870236,-0.045382615,0.03812586,0.060426526,-0.017003024,0.02328925,-0.093020774,-0.024072766,0.042720392,-0.022444837,-0.011597358,0.01209231,-0.005953629,0.029691145,-0.011711495,-0.053345133,-0.081757374,-0.049303155,0.032193717,-0.038507868,-0.0060101775,-0.031470332,-0.07362443,0.03575578,0.0245484,0.01985519,-0.15116754,0.050386053,-0.0389634,0.07548389,-0.09264168,-0.012602816,-0.0035539444,-0.046416666,0.015100493,0.100691915,-0.037529,0.04600518,-0.021859422,0.014675989,-0.097722545,-0.053200893,-0.017040461,-0.01629616,-0.012339222,0.06543302,0.07252074,-0.044301156,0.03054835,-0.038570553,0.019310065,-0.005527747,-0.029386865,-0.058248505,0.065443166,0.028506057,0.0878254,0.058387812,0.081721246,-0.0005962058,0.0569388,0.00960514,0.011667701,0.014456492,-0.07218306,-3.0444767e-33,0.06687367,-0.019213954,0.058202963,0.025171444,-0.028259544,-0.04171671,0.03127722,0.04865431,-0.0721459,-0.050143186,-0.029907746,0.06248994,-0.05096076,-0.065192886,0.010210447,-0.077256374,-0.043370593,-0.0901922,-0.066522665,0.026072454,-0.022315364,0.075375885,-0.029968103,-0.0050771856,-0.0046955296,0.0065650316,-0.06791897,0.07368851,0.056069646,-0.021026025,-0.011352883,0.038811427,-0.090847075,-0.056490287,-0.039881036,0.032885592,-0.029586185,0.04753608,0.006341778,-0.0034275919,0.03681623,0.09326265,-0.0532089,0.032396164,0.012605483,0.045778517,-0.0006472902,0.029721813,0.0734947,-0.037117425,-0.09356724,-0.0071634455,-0.032400057,0.010453378,0.06286513,0.006053158,0.060524322,0.091866635,0.063640274,0.055919155,0.053222507,-0.019174587,0.089296214,-0.058922265,0.013745364,-0.022907477,0.028347451,-0.040759467,-0.06820364,-0.061398294,-0.046857864,0.0060627847,0.010442015,-0.07662317,0.031950947,-0.107813306,-0.0016125626,-0.033413693,-0.09989722,-0.03580293,-0.036118146,-0.08276185,0.1280713,0.07596603,0.049225595,-0.026312606,0.0031930343,-0.0012469466,0.000872793,-0.040353082,-0.034516506,-0.036047976,0.004999689,-0.01570079,-0.09783203,1.4186912e-33,-0.0130903,0.04742377,0.02889633,0.09809088,0.043639127,-0.0747531,0.03402747,0.029147558,0.019848268,0.111865185,0.04845458,-0.016222177,0.025501896,0.09813583,-0.002833195,0.0038929228,0.048537854,0.014111116,0.036570966,-0.017512968,-0.079755045,0.043452248,0.10843892,0.061968718,-0.015678233,0.0060376576,0.010718569,-0.03907552,0.040307663,0.04553227,0.041731495,0.083179526,-0.0844024,-0.023447752,0.014485363,-0.00788756,-0.11410207,0.039309476,0.011220049,0.12889338,-0.07970701,0.010658111,-0.06657531,-0.022037188,-0.043459363,-0.030291978,0.05803613,0.06109021,-0.09415837,0.030727232,0.011858702,0.013526454,-0.049580045,-0.052486703,0.013218615,0.0050231596,0.07660412,-0.0046343356,-0.0692584,-0.03353194,0.022288544,0.022836983,0.01361394,-0.036331616,0.008194689,0.09226991,0.08229875,-0.047639843,-0.005033471,0.0066081914,-0.106666416,-0.06385503,0.006385333,0.06318358,0.03146985,-0.060764216,-0.043287244,-0.042385958,-0.07963737,-0.043478705,-0.031840656,0.060027804,0.027829666,0.03728562,-0.07014622,0.0649568,-0.052579004,-0.0034938955,-0.15390792,-0.04426162,0.028217599,0.05593145,0.023251446,0.029115746,-0.040321343,-1.8832651e-08,-0.037536032,0.0005056563,0.013919978,-0.038129315,0.061486673,0.005534046,-0.010768211,-0.16527015,0.061414365,0.019963786,-0.019666849,-0.059065994,0.00083181454,-0.016695915,0.01090875,0.02685501,-0.0008743374,-0.060967337,-0.049258173,0.012731965,-0.03973743,-0.0018210232,-0.03797001,0.00035104176,0.04477184,0.04185603,-0.03215796,-0.09103283,0.0467481,0.017810425,0.037437033,0.054782324,-0.0059442874,0.022557084,0.053393807,-0.03746794,-0.07080435,0.10548834,-0.04215435,0.042547524,0.05064807,0.07560324,0.04072184,-0.0011278372,0.00702018,-0.070807114,0.040035766,0.051417697,0.011316928,-0.028955372,0.04197099,-0.0026867494,-0.056857754,0.0761941,0.030621601,-0.052413527,-0.021492323,0.0957355,0.1013111,-0.082977295,-0.036080915,-0.03327806,0.0029404552,-0.03586191]'::vector)", "Filter": "(cohort = 'static'::text)" } ] } } ] }, "count_before": 100260, "idle_duration_s": 10.0, "idle_queries": { "count": 100, "ann_recall": 0.995, "ndcg": 0.7931268675492565, "latency_ms": { "min": 4.425083985552192, "p50": 18.565895967185497, "p95": 22.726893739309162, "p99": 25.495429821312428, "max": 25.67816700320691 } }, "exact_ndcg": 0.7931268675492565, "resources_before": { "usage_usec": 543590494, "user_usec": 510145904, "system_usec": 33444589, "nice_usec": 0, "nr_periods": 6949, "nr_throttled": 2139, "throttled_usec": 225099275, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 1582985216, "pids": 23 }, "resources_after": { "usage_usec": 554840943, "user_usec": 517975283, "system_usec": 36865659, "nice_usec": 0, "nr_periods": 7250, "nr_throttled": 2139, "throttled_usec": 225099275, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 1605091328, "pids": 32 }, "count_after": 100260, "backend_after": { "version": "0.8.7", "postgres": "PostgreSQL 16.15 (Debian 16.15-1.pgdg12+2) on aarch64-unknown-linux-gnu, compiled by gcc (Debian 12.2.0-14+deb12u1) 12.2.0, 64-bit", "ef_search": 128, "ack": "autocommit, READ COMMITTED", "transport": "psycopg, connection per worker", "query_plan": [ { "Plan": { "Node Type": "Limit", "Parallel Aware": false, "Async Capable": false, "Startup Cost": 1676.0, "Total Cost": 1693.13, "Plan Rows": 10, "Plan Width": 18, "Plans": [ { "Node Type": "Index Scan", "Parent Relationship": "Outer", "Parallel Aware": false, "Async Capable": false, "Scan Direction": "Forward", "Index Name": "fb2_429e9a_029_hnsw", "Relation Name": "fb2_429e9a_029", "Alias": "fb2_429e9a_029", "Startup Cost": 1676.0, "Total Cost": 173383.14, "Plan Rows": 100207, "Plan Width": 18, "Order By": "(embedding <=> '[-0.08946895,-0.094362676,0.092023626,-0.0040844884,0.043233823,0.00062519487,0.1075197,-0.035732996,0.013746996,-0.024103122,0.0021706754,-0.04720602,0.04787615,-0.02043222,0.058315646,0.081252135,-0.05375015,-0.030799072,-0.0077205584,0.035735283,-0.04474784,0.01614922,-0.06905586,0.04194401,-0.08832908,0.007873556,0.0044992934,0.021968655,0.010953364,0.07212316,0.023185596,0.001176476,0.04725166,-0.03580423,-0.032344446,0.004752835,-0.056179125,-0.0353235,-0.037251893,-0.005143068,-0.027599342,0.024403147,0.031338032,0.0056410627,0.008425455,-0.0087741045,-0.039870497,-0.011831836,0.042621233,0.048142947,0.01603842,0.06745542,-0.038018145,0.064212486,-0.009636566,0.01607475,-0.0027868534,-0.03761097,-0.011889431,-0.01084577,0.083403684,0.028946918,0.061870236,-0.045382615,0.03812586,0.060426526,-0.017003024,0.02328925,-0.093020774,-0.024072766,0.042720392,-0.022444837,-0.011597358,0.01209231,-0.005953629,0.029691145,-0.011711495,-0.053345133,-0.081757374,-0.049303155,0.032193717,-0.038507868,-0.0060101775,-0.031470332,-0.07362443,0.03575578,0.0245484,0.01985519,-0.15116754,0.050386053,-0.0389634,0.07548389,-0.09264168,-0.012602816,-0.0035539444,-0.046416666,0.015100493,0.100691915,-0.037529,0.04600518,-0.021859422,0.014675989,-0.097722545,-0.053200893,-0.017040461,-0.01629616,-0.012339222,0.06543302,0.07252074,-0.044301156,0.03054835,-0.038570553,0.019310065,-0.005527747,-0.029386865,-0.058248505,0.065443166,0.028506057,0.0878254,0.058387812,0.081721246,-0.0005962058,0.0569388,0.00960514,0.011667701,0.014456492,-0.07218306,-3.0444767e-33,0.06687367,-0.019213954,0.058202963,0.025171444,-0.028259544,-0.04171671,0.03127722,0.04865431,-0.0721459,-0.050143186,-0.029907746,0.06248994,-0.05096076,-0.065192886,0.010210447,-0.077256374,-0.043370593,-0.0901922,-0.066522665,0.026072454,-0.022315364,0.075375885,-0.029968103,-0.0050771856,-0.0046955296,0.0065650316,-0.06791897,0.07368851,0.056069646,-0.021026025,-0.011352883,0.038811427,-0.090847075,-0.056490287,-0.039881036,0.032885592,-0.029586185,0.04753608,0.006341778,-0.0034275919,0.03681623,0.09326265,-0.0532089,0.032396164,0.012605483,0.045778517,-0.0006472902,0.029721813,0.0734947,-0.037117425,-0.09356724,-0.0071634455,-0.032400057,0.010453378,0.06286513,0.006053158,0.060524322,0.091866635,0.063640274,0.055919155,0.053222507,-0.019174587,0.089296214,-0.058922265,0.013745364,-0.022907477,0.028347451,-0.040759467,-0.06820364,-0.061398294,-0.046857864,0.0060627847,0.010442015,-0.07662317,0.031950947,-0.107813306,-0.0016125626,-0.033413693,-0.09989722,-0.03580293,-0.036118146,-0.08276185,0.1280713,0.07596603,0.049225595,-0.026312606,0.0031930343,-0.0012469466,0.000872793,-0.040353082,-0.034516506,-0.036047976,0.004999689,-0.01570079,-0.09783203,1.4186912e-33,-0.0130903,0.04742377,0.02889633,0.09809088,0.043639127,-0.0747531,0.03402747,0.029147558,0.019848268,0.111865185,0.04845458,-0.016222177,0.025501896,0.09813583,-0.002833195,0.0038929228,0.048537854,0.014111116,0.036570966,-0.017512968,-0.079755045,0.043452248,0.10843892,0.061968718,-0.015678233,0.0060376576,0.010718569,-0.03907552,0.040307663,0.04553227,0.041731495,0.083179526,-0.0844024,-0.023447752,0.014485363,-0.00788756,-0.11410207,0.039309476,0.011220049,0.12889338,-0.07970701,0.010658111,-0.06657531,-0.022037188,-0.043459363,-0.030291978,0.05803613,0.06109021,-0.09415837,0.030727232,0.011858702,0.013526454,-0.049580045,-0.052486703,0.013218615,0.0050231596,0.07660412,-0.0046343356,-0.0692584,-0.03353194,0.022288544,0.022836983,0.01361394,-0.036331616,0.008194689,0.09226991,0.08229875,-0.047639843,-0.005033471,0.0066081914,-0.106666416,-0.06385503,0.006385333,0.06318358,0.03146985,-0.060764216,-0.043287244,-0.042385958,-0.07963737,-0.043478705,-0.031840656,0.060027804,0.027829666,0.03728562,-0.07014622,0.0649568,-0.052579004,-0.0034938955,-0.15390792,-0.04426162,0.028217599,0.05593145,0.023251446,0.029115746,-0.040321343,-1.8832651e-08,-0.037536032,0.0005056563,0.013919978,-0.038129315,0.061486673,0.005534046,-0.010768211,-0.16527015,0.061414365,0.019963786,-0.019666849,-0.059065994,0.00083181454,-0.016695915,0.01090875,0.02685501,-0.0008743374,-0.060967337,-0.049258173,0.012731965,-0.03973743,-0.0018210232,-0.03797001,0.00035104176,0.04477184,0.04185603,-0.03215796,-0.09103283,0.0467481,0.017810425,0.037437033,0.054782324,-0.0059442874,0.022557084,0.053393807,-0.03746794,-0.07080435,0.10548834,-0.04215435,0.042547524,0.05064807,0.07560324,0.04072184,-0.0011278372,0.00702018,-0.070807114,0.040035766,0.051417697,0.011316928,-0.028955372,0.04197099,-0.0026867494,-0.056857754,0.0761941,0.030621601,-0.052413527,-0.021492323,0.0957355,0.1013111,-0.082977295,-0.036080915,-0.03327806,0.0029404552,-0.03586191]'::vector)", "Filter": "(cohort = 'static'::text)" } ] } } ] } }, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "eligible": true, "protocol": "mixed-mutations-v2" }, "examples": [ { "event": 0, "operation": "insert", "doc_id": "mutation-000000", "scheduled_s": 0.0, "submitted_s": 0.00021487497724592686, "schedule_delay_ms": 0.21487497724592686, "precondition": true, "status": "success", "ack_ms": 22.157541010528803, "write_started_ms": 0.8720000041648746, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 24.72783299162984, "stable_since_ms": 24.72783299162984, "confirmed_ms": 76.8542910227552, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 33.11483294237405, "stable_since_ms": 33.11483294237405, "confirmed_ms": 88.92020804341882, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 33.11483294237405, "confirmed_ms": 88.92020804341882, "search_minus_ack_ms": 10.957291931845248 }, { "event": 1, "operation": "update", "doc_id": "mutation-000001", "scheduled_s": 0.025, "submitted_s": 0.0251459579449147, "schedule_delay_ms": 0.1459579449146972, "precondition": true, "status": "success", "ack_ms": 5.29341702349484, "write_started_ms": 0.04558300133794546, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 11.290458030998707, "stable_since_ms": 11.290458030998707, "confirmed_ms": 62.60037503670901, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 16.50425000116229, "stable_since_ms": 16.50425000116229, "confirmed_ms": 71.81966700591147, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 16.50425000116229, "confirmed_ms": 71.81966700591147, "search_minus_ack_ms": 11.210832977667451 }, { "event": 2, "operation": "delete", "doc_id": "mutation-000002", "scheduled_s": 0.05, "submitted_s": 0.0501791249262169, "schedule_delay_ms": 0.17912492621689757, "precondition": true, "status": "success", "ack_ms": 0.918666017241776, "write_started_ms": 0.11437502689659595, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 11.475916020572186, "stable_since_ms": 11.475916020572186, "confirmed_ms": 62.694958061911166, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 13.578666024841368, "stable_since_ms": 13.578666024841368, "confirmed_ms": 69.92729101330042, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 13.578666024841368, "confirmed_ms": 69.92729101330042, "search_minus_ack_ms": 12.660000007599592 } ] }, { "run": "fb2_429e9a_030", "trial": 5, "backend": "weaviate", "scenario": "scifact-small", "status": "complete", "eligible": true, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "summary": { "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "wall_s": 30.00080145802349, "operations": { "insert": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 5.931166000664234, "p50": 17.845542053692043, "p95": 29.90001904545352, "p99": 30.897240793565313, "max": 35.437958082184196 }, "confirmed_ms": { "min": 65.8972499659285, "p50": 79.69310402404517, "p95": 96.51153514860198, "p99": 101.86956083984114, "max": 101.87558305915445 }, "ack_ms": { "min": 2.8949170373380184, "p50": 8.959666942246258, "p95": 12.799145834287629, "p99": 15.673943876754503, "max": 17.813417012803257 }, "search_minus_ack_ms": { "min": 0.09124993812292814, "p50": 2.8532500728033483, "p95": 17.706789885414764, "p99": 19.15246505057441, "max": 22.61400001589209 }, "regressions": 0, "confirmed_within_100ms": 0.98, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "update": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 6.832708953879774, "p50": 18.03552103228867, "p95": 28.83286604192108, "p99": 30.47789008356631, "max": 31.34979191236198 }, "confirmed_ms": { "min": 67.40595900919288, "p50": 79.62164602940902, "p95": 98.48387567908503, "p99": 103.21213573450223, "max": 103.54733292479068 }, "ack_ms": { "min": 4.747707978822291, "p50": 8.871499507222325, "p95": 12.56927708745934, "p99": 13.592261241283314, "max": 22.569417022168636 }, "search_minus_ack_ms": { "min": 0.11004204861819744, "p50": 3.275270981248468, "p95": 16.648777143564075, "p99": 17.865415671840317, "max": 19.568874966353178 }, "regressions": 0, "confirmed_within_100ms": 0.97, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "delete": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 7.0470409700647, "p50": 16.586333513259888, "p95": 25.018210418056693, "p99": 29.359056230168807, "max": 32.74299995973706 }, "confirmed_ms": { "min": 68.73458402696997, "p50": 77.67068751854822, "p95": 90.27820624178275, "p99": 98.09260826674293, "max": 102.3974580457434 }, "ack_ms": { "min": 2.225290983915329, "p50": 5.172999983187765, "p95": 10.9779640333727, "p99": 12.948825289495296, "max": 14.408207964152098 }, "search_minus_ack_ms": { "min": -0.10787509381771088, "p50": 4.040750034619123, "p95": 16.71175410738215, "p99": 19.09972148714589, "max": 22.58250000886619 }, "regressions": 0, "confirmed_within_100ms": 0.99, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 } }, "total_operations": 300, "statuses": { "success": 300, "timeout": 0, "error": 0, "invalid_precondition": 0, "client_overload": 0 }, "query": { "count": 300, "success": 300, "ann_recall_at_10": 1.0, "ndcg_at_10": 0.6567157762068453, "latency_ms": { "min": 3.825250081717968, "p50": 9.338020463474095, "p95": 14.374172559473664, "p99": 20.064224174711846, "max": 22.90541701950133 }, "dispatch_delay_ms": { "min": 0.13887498062103987, "p50": 1.340354501735419, "p95": 2.360824984498322, "p99": 2.791180496569721, "max": 3.217250108718872 } }, "diagnostics": { "schedule_delay_ms": { "min": 0.0026670051731514377, "p50": 0.32931251917123205, "p95": 0.6195478700101342, "p99": 0.6430441129487008, "max": 0.828792061657424 }, "probe_dispatch_ms": { "min": 0.026124995201826096, "p50": 0.4283749731257558, "p95": 1.9386000814847648, "p99": 2.7251165895722798, "max": 10.301167028956115 }, "probe_service_ms": { "min": 0.8027500007301569, "p50": 5.33470802474767, "p95": 10.260737512726337, "p99": 13.605568460188795, "max": 22.887959028594196 }, "write_service_ms": { "min": 1.9776660483330488, "p50": 7.740479544736445, "p95": 11.548183095874265, "p99": 13.51225747494026, "max": 22.142083034850657 } }, "actual_submission_rate": 10.000070569083272, "metadata": { "backend": "weaviate", "collection": "fb2_429e9a_030", "dataset": { "dataset": "scifact", "source": "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/scifact.zip", "source_sha256": "536e14446a0ba56ed1398ab1055f39fe852686ecad24a6306c80c490fa8e0165", "source_md5": "5f7d1de60b170fc8027bb7898e2efca1", "records": 5183, "queries": 100, "relevant_documents_retained": 107, "seed": 20261028, "license": "Corpus: ODC-By-1.0; claims/evidence: CC-BY-4.0", "license_source": "https://raw.githubusercontent.com/allenai/scifact/master/LICENSE.md", "attribution": "BEIR and original dataset authors; Wikipedia contributors for NQ, SciFact authors and cited article authors for SciFact.", "transformation": "200-word title+text prefix; seeded reservoir of documents with relevant documents for selected test queries retained at front. Simulated mutations; no historical replay.", "model": "sentence-transformers/all-MiniLM-L6-v2", "model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "dimensions": 384, "normalized": true, "max_tokens": 256, "device": "mps", "files": { "corpus.jsonl": "189d58f273468352c9f73c6333c5120dcfb7ba1695984f073f0ebb6c100ed0cd", "queries.json": "c8b333c73353a8c4aeb7d6b63067c9a241aa5d39a0314f0c315ec63019ee9558", "vectors.npy": "b6b21404977151ce565e7d6d57d9931a4679e1ecf4aa44fcd9b29eb54955d09e", "query_vectors.npy": "02324eb7c3b3d2e2f0ec3208453e2ec0efd5d61d9fc845b1f8b7950e913f2730", "UPSTREAM-LICENSE.md": "56ee69bc639cbab8f0b2e4b67091ec5e9390aa99a1d65f0a35e4274686c9a806" } }, "base_count": 4000, "profile": { "dataset": "scifact", "base_count": 4000, "rate": 10, "query_rate": 10, "arrival": "steady" }, "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "preloaded_mutable": 200, "started_utc": "2026-10-29T03:15:52Z", "setup_s": 1.1762347919866443, "backend_before": { "version": "1.40.3", "schema": { "class": "Ffb2429e9a030", "invertedIndexConfig": { "bm25": { "b": 0.75, "k1": 1.2 }, "cleanupIntervalSeconds": 60, "stopwords": { "additions": null, "preset": "en", "removals": null }, "usingBlockMaxWAND": true }, "multiTenancyConfig": { "autoTenantActivation": false, "autoTenantCreation": false, "enabled": false }, "properties": [ { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "doc_id", "tokenization": "field" }, { "dataType": [ "int" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": false, "name": "version" }, { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "cohort", "tokenization": "field" }, { "dataType": [ "text" ], "indexFilterable": false, "indexRangeFilters": false, "indexSearchable": false, "name": "text", "tokenization": "word" } ], "shardingConfig": { "actualCount": 1, "actualVirtualCount": 128, "desiredCount": 1, "desiredVirtualCount": 128, "function": "murmur3", "key": "_id", "strategy": "hash", "virtualPerPhysical": 128 }, "vectorIndexConfig": { "bq": { "enabled": false }, "cleanupIntervalSeconds": 300, "distance": "cosine", "dynamicEfFactor": 8, "dynamicEfMax": 500, "dynamicEfMin": 100, "ef": 128, "efConstruction": 128, "filterStrategy": "acorn", "flatSearchCutoff": 0, "maxConnections": 16, "multivector": { "aggregation": "maxSim", "enabled": false, "muvera": { "dprojections": 16, "enabled": false, "ksim": 4, "repetitions": 10 } }, "pq": { "bitCompression": false, "centroids": 256, "enabled": false, "encoder": { "distribution": "log-normal", "type": "kmeans" }, "segments": 0, "trainingLimit": 100000 }, "rq": { "bits": 8, "centering": false, "enabled": false, "rescoreLimit": 20, "trainingLimit": 10000 }, "skip": false, "skipDefaultQuantization": false, "sq": { "enabled": false, "rescoreLimit": 20, "trainingLimit": 100000 }, "trackDefaultQuantization": false, "vectorCacheMaxObjects": 1000000000000 }, "vectorIndexType": "hnsw", "vectorizer": "none", "replicationConfig": { "deletionStrategy": "TimeBasedResolution", "factor": 1, "asyncEnabled": false } }, "shards": [ { "name": "Cv8GlgnufS8X", "per_node_status": { "node1": "READY" }, "status": "READY", "vectorQueueSize": 0 } ], "ack": "synchronous indexing, object PUT/POST/DELETE", "transport": "HTTP REST writes + GraphQL search, pooled httpx" }, "count_before": 4200, "idle_duration_s": 10.0, "idle_queries": { "count": 100, "ann_recall": 1.0, "ndcg": 0.6567157762068453, "latency_ms": { "min": 4.088083049282432, "p50": 10.250250052195042, "p95": 12.069102132227272, "p99": 13.985161656746646, "max": 15.13966592028737 } }, "exact_ndcg": 0.6567157762068453, "resources_before": { "usage_usec": 376466883, "user_usec": 367999474, "system_usec": 8467409, "nice_usec": 0, "nr_periods": 8833, "nr_throttled": 487, "throttled_usec": 1573043, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 541270016, "pids": 11 }, "resources_after": { "usage_usec": 384850183, "user_usec": 375971256, "system_usec": 8878926, "nice_usec": 0, "nr_periods": 9134, "nr_throttled": 487, "throttled_usec": 1573043, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 549494784, "pids": 11 }, "count_after": 4200, "backend_after": { "version": "1.40.3", "schema": { "class": "Ffb2429e9a030", "invertedIndexConfig": { "bm25": { "b": 0.75, "k1": 1.2 }, "cleanupIntervalSeconds": 60, "stopwords": { "additions": null, "preset": "en", "removals": null }, "usingBlockMaxWAND": true }, "multiTenancyConfig": { "autoTenantActivation": false, "autoTenantCreation": false, "enabled": false }, "properties": [ { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "doc_id", "tokenization": "field" }, { "dataType": [ "int" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": false, "name": "version" }, { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "cohort", "tokenization": "field" }, { "dataType": [ "text" ], "indexFilterable": false, "indexRangeFilters": false, "indexSearchable": false, "name": "text", "tokenization": "word" } ], "shardingConfig": { "actualCount": 1, "actualVirtualCount": 128, "desiredCount": 1, "desiredVirtualCount": 128, "function": "murmur3", "key": "_id", "strategy": "hash", "virtualPerPhysical": 128 }, "vectorIndexConfig": { "bq": { "enabled": false }, "cleanupIntervalSeconds": 300, "distance": "cosine", "dynamicEfFactor": 8, "dynamicEfMax": 500, "dynamicEfMin": 100, "ef": 128, "efConstruction": 128, "filterStrategy": "acorn", "flatSearchCutoff": 0, "maxConnections": 16, "multivector": { "aggregation": "maxSim", "enabled": false, "muvera": { "dprojections": 16, "enabled": false, "ksim": 4, "repetitions": 10 } }, "pq": { "bitCompression": false, "centroids": 256, "enabled": false, "encoder": { "distribution": "log-normal", "type": "kmeans" }, "segments": 0, "trainingLimit": 100000 }, "rq": { "bits": 8, "centering": false, "enabled": false, "rescoreLimit": 20, "trainingLimit": 10000 }, "skip": false, "skipDefaultQuantization": false, "sq": { "enabled": false, "rescoreLimit": 20, "trainingLimit": 100000 }, "trackDefaultQuantization": false, "vectorCacheMaxObjects": 1000000000000 }, "vectorIndexType": "hnsw", "vectorizer": "none", "replicationConfig": { "deletionStrategy": "TimeBasedResolution", "factor": 1, "asyncEnabled": false } }, "shards": [ { "name": "Cv8GlgnufS8X", "per_node_status": { "node1": "READY" }, "status": "READY", "vectorQueueSize": 0 } ], "ack": "synchronous indexing, object PUT/POST/DELETE", "transport": "HTTP REST writes + GraphQL search, pooled httpx" } }, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "eligible": true, "protocol": "mixed-mutations-v2" }, "examples": [ { "event": 0, "operation": "insert", "doc_id": "mutation-000000", "scheduled_s": 0.0, "submitted_s": 0.0002418330404907465, "schedule_delay_ms": 0.2418330404907465, "precondition": true, "status": "success", "ack_ms": 5.37579192314297, "write_started_ms": 1.5312499599531293, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 14.541958924382925, "stable_since_ms": 14.541958924382925, "confirmed_ms": 66.9616669183597, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 6.046083988621831, "stable_since_ms": 6.046083988621831, "confirmed_ms": 64.33483399450779, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 14.541958924382925, "confirmed_ms": 66.9616669183597, "search_minus_ack_ms": 0.6702920654788613 }, { "event": 1, "operation": "update", "doc_id": "mutation-000001", "scheduled_s": 0.1, "submitted_s": 0.10014941706322134, "schedule_delay_ms": 0.14941706322132986, "precondition": true, "status": "success", "ack_ms": 7.999249966815114, "write_started_ms": 0.49124995712190866, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 18.55487492866814, "stable_since_ms": 18.55487492866814, "confirmed_ms": 73.0021659983322, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 10.62149996869266, "stable_since_ms": 10.62149996869266, "confirmed_ms": 73.1257499428466, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 18.55487492866814, "confirmed_ms": 73.1257499428466, "search_minus_ack_ms": 2.6222500018775463 }, { "event": 2, "operation": "delete", "doc_id": "mutation-000002", "scheduled_s": 0.2, "submitted_s": 0.2004512919811532, "schedule_delay_ms": 0.45129198115317903, "precondition": true, "status": "success", "ack_ms": 6.756958086043596, "write_started_ms": 0.6535410648211837, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 7.156040985137224, "stable_since_ms": 7.156040985137224, "confirmed_ms": 63.301000045612454, "consecutive": 3, "regressions": 0, "requests": 3 }, "search": { "first_ms": 8.38966597802937, "stable_since_ms": 8.38966597802937, "confirmed_ms": 73.21587507613003, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 8.38966597802937, "confirmed_ms": 73.21587507613003, "search_minus_ack_ms": 1.632707891985774 } ] }, { "run": "fb2_429e9a_031", "trial": 3, "backend": "pgvector", "scenario": "scifact-small", "status": "complete", "eligible": true, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "summary": { "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "wall_s": 30.000517625012435, "operations": { "insert": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 14.648874988779426, "p50": 20.112811995204538, "p95": 22.381539863999933, "p99": 22.91196372709239, "max": 27.086957939900458 }, "confirmed_ms": { "min": 69.71308297943324, "p50": 80.29368799179792, "p95": 83.67925615166314, "p99": 84.797215963481, "max": 85.75504203327 }, "ack_ms": { "min": 3.7768339971080422, "p50": 7.668750011362135, "p95": 9.477733628591523, "p99": 10.468740086071234, "max": 18.40012497268617 }, "search_minus_ack_ms": { "min": 8.686832967214286, "p50": 12.09835399640724, "p95": 13.997262815246359, "p99": 14.31245216983371, "max": 15.235832892358303 }, "regressions": 0, "confirmed_within_100ms": 1.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "update": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 14.479665900580585, "p50": 19.885542045813054, "p95": 22.239129821537063, "p99": 23.298136172816164, "max": 24.557334021665156 }, "confirmed_ms": { "min": 69.410250056535, "p50": 79.99197900062427, "p95": 84.7152583883144, "p99": 88.49064899375665, "max": 93.02633407060057 }, "ack_ms": { "min": 3.58775001950562, "p50": 7.687895966228098, "p95": 9.13980181212537, "p99": 10.423756183590744, "max": 10.993625037372112 }, "search_minus_ack_ms": { "min": 4.206957994028926, "p50": 12.006125471089035, "p95": 14.216338895494118, "p99": 15.207935028010981, "max": 15.560375060886145 }, "regressions": 0, "confirmed_within_100ms": 1.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "delete": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 11.342250043526292, "p50": 13.985583558678627, "p95": 21.36173989274539, "p99": 21.574384012492374, "max": 21.764957928098738 }, "confirmed_ms": { "min": 62.887500040233135, "p50": 67.79797899071127, "p95": 81.88061274122447, "p99": 82.1400981710758, "max": 83.3212920697406 }, "ack_ms": { "min": 0.9965000208467245, "p50": 2.7459164848551154, "p95": 3.908672876423224, "p99": 4.489616507198664, "max": 4.9259590450674295 }, "search_minus_ack_ms": { "min": 1.0110830189660192, "p50": 3.219333477318287, "p95": 17.70867111044936, "p99": 18.7845372245647, "max": 18.82129197474569 }, "regressions": 0, "confirmed_within_100ms": 1.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 } }, "total_operations": 300, "statuses": { "success": 300, "timeout": 0, "error": 0, "invalid_precondition": 0, "client_overload": 0 }, "query": { "count": 300, "success": 300, "ann_recall_at_10": 0.9993333333333334, "ndcg_at_10": 0.6567157762068453, "latency_ms": { "min": 1.4346250100061297, "p50": 4.423145961482078, "p95": 5.788843223126605, "p99": 6.483514941064636, "max": 14.434249955229461 }, "dispatch_delay_ms": { "min": 0.07174990605562925, "p50": 1.0081250220537186, "p95": 1.700345810968429, "p99": 1.951882488792762, "max": 2.0777499303221703 } }, "diagnostics": { "schedule_delay_ms": { "min": 0.007041939532825836, "p50": 0.32204147428283036, "p95": 0.6013874639756001, "p99": 0.6310461016837543, "max": 0.6422499194744802 }, "probe_dispatch_ms": { "min": 0.028957962058484554, "p50": 0.4195414949208498, "p95": 1.444922381779179, "p99": 1.9002243084833028, "max": 2.6136250235140324 }, "probe_service_ms": { "min": 0.38120895624160767, "p50": 2.161167038138956, "p95": 5.511824914719909, "p99": 6.271505022887143, "max": 15.414000023156404 }, "write_service_ms": { "min": 0.8786669932305813, "p50": 6.772187480237335, "p95": 8.752620540326461, "p99": 10.003203242085874, "max": 17.274584039114416 } }, "actual_submission_rate": 9.999973801746576, "metadata": { "backend": "pgvector", "collection": "fb2_429e9a_031", "dataset": { "dataset": "scifact", "source": "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/scifact.zip", "source_sha256": "536e14446a0ba56ed1398ab1055f39fe852686ecad24a6306c80c490fa8e0165", "source_md5": "5f7d1de60b170fc8027bb7898e2efca1", "records": 5183, "queries": 100, "relevant_documents_retained": 107, "seed": 20261028, "license": "Corpus: ODC-By-1.0; claims/evidence: CC-BY-4.0", "license_source": "https://raw.githubusercontent.com/allenai/scifact/master/LICENSE.md", "attribution": "BEIR and original dataset authors; Wikipedia contributors for NQ, SciFact authors and cited article authors for SciFact.", "transformation": "200-word title+text prefix; seeded reservoir of documents with relevant documents for selected test queries retained at front. Simulated mutations; no historical replay.", "model": "sentence-transformers/all-MiniLM-L6-v2", "model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "dimensions": 384, "normalized": true, "max_tokens": 256, "device": "mps", "files": { "corpus.jsonl": "189d58f273468352c9f73c6333c5120dcfb7ba1695984f073f0ebb6c100ed0cd", "queries.json": "c8b333c73353a8c4aeb7d6b63067c9a241aa5d39a0314f0c315ec63019ee9558", "vectors.npy": "b6b21404977151ce565e7d6d57d9931a4679e1ecf4aa44fcd9b29eb54955d09e", "query_vectors.npy": "02324eb7c3b3d2e2f0ec3208453e2ec0efd5d61d9fc845b1f8b7950e913f2730", "UPSTREAM-LICENSE.md": "56ee69bc639cbab8f0b2e4b67091ec5e9390aa99a1d65f0a35e4274686c9a806" } }, "base_count": 4000, "profile": { "dataset": "scifact", "base_count": 4000, "rate": 10, "query_rate": 10, "arrival": "steady" }, "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "preloaded_mutable": 200, "started_utc": "2026-10-29T03:16:37Z", "setup_s": 1.6783158750040457, "backend_before": { "version": "0.8.7", "postgres": "PostgreSQL 16.15 (Debian 16.15-1.pgdg12+2) on aarch64-unknown-linux-gnu, compiled by gcc (Debian 12.2.0-14+deb12u1) 12.2.0, 64-bit", "ef_search": 128, "ack": "autocommit, READ COMMITTED", "transport": "psycopg, connection per worker", "query_plan": [ { "Plan": { "Node Type": "Limit", "Parallel Aware": false, "Async Capable": false, "Startup Cost": 1216.56, "Total Cost": 1229.86, "Plan Rows": 10, "Plan Width": 16, "Plans": [ { "Node Type": "Index Scan", "Parent Relationship": "Outer", "Parallel Aware": false, "Async Capable": false, "Scan Direction": "Forward", "Index Name": "fb2_429e9a_031_hnsw", "Relation Name": "fb2_429e9a_031", "Alias": "fb2_429e9a_031", "Startup Cost": 1216.56, "Total Cost": 6538.0, "Plan Rows": 4000, "Plan Width": 16, "Order By": "(embedding <=> '[-0.025241137,-0.0037172192,-0.046351995,0.046476826,0.01966622,0.015340689,0.01234306,0.07297605,-0.018329926,-0.013840419,-0.07423897,0.0226357,-0.060784698,-0.028650705,0.05173048,-0.03129233,0.07885052,-0.024390804,0.034739245,0.002474597,0.06695868,-0.03807076,0.016879125,-0.007840027,0.026065528,0.014293832,-0.07275669,0.053708147,-0.020156316,0.07862689,-0.122224644,0.023270043,-0.03987285,-0.029884664,0.008407885,0.06157951,-0.045617852,-0.0013952592,-0.1272149,0.012878671,0.03199118,-0.054489113,-0.07601716,0.016042225,-0.02170239,0.0031456149,-0.03146653,0.036349196,0.030850349,0.101148084,-0.04483407,-0.02210456,0.015648471,0.008023243,0.02329231,-0.047311362,-0.00033133395,-0.060376305,-0.031292018,-0.052068412,-0.055494312,0.03995384,-0.05426452,-0.027163653,0.060765423,-0.029587762,0.062952094,-0.032716926,0.03284741,0.03383774,0.025922673,0.09215419,-0.045252554,0.033018086,-0.0063568754,0.003192685,0.036198895,0.071328856,0.061275735,-0.03834579,0.04380842,0.10884386,-0.06443366,0.027753524,0.016000912,0.031735342,-0.048004173,0.04857545,-0.058097336,0.043568186,-0.014455384,0.008579895,0.023373613,0.0023358837,0.070720226,-0.008269884,-0.011370413,-0.016605737,-0.002399004,-0.03909848,0.017561054,0.042421922,0.011341604,0.025210828,0.09824794,0.0038353435,0.015525245,0.082866095,0.0066154706,0.027564062,0.051245574,-0.009089501,0.13282458,0.04902157,-0.013380719,0.042171925,0.033162206,0.048019677,0.029081784,0.116638325,0.00094674755,0.07029037,0.004596228,-0.02894104,-0.0037263758,0.0041222423,-0.04681563,2.8784053e-33,0.09736768,0.02274426,-0.05078067,0.03864741,0.052899208,0.0496459,-0.060387712,-0.08358956,0.07020227,0.042317823,-0.037229892,-0.1324623,0.08240096,0.039485864,-0.09648351,-0.04383094,-0.06106865,0.025340972,0.031598456,0.04274742,-0.0010978275,0.0007333723,0.041061625,0.011878232,-0.12491255,0.06554766,-0.06920273,0.05781889,-0.05118846,0.043497268,-0.029946037,-0.08991292,0.095012516,-0.12172844,-0.048556324,-0.022020115,-0.014565633,-0.045546,-0.05527385,0.06144455,-0.040042106,0.011806558,-0.13014492,-0.024195487,0.008652814,-0.05887302,0.074748434,-0.03390347,-0.026171094,-0.046370335,-0.02871423,0.01613992,0.050530907,-0.072944105,0.0142246755,-0.0029443551,-0.028645331,0.024415584,-0.021900453,-0.035877362,-0.012359783,0.0469807,0.062022846,0.09110963,-0.011605537,-0.02338493,0.0334725,-0.03523439,-0.039391812,0.020923749,0.045133665,-0.0050133714,0.05868334,0.05490907,0.002556749,0.04097332,0.008711031,-0.06200574,0.03028164,-0.11109333,0.0780934,0.012912109,0.07097299,-0.04068633,0.008197527,0.030907633,0.05238045,0.0011536852,-0.01918761,-0.15324447,0.016286323,-0.05679703,-0.04369561,-0.06417541,0.02090238,-3.0734922e-33,-0.005520113,0.007917555,0.051423088,-0.08971186,0.009558501,0.04202714,-0.032392696,0.007947915,-0.033026196,0.057292204,0.049505785,-0.017359108,-0.087639086,-0.047633022,0.012320442,0.036888227,-0.032912884,-0.05555789,-0.030620035,0.040593192,0.056201898,-0.0435655,0.031078862,0.0809475,0.0595156,-0.008109408,-0.10603491,0.0726334,-0.06864153,-0.034877256,0.023989158,-0.04435082,-0.019972347,-0.12077245,0.044683173,0.014618874,-0.020796062,0.114251524,0.01766482,0.0160371,0.012701173,-0.0053563626,0.074527174,-0.021048209,-0.022659509,0.08406541,-0.06423724,-0.071950026,0.05369397,0.01140933,-0.088458955,-0.01212312,0.0024443534,0.12023613,-0.027182862,-0.022696352,0.020270811,0.08858118,-0.10382351,0.086645365,-0.06383489,-0.04042209,0.04494627,0.016389402,0.13325493,0.08580276,0.060353823,0.03661967,0.041859947,0.021768665,0.0073611615,-0.009300088,-0.033082195,-0.040072612,0.06993921,0.02415639,0.012099922,0.034952287,-0.10813522,-0.024580343,0.05526578,-0.043421194,-0.00020226264,0.013380121,-0.013031641,-0.0023380888,0.020561703,-0.026072076,-0.058541298,-0.027999775,-0.024194289,-0.07036205,-0.05425848,-0.07206627,-0.026656583,-2.3484924e-08,-0.037731905,0.015433777,-0.059194244,-0.062783524,-0.047979638,-0.024383545,-0.1138203,0.040749338,-0.038158197,0.008040415,0.024223538,-0.018507883,0.108268395,0.025551548,0.068713784,-0.043465756,0.017785417,0.07973369,-0.0075122416,-0.04605511,-0.047828607,-0.017772881,0.05118608,0.0022625374,0.03168905,0.0143064195,0.017061999,-0.05608364,0.0064040497,0.0072717075,-0.013301113,0.018809363,-0.017954899,0.08202268,-0.06270375,0.029983893,0.0048537604,-0.035181895,-0.06037397,-0.0063187,0.0034529127,0.011674293,0.013009845,0.012574133,0.031151973,-0.087101504,0.030415354,0.036086276,-0.06438181,-0.08931628,0.0122331055,-0.0099123195,0.008941321,-0.024656717,-0.025116868,-0.0044285995,-0.065338664,0.0398255,0.039999187,0.011154338,0.010968719,-0.048901215,0.096052974,-0.08210191]'::vector)", "Filter": "(cohort = 'static'::text)" } ] } } ] }, "count_before": 4200, "idle_duration_s": 10.0, "idle_queries": { "count": 100, "ann_recall": 1.0, "ndcg": 0.6567157762068453, "latency_ms": { "min": 2.71612498909235, "p50": 6.188395956996828, "p95": 7.168371503939852, "p99": 8.92947370652116, "max": 21.69787499587983 } }, "exact_ndcg": 0.6567157762068453, "resources_before": { "usage_usec": 556846341, "user_usec": 519726522, "system_usec": 37119819, "nice_usec": 0, "nr_periods": 7400, "nr_throttled": 2139, "throttled_usec": 225099275, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 1150337024, "pids": 23 }, "resources_after": { "usage_usec": 563313986, "user_usec": 525706244, "system_usec": 37607741, "nice_usec": 0, "nr_periods": 7701, "nr_throttled": 2139, "throttled_usec": 225099275, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 1159426048, "pids": 27 }, "count_after": 4200, "backend_after": { "version": "0.8.7", "postgres": "PostgreSQL 16.15 (Debian 16.15-1.pgdg12+2) on aarch64-unknown-linux-gnu, compiled by gcc (Debian 12.2.0-14+deb12u1) 12.2.0, 64-bit", "ef_search": 128, "ack": "autocommit, READ COMMITTED", "transport": "psycopg, connection per worker", "query_plan": [ { "Plan": { "Node Type": "Limit", "Parallel Aware": false, "Async Capable": false, "Startup Cost": 1251.23, "Total Cost": 1264.83, "Plan Rows": 10, "Plan Width": 16, "Plans": [ { "Node Type": "Index Scan", "Parent Relationship": "Outer", "Parallel Aware": false, "Async Capable": false, "Scan Direction": "Forward", "Index Name": "fb2_429e9a_031_hnsw", "Relation Name": "fb2_429e9a_031", "Alias": "fb2_429e9a_031", "Startup Cost": 1251.23, "Total Cost": 6698.18, "Plan Rows": 4007, "Plan Width": 16, "Order By": "(embedding <=> '[-0.025241137,-0.0037172192,-0.046351995,0.046476826,0.01966622,0.015340689,0.01234306,0.07297605,-0.018329926,-0.013840419,-0.07423897,0.0226357,-0.060784698,-0.028650705,0.05173048,-0.03129233,0.07885052,-0.024390804,0.034739245,0.002474597,0.06695868,-0.03807076,0.016879125,-0.007840027,0.026065528,0.014293832,-0.07275669,0.053708147,-0.020156316,0.07862689,-0.122224644,0.023270043,-0.03987285,-0.029884664,0.008407885,0.06157951,-0.045617852,-0.0013952592,-0.1272149,0.012878671,0.03199118,-0.054489113,-0.07601716,0.016042225,-0.02170239,0.0031456149,-0.03146653,0.036349196,0.030850349,0.101148084,-0.04483407,-0.02210456,0.015648471,0.008023243,0.02329231,-0.047311362,-0.00033133395,-0.060376305,-0.031292018,-0.052068412,-0.055494312,0.03995384,-0.05426452,-0.027163653,0.060765423,-0.029587762,0.062952094,-0.032716926,0.03284741,0.03383774,0.025922673,0.09215419,-0.045252554,0.033018086,-0.0063568754,0.003192685,0.036198895,0.071328856,0.061275735,-0.03834579,0.04380842,0.10884386,-0.06443366,0.027753524,0.016000912,0.031735342,-0.048004173,0.04857545,-0.058097336,0.043568186,-0.014455384,0.008579895,0.023373613,0.0023358837,0.070720226,-0.008269884,-0.011370413,-0.016605737,-0.002399004,-0.03909848,0.017561054,0.042421922,0.011341604,0.025210828,0.09824794,0.0038353435,0.015525245,0.082866095,0.0066154706,0.027564062,0.051245574,-0.009089501,0.13282458,0.04902157,-0.013380719,0.042171925,0.033162206,0.048019677,0.029081784,0.116638325,0.00094674755,0.07029037,0.004596228,-0.02894104,-0.0037263758,0.0041222423,-0.04681563,2.8784053e-33,0.09736768,0.02274426,-0.05078067,0.03864741,0.052899208,0.0496459,-0.060387712,-0.08358956,0.07020227,0.042317823,-0.037229892,-0.1324623,0.08240096,0.039485864,-0.09648351,-0.04383094,-0.06106865,0.025340972,0.031598456,0.04274742,-0.0010978275,0.0007333723,0.041061625,0.011878232,-0.12491255,0.06554766,-0.06920273,0.05781889,-0.05118846,0.043497268,-0.029946037,-0.08991292,0.095012516,-0.12172844,-0.048556324,-0.022020115,-0.014565633,-0.045546,-0.05527385,0.06144455,-0.040042106,0.011806558,-0.13014492,-0.024195487,0.008652814,-0.05887302,0.074748434,-0.03390347,-0.026171094,-0.046370335,-0.02871423,0.01613992,0.050530907,-0.072944105,0.0142246755,-0.0029443551,-0.028645331,0.024415584,-0.021900453,-0.035877362,-0.012359783,0.0469807,0.062022846,0.09110963,-0.011605537,-0.02338493,0.0334725,-0.03523439,-0.039391812,0.020923749,0.045133665,-0.0050133714,0.05868334,0.05490907,0.002556749,0.04097332,0.008711031,-0.06200574,0.03028164,-0.11109333,0.0780934,0.012912109,0.07097299,-0.04068633,0.008197527,0.030907633,0.05238045,0.0011536852,-0.01918761,-0.15324447,0.016286323,-0.05679703,-0.04369561,-0.06417541,0.02090238,-3.0734922e-33,-0.005520113,0.007917555,0.051423088,-0.08971186,0.009558501,0.04202714,-0.032392696,0.007947915,-0.033026196,0.057292204,0.049505785,-0.017359108,-0.087639086,-0.047633022,0.012320442,0.036888227,-0.032912884,-0.05555789,-0.030620035,0.040593192,0.056201898,-0.0435655,0.031078862,0.0809475,0.0595156,-0.008109408,-0.10603491,0.0726334,-0.06864153,-0.034877256,0.023989158,-0.04435082,-0.019972347,-0.12077245,0.044683173,0.014618874,-0.020796062,0.114251524,0.01766482,0.0160371,0.012701173,-0.0053563626,0.074527174,-0.021048209,-0.022659509,0.08406541,-0.06423724,-0.071950026,0.05369397,0.01140933,-0.088458955,-0.01212312,0.0024443534,0.12023613,-0.027182862,-0.022696352,0.020270811,0.08858118,-0.10382351,0.086645365,-0.06383489,-0.04042209,0.04494627,0.016389402,0.13325493,0.08580276,0.060353823,0.03661967,0.041859947,0.021768665,0.0073611615,-0.009300088,-0.033082195,-0.040072612,0.06993921,0.02415639,0.012099922,0.034952287,-0.10813522,-0.024580343,0.05526578,-0.043421194,-0.00020226264,0.013380121,-0.013031641,-0.0023380888,0.020561703,-0.026072076,-0.058541298,-0.027999775,-0.024194289,-0.07036205,-0.05425848,-0.07206627,-0.026656583,-2.3484924e-08,-0.037731905,0.015433777,-0.059194244,-0.062783524,-0.047979638,-0.024383545,-0.1138203,0.040749338,-0.038158197,0.008040415,0.024223538,-0.018507883,0.108268395,0.025551548,0.068713784,-0.043465756,0.017785417,0.07973369,-0.0075122416,-0.04605511,-0.047828607,-0.017772881,0.05118608,0.0022625374,0.03168905,0.0143064195,0.017061999,-0.05608364,0.0064040497,0.0072717075,-0.013301113,0.018809363,-0.017954899,0.08202268,-0.06270375,0.029983893,0.0048537604,-0.035181895,-0.06037397,-0.0063187,0.0034529127,0.011674293,0.013009845,0.012574133,0.031151973,-0.087101504,0.030415354,0.036086276,-0.06438181,-0.08931628,0.0122331055,-0.0099123195,0.008941321,-0.024656717,-0.025116868,-0.0044285995,-0.065338664,0.0398255,0.039999187,0.011154338,0.010968719,-0.048901215,0.096052974,-0.08210191]'::vector)", "Filter": "(cohort = 'static'::text)" } ] } } ] } }, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "eligible": true, "protocol": "mixed-mutations-v2" }, "examples": [ { "event": 0, "operation": "insert", "doc_id": "mutation-000000", "scheduled_s": 0.0, "submitted_s": 0.00027541699819266796, "schedule_delay_ms": 0.27541699819266796, "precondition": true, "status": "success", "ack_ms": 18.40012497268617, "write_started_ms": 1.1255409335717559, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 24.826582986861467, "stable_since_ms": 24.826582986861467, "confirmed_ms": 76.54145802371204, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 27.086957939900458, "stable_since_ms": 27.086957939900458, "confirmed_ms": 81.9161250256002, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 27.086957939900458, "confirmed_ms": 81.9161250256002, "search_minus_ack_ms": 8.686832967214286 }, { "event": 1, "operation": "update", "doc_id": "mutation-000001", "scheduled_s": 0.1, "submitted_s": 0.10056016698945314, "schedule_delay_ms": 0.5601669894531314, "precondition": true, "status": "success", "ack_ms": 6.336833001114428, "write_started_ms": 0.17091596964746714, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 13.272707932628691, "stable_since_ms": 13.272707932628691, "confirmed_ms": 67.08495796192437, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 16.844040947034955, "stable_since_ms": 16.844040947034955, "confirmed_ms": 75.5058330250904, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 16.844040947034955, "confirmed_ms": 75.5058330250904, "search_minus_ack_ms": 10.507207945920527 }, { "event": 2, "operation": "delete", "doc_id": "mutation-000002", "scheduled_s": 0.2, "submitted_s": 0.2005853329319507, "schedule_delay_ms": 0.5853329319506773, "precondition": true, "status": "success", "ack_ms": 4.9259590450674295, "write_started_ms": 0.3665840486064553, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 14.76766704581678, "stable_since_ms": 14.76766704581678, "confirmed_ms": 68.99875006638467, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 21.32533409167081, "stable_since_ms": 21.32533409167081, "confirmed_ms": 83.3212920697406, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 21.32533409167081, "confirmed_ms": 83.3212920697406, "search_minus_ack_ms": 16.39937504660338 } ] }, { "run": "fb2_429e9a_032", "trial": 4, "backend": "qdrant", "scenario": "scifact-small", "status": "complete", "eligible": true, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "summary": { "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "wall_s": 30.05226829193998, "operations": { "insert": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 16.835291986353695, "p50": 64.61335445055738, "p95": 74.74584174924529, "p99": 104.46358548128046, "max": 104.8969590337947 }, "confirmed_ms": { "min": 107.29420802090317, "p50": 207.70752051612362, "p95": 213.808337575756, "p99": 216.1546304298099, "max": 216.26654104329646 }, "ack_ms": { "min": 4.117625067010522, "p50": 8.878374996129423, "p95": 11.994058627169577, "p99": 15.888602832565127, "max": 24.11162492353469 }, "search_minus_ack_ms": { "min": 0.15945895574986935, "p50": 55.464583507273346, "p95": 59.4770356779918, "p99": 97.66930256504565, "max": 98.96566707175225 }, "regressions": 0, "confirmed_within_100ms": 0.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "update": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 17.673667054623365, "p50": 64.2373125301674, "p95": 75.39194647106342, "p99": 104.38800967880526, "max": 108.20462508127093 }, "confirmed_ms": { "min": 107.43866593111306, "p50": 207.92960451217368, "p95": 213.29157467698678, "p99": 242.4663354514633, "max": 252.51091609243304 }, "ack_ms": { "min": 3.2928750151768327, "p50": 8.365271030925214, "p95": 17.048435384640467, "p99": 23.510078737745083, "max": 27.824374963529408 }, "search_minus_ack_ms": { "min": 0.1460830681025982, "p50": 55.443958553951234, "p95": 58.749341720249504, "p99": 97.14581516338514, "max": 99.90783408284187 }, "regressions": 0, "confirmed_within_100ms": 0.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "delete": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 6.631667027249932, "p50": 18.67612497881055, "p95": 62.764420540770516, "p99": 63.361004603793845, "max": 63.65845899563283 }, "confirmed_ms": { "min": 105.98641692195088, "p50": 111.94227094529197, "p95": 205.9482375276275, "p99": 206.7523385572713, "max": 209.2567919753492 }, "ack_ms": { "min": 2.925584092736244, "p50": 7.169666467234492, "p95": 9.045702056027949, "p99": 9.776887247571752, "max": 15.59850003104657 }, "search_minus_ack_ms": { "min": -1.3899579644203186, "p50": 0.34679198870435357, "p95": 55.050133337499574, "p99": 55.84439366590232, "max": 56.34945794008672 }, "regressions": 0, "confirmed_within_100ms": 0.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 } }, "total_operations": 300, "statuses": { "success": 300, "timeout": 0, "error": 0, "invalid_precondition": 0, "client_overload": 0 }, "query": { "count": 300, "success": 300, "ann_recall_at_10": 1.0, "ndcg_at_10": 0.6567157762068453, "latency_ms": { "min": 2.386542037129402, "p50": 7.237979443743825, "p95": 45.82351447897963, "p99": 47.89282967103645, "max": 49.07966707833111 }, "dispatch_delay_ms": { "min": 0.06225006654858589, "p50": 1.2689789291471243, "p95": 2.177096181549132, "p99": 2.785314028151332, "max": 24.15445796214044 } }, "diagnostics": { "schedule_delay_ms": { "min": 0.009999983012320968, "p50": 0.3582709818141794, "p95": 0.632760686567746, "p99": 0.6606575064368059, "max": 0.6799999624487185 }, "probe_dispatch_ms": { "min": 0.0227498821914196, "p50": 0.3962080227211118, "p95": 1.570591959170997, "p99": 2.3800117056816807, "max": 23.16262503154576 }, "probe_service_ms": { "min": 0.9719589725136757, "p50": 5.699957953765988, "p95": 47.1915831323713, "p99": 48.63243067637086, "max": 57.3407500050962 }, "write_service_ms": { "min": 2.6752090780064464, "p50": 7.635478978045285, "p95": 10.944920522160832, "p99": 20.306939551373915, "max": 27.506291982717812 } }, "actual_submission_rate": 9.999939897381012, "metadata": { "backend": "qdrant", "collection": "fb2_429e9a_032", "dataset": { "dataset": "scifact", "source": "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/scifact.zip", "source_sha256": "536e14446a0ba56ed1398ab1055f39fe852686ecad24a6306c80c490fa8e0165", "source_md5": "5f7d1de60b170fc8027bb7898e2efca1", "records": 5183, "queries": 100, "relevant_documents_retained": 107, "seed": 20261028, "license": "Corpus: ODC-By-1.0; claims/evidence: CC-BY-4.0", "license_source": "https://raw.githubusercontent.com/allenai/scifact/master/LICENSE.md", "attribution": "BEIR and original dataset authors; Wikipedia contributors for NQ, SciFact authors and cited article authors for SciFact.", "transformation": "200-word title+text prefix; seeded reservoir of documents with relevant documents for selected test queries retained at front. Simulated mutations; no historical replay.", "model": "sentence-transformers/all-MiniLM-L6-v2", "model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "dimensions": 384, "normalized": true, "max_tokens": 256, "device": "mps", "files": { "corpus.jsonl": "189d58f273468352c9f73c6333c5120dcfb7ba1695984f073f0ebb6c100ed0cd", "queries.json": "c8b333c73353a8c4aeb7d6b63067c9a241aa5d39a0314f0c315ec63019ee9558", "vectors.npy": "b6b21404977151ce565e7d6d57d9931a4679e1ecf4aa44fcd9b29eb54955d09e", "query_vectors.npy": "02324eb7c3b3d2e2f0ec3208453e2ec0efd5d61d9fc845b1f8b7950e913f2730", "UPSTREAM-LICENSE.md": "56ee69bc639cbab8f0b2e4b67091ec5e9390aa99a1d65f0a35e4274686c9a806" } }, "base_count": 4000, "profile": { "dataset": "scifact", "base_count": 4000, "rate": 10, "query_rate": 10, "arrival": "steady" }, "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "preloaded_mutable": 200, "started_utc": "2026-10-29T03:17:23Z", "setup_s": 1.6991398331010714, "backend_before": { "version": "1.20.0", "collection": { "status": "green", "optimizer_status": "ok", "indexed_vectors_count": 4200, "points_count": 4200, "segments_count": 2, "config": { "params": { "vectors": { "size": 384, "distance": "Cosine" }, "shard_number": 1, "replication_factor": 1, "write_consistency_factor": 1, "on_disk_payload": true }, "hnsw_config": { "m": 16, "ef_construct": 128, "full_scan_threshold": 10, "max_indexing_threads": 0, "on_disk": false }, "optimizer_config": { "deleted_threshold": 0.2, "vacuum_min_vector_number": 1000, "default_segment_number": 0, "max_segment_size": null, "memmap_threshold": null, "indexing_threshold": 1024, "flush_interval_sec": 5, "max_optimization_threads": null, "prevent_unoptimized": null }, "wal_config": { "wal_capacity_mb": 32, "wal_segments_ahead": 0, "wal_retain_closed": 1 }, "quantization_config": null }, "payload_schema": { "cohort": { "data_type": "keyword", "points": 4200 } }, "update_queue": { "length": 0 }, "created_at": "2026-10-29T03:17:23.587Z" }, "ack": "wait=true", "transport": "HTTP REST, pooled httpx" }, "count_before": 4200, "idle_duration_s": 10.0, "idle_queries": { "count": 100, "ann_recall": 1.0, "ndcg": 0.6567157762068453, "latency_ms": { "min": 2.696541021578014, "p50": 6.5410000388510525, "p95": 8.261277014389634, "p99": 8.855622351402609, "max": 45.864832936786115 } }, "exact_ndcg": 0.6567157762068453, "resources_before": { "usage_usec": 255705803, "user_usec": 240788644, "system_usec": 14917159, "nice_usec": 175016000, "nr_periods": 5788, "nr_throttled": 382, "throttled_usec": 2082403, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 367525888, "pids": 32 }, "resources_after": { "usage_usec": 260597731, "user_usec": 244897266, "system_usec": 15700465, "nice_usec": 175016000, "nr_periods": 6090, "nr_throttled": 382, "throttled_usec": 2082403, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 372088832, "pids": 32 }, "count_after": 4200, "backend_after": { "version": "1.20.0", "collection": { "status": "green", "optimizer_status": "ok", "indexed_vectors_count": 4200, "points_count": 4200, "segments_count": 2, "config": { "params": { "vectors": { "size": 384, "distance": "Cosine" }, "shard_number": 1, "replication_factor": 1, "write_consistency_factor": 1, "on_disk_payload": true }, "hnsw_config": { "m": 16, "ef_construct": 128, "full_scan_threshold": 10, "max_indexing_threads": 0, "on_disk": false }, "optimizer_config": { "deleted_threshold": 0.2, "vacuum_min_vector_number": 1000, "default_segment_number": 0, "max_segment_size": null, "memmap_threshold": null, "indexing_threshold": 1024, "flush_interval_sec": 5, "max_optimization_threads": null, "prevent_unoptimized": null }, "wal_config": { "wal_capacity_mb": 32, "wal_segments_ahead": 0, "wal_retain_closed": 1 }, "quantization_config": null }, "payload_schema": { "cohort": { "data_type": "keyword", "points": 4200 } }, "update_queue": { "length": 0 }, "created_at": "2026-10-29T03:17:23.587Z" }, "ack": "wait=true", "transport": "HTTP REST, pooled httpx" } }, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "eligible": true, "protocol": "mixed-mutations-v2" }, "examples": [ { "event": 0, "operation": "insert", "doc_id": "mutation-000000", "scheduled_s": 0.0, "submitted_s": 0.0002465000143274665, "schedule_delay_ms": 0.2465000143274665, "precondition": true, "status": "success", "ack_ms": 7.728541968390346, "write_started_ms": 1.6661249101161957, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 14.979666913859546, "stable_since_ms": 14.979666913859546, "confirmed_ms": 69.95854200795293, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 60.461041983217, "stable_since_ms": 60.461041983217, "confirmed_ms": 203.36641697213054, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 60.461041983217, "confirmed_ms": 203.36641697213054, "search_minus_ack_ms": 52.732500014826655 }, { "event": 1, "operation": "update", "doc_id": "mutation-000001", "scheduled_s": 0.1, "submitted_s": 0.10006516694556922, "schedule_delay_ms": 0.06516694556921165, "precondition": true, "status": "success", "ack_ms": 10.3150000795722, "write_started_ms": 1.2257080525159836, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 20.22925007622689, "stable_since_ms": 20.22925007622689, "confirmed_ms": 77.32733304146677, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 63.67858301382512, "stable_since_ms": 63.67858301382512, "confirmed_ms": 206.64041605778039, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 63.67858301382512, "confirmed_ms": 206.64041605778039, "search_minus_ack_ms": 53.36358293425292 }, { "event": 2, "operation": "delete", "doc_id": "mutation-000002", "scheduled_s": 0.2, "submitted_s": 0.20004354196134955, "schedule_delay_ms": 0.04354196134953581, "precondition": true, "status": "success", "ack_ms": 5.2552910055965185, "write_started_ms": 0.6079999729990959, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 16.421250067651272, "stable_since_ms": 16.421250067651272, "confirmed_ms": 72.20675004646182, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 59.89250005222857, "stable_since_ms": 59.89250005222857, "confirmed_ms": 202.78304105158895, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 59.89250005222857, "confirmed_ms": 202.78304105158895, "search_minus_ack_ms": 54.63720904663205 } ] }, { "run": "fb2_429e9a_033", "trial": 2, "backend": "pgvector", "scenario": "nq-100k", "status": "complete", "eligible": true, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "summary": { "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "wall_s": 30.001325707999058, "operations": { "insert": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 18.773416988551617, "p50": 30.060520977713168, "p95": 36.606266745366156, "p99": 39.399485364556845, "max": 143.02212500479072 }, "confirmed_ms": { "min": 75.22504101507366, "p50": 90.56533349212259, "p95": 101.72839760780334, "p99": 104.28414660156756, "max": 201.98274997528642 }, "ack_ms": { "min": 6.606166018173099, "p50": 18.403583497274667, "p95": 24.7402485110797, "p99": 27.445232514292535, "max": 130.37874992005527 }, "search_minus_ack_ms": { "min": 8.300457964651287, "p50": 11.515979014802724, "p95": 13.252947595901787, "p99": 13.543934365734476, "max": 16.93233393598348 }, "regressions": 0, "confirmed_within_100ms": 0.94, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "update": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 16.5986669017002, "p50": 27.30077103478834, "p95": 33.11547092744149, "p99": 35.38954660762105, "max": 42.567500029690564 }, "confirmed_ms": { "min": 71.40374998562038, "p50": 87.69027050584555, "p95": 95.30571732320823, "p99": 99.18817523634063, "max": 102.50558308325708 }, "ack_ms": { "min": 5.618666997179389, "p50": 15.458958456292748, "p95": 21.493130619637668, "p99": 22.608896660385692, "max": 30.25879105553031 }, "search_minus_ack_ms": { "min": 7.939999923110008, "p50": 11.858125042635947, "p95": 13.511422969168052, "p99": 13.910155470948668, "max": 14.135917066596448 }, "regressions": 0, "confirmed_within_100ms": 0.99, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "delete": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 11.70208293478936, "p50": 13.40541651006788, "p95": 31.48492951877415, "p99": 33.64468383719214, "max": 34.929290995933115 }, "confirmed_ms": { "min": 64.12195903249085, "p50": 75.40166651597247, "p95": 97.07282468443736, "p99": 101.95431317435579, "max": 103.40437502600253 }, "ack_ms": { "min": 0.963832950219512, "p50": 2.2356874542310834, "p95": 4.368581523885949, "p99": 6.154609655495734, "max": 6.400583079084754 }, "search_minus_ack_ms": { "min": 3.8589579053223133, "p50": 10.395041550509632, "p95": 27.291579131269824, "p99": 29.895459115505222, "max": 30.46066698152572 }, "regressions": 0, "confirmed_within_100ms": 0.98, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 } }, "total_operations": 300, "statuses": { "success": 300, "timeout": 0, "error": 0, "invalid_precondition": 0, "client_overload": 0 }, "query": { "count": 300, "success": 300, "ann_recall_at_10": 0.9929999999999999, "ndcg_at_10": 0.7931268675492565, "latency_ms": { "min": 3.946457989513874, "p50": 10.108666552696377, "p95": 14.819730957970023, "p99": 17.178164609940715, "max": 18.981790984980762 }, "dispatch_delay_ms": { "min": 0.09162502828985453, "p50": 0.9929375373758376, "p95": 1.7482333583757284, "p99": 2.1407109405845386, "max": 2.45274999178946 } }, "diagnostics": { "schedule_delay_ms": { "min": 0.012792064808309078, "p50": 0.30764600960608846, "p95": 0.5933583807196108, "p99": 0.6347101591527682, "max": 0.6382080493487763 }, "probe_dispatch_ms": { "min": 0.018666032701730728, "p50": 0.4172499757260084, "p95": 1.4156166289467365, "p99": 2.0714886568021025, "max": 2.6471660239621997 }, "probe_service_ms": { "min": 0.23729202803224325, "p50": 2.042790991254151, "p95": 11.895693809492503, "p99": 16.206544460728765, "max": 24.700333015061915 }, "write_service_ms": { "min": 0.82566705532372, "p50": 13.777854037471116, "p95": 22.52185352263041, "p99": 24.80073143611661, "max": 129.50899999123067 } }, "actual_submission_rate": 9.999952620309825, "metadata": { "backend": "pgvector", "collection": "fb2_429e9a_033", "dataset": { "dataset": "nq", "source": "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/nq.zip", "source_sha256": "2553bf7bfbab47b1436ca00a34bce57320e18e611fd00999a2a3a1b4714be752", "source_md5": "d4d3d2e48787a744b6f6e691ff534307", "records": 124000, "queries": 100, "relevant_documents_retained": 120, "seed": 20261028, "license": "CC-BY-SA-3.0", "license_source": "https://raw.githubusercontent.com/google-research-datasets/natural-questions/master/nq_open/README.md", "attribution": "BEIR and original dataset authors; Wikipedia contributors for NQ, SciFact authors and cited article authors for SciFact.", "transformation": "200-word title+text prefix; seeded reservoir of documents with relevant documents for selected test queries retained at front. Simulated mutations; no historical replay.", "model": "sentence-transformers/all-MiniLM-L6-v2", "model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "dimensions": 384, "normalized": true, "max_tokens": 256, "device": "mps", "files": { "corpus.jsonl": "9dc487d785e6ee493eccaf764daa3bf8720a8f4fc8c406246065e1ad33a91f5d", "queries.json": "5603f52876ada5d36118b65dea490f1d43900bc159e1804e637796eb9115e999", "vectors.npy": "cb7421eb6ca5964231403f0a922843088f6fd0e01cbf67f2a1076cc7101005db", "query_vectors.npy": "5e666f8d966d8db6b0fd93d4963234d2dffa0ee3910b8fad069a8ee201e8b5d4", "UPSTREAM-LICENSE.md": "8b675521da3fa142ac6bda38c9bcfcc8cff0e2d6128be53a062ed55fb53f7b87" } }, "base_count": 100000, "profile": { "dataset": "nq", "base_count": 100000, "rate": 10, "query_rate": 10, "arrival": "steady" }, "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "preloaded_mutable": 200, "started_utc": "2026-10-29T03:18:11Z", "setup_s": 63.00448658398818, "backend_before": { "version": "0.8.7", "postgres": "PostgreSQL 16.15 (Debian 16.15-1.pgdg12+2) on aarch64-unknown-linux-gnu, compiled by gcc (Debian 12.2.0-14+deb12u1) 12.2.0, 64-bit", "ef_search": 128, "ack": "autocommit, READ COMMITTED", "transport": "psycopg, connection per worker", "query_plan": [ { "Plan": { "Node Type": "Limit", "Parallel Aware": false, "Async Capable": false, "Startup Cost": 1674.2, "Total Cost": 1691.32, "Plan Rows": 10, "Plan Width": 18, "Plans": [ { "Node Type": "Index Scan", "Parent Relationship": "Outer", "Parallel Aware": false, "Async Capable": false, "Scan Direction": "Forward", "Index Name": "fb2_429e9a_033_hnsw", "Relation Name": "fb2_429e9a_033", "Alias": "fb2_429e9a_033", "Startup Cost": 1674.2, "Total Cost": 172866.01, "Plan Rows": 100003, "Plan Width": 18, "Order By": "(embedding <=> '[-0.08946895,-0.094362676,0.092023626,-0.0040844884,0.043233823,0.00062519487,0.1075197,-0.035732996,0.013746996,-0.024103122,0.0021706754,-0.04720602,0.04787615,-0.02043222,0.058315646,0.081252135,-0.05375015,-0.030799072,-0.0077205584,0.035735283,-0.04474784,0.01614922,-0.06905586,0.04194401,-0.08832908,0.007873556,0.0044992934,0.021968655,0.010953364,0.07212316,0.023185596,0.001176476,0.04725166,-0.03580423,-0.032344446,0.004752835,-0.056179125,-0.0353235,-0.037251893,-0.005143068,-0.027599342,0.024403147,0.031338032,0.0056410627,0.008425455,-0.0087741045,-0.039870497,-0.011831836,0.042621233,0.048142947,0.01603842,0.06745542,-0.038018145,0.064212486,-0.009636566,0.01607475,-0.0027868534,-0.03761097,-0.011889431,-0.01084577,0.083403684,0.028946918,0.061870236,-0.045382615,0.03812586,0.060426526,-0.017003024,0.02328925,-0.093020774,-0.024072766,0.042720392,-0.022444837,-0.011597358,0.01209231,-0.005953629,0.029691145,-0.011711495,-0.053345133,-0.081757374,-0.049303155,0.032193717,-0.038507868,-0.0060101775,-0.031470332,-0.07362443,0.03575578,0.0245484,0.01985519,-0.15116754,0.050386053,-0.0389634,0.07548389,-0.09264168,-0.012602816,-0.0035539444,-0.046416666,0.015100493,0.100691915,-0.037529,0.04600518,-0.021859422,0.014675989,-0.097722545,-0.053200893,-0.017040461,-0.01629616,-0.012339222,0.06543302,0.07252074,-0.044301156,0.03054835,-0.038570553,0.019310065,-0.005527747,-0.029386865,-0.058248505,0.065443166,0.028506057,0.0878254,0.058387812,0.081721246,-0.0005962058,0.0569388,0.00960514,0.011667701,0.014456492,-0.07218306,-3.0444767e-33,0.06687367,-0.019213954,0.058202963,0.025171444,-0.028259544,-0.04171671,0.03127722,0.04865431,-0.0721459,-0.050143186,-0.029907746,0.06248994,-0.05096076,-0.065192886,0.010210447,-0.077256374,-0.043370593,-0.0901922,-0.066522665,0.026072454,-0.022315364,0.075375885,-0.029968103,-0.0050771856,-0.0046955296,0.0065650316,-0.06791897,0.07368851,0.056069646,-0.021026025,-0.011352883,0.038811427,-0.090847075,-0.056490287,-0.039881036,0.032885592,-0.029586185,0.04753608,0.006341778,-0.0034275919,0.03681623,0.09326265,-0.0532089,0.032396164,0.012605483,0.045778517,-0.0006472902,0.029721813,0.0734947,-0.037117425,-0.09356724,-0.0071634455,-0.032400057,0.010453378,0.06286513,0.006053158,0.060524322,0.091866635,0.063640274,0.055919155,0.053222507,-0.019174587,0.089296214,-0.058922265,0.013745364,-0.022907477,0.028347451,-0.040759467,-0.06820364,-0.061398294,-0.046857864,0.0060627847,0.010442015,-0.07662317,0.031950947,-0.107813306,-0.0016125626,-0.033413693,-0.09989722,-0.03580293,-0.036118146,-0.08276185,0.1280713,0.07596603,0.049225595,-0.026312606,0.0031930343,-0.0012469466,0.000872793,-0.040353082,-0.034516506,-0.036047976,0.004999689,-0.01570079,-0.09783203,1.4186912e-33,-0.0130903,0.04742377,0.02889633,0.09809088,0.043639127,-0.0747531,0.03402747,0.029147558,0.019848268,0.111865185,0.04845458,-0.016222177,0.025501896,0.09813583,-0.002833195,0.0038929228,0.048537854,0.014111116,0.036570966,-0.017512968,-0.079755045,0.043452248,0.10843892,0.061968718,-0.015678233,0.0060376576,0.010718569,-0.03907552,0.040307663,0.04553227,0.041731495,0.083179526,-0.0844024,-0.023447752,0.014485363,-0.00788756,-0.11410207,0.039309476,0.011220049,0.12889338,-0.07970701,0.010658111,-0.06657531,-0.022037188,-0.043459363,-0.030291978,0.05803613,0.06109021,-0.09415837,0.030727232,0.011858702,0.013526454,-0.049580045,-0.052486703,0.013218615,0.0050231596,0.07660412,-0.0046343356,-0.0692584,-0.03353194,0.022288544,0.022836983,0.01361394,-0.036331616,0.008194689,0.09226991,0.08229875,-0.047639843,-0.005033471,0.0066081914,-0.106666416,-0.06385503,0.006385333,0.06318358,0.03146985,-0.060764216,-0.043287244,-0.042385958,-0.07963737,-0.043478705,-0.031840656,0.060027804,0.027829666,0.03728562,-0.07014622,0.0649568,-0.052579004,-0.0034938955,-0.15390792,-0.04426162,0.028217599,0.05593145,0.023251446,0.029115746,-0.040321343,-1.8832651e-08,-0.037536032,0.0005056563,0.013919978,-0.038129315,0.061486673,0.005534046,-0.010768211,-0.16527015,0.061414365,0.019963786,-0.019666849,-0.059065994,0.00083181454,-0.016695915,0.01090875,0.02685501,-0.0008743374,-0.060967337,-0.049258173,0.012731965,-0.03973743,-0.0018210232,-0.03797001,0.00035104176,0.04477184,0.04185603,-0.03215796,-0.09103283,0.0467481,0.017810425,0.037437033,0.054782324,-0.0059442874,0.022557084,0.053393807,-0.03746794,-0.07080435,0.10548834,-0.04215435,0.042547524,0.05064807,0.07560324,0.04072184,-0.0011278372,0.00702018,-0.070807114,0.040035766,0.051417697,0.011316928,-0.028955372,0.04197099,-0.0026867494,-0.056857754,0.0761941,0.030621601,-0.052413527,-0.021492323,0.0957355,0.1013111,-0.082977295,-0.036080915,-0.03327806,0.0029404552,-0.03586191]'::vector)", "Filter": "(cohort = 'static'::text)" } ] } } ] }, "count_before": 100200, "idle_duration_s": 10.0, "idle_queries": { "count": 100, "ann_recall": 0.993, "ndcg": 0.7931268675492565, "latency_ms": { "min": 6.443625083193183, "p50": 18.57816649135202, "p95": 23.247466614702716, "p99": 25.562786796363074, "max": 26.288334047421813 } }, "exact_ndcg": 0.7931268675492565, "resources_before": { "usage_usec": 663674030, "user_usec": 622350450, "system_usec": 41323580, "nice_usec": 0, "nr_periods": 8479, "nr_throttled": 2601, "throttled_usec": 274109827, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 1583620096, "pids": 23 }, "resources_after": { "usage_usec": 674218921, "user_usec": 629446124, "system_usec": 44772797, "nice_usec": 0, "nr_periods": 8780, "nr_throttled": 2601, "throttled_usec": 274109827, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 1608654848, "pids": 33 }, "count_after": 100200, "backend_after": { "version": "0.8.7", "postgres": "PostgreSQL 16.15 (Debian 16.15-1.pgdg12+2) on aarch64-unknown-linux-gnu, compiled by gcc (Debian 12.2.0-14+deb12u1) 12.2.0, 64-bit", "ef_search": 128, "ack": "autocommit, READ COMMITTED", "transport": "psycopg, connection per worker", "query_plan": [ { "Plan": { "Node Type": "Limit", "Parallel Aware": false, "Async Capable": false, "Startup Cost": 1675.45, "Total Cost": 1692.58, "Plan Rows": 10, "Plan Width": 18, "Plans": [ { "Node Type": "Index Scan", "Parent Relationship": "Outer", "Parallel Aware": false, "Async Capable": false, "Scan Direction": "Forward", "Index Name": "fb2_429e9a_033_hnsw", "Relation Name": "fb2_429e9a_033", "Alias": "fb2_429e9a_033", "Startup Cost": 1675.45, "Total Cost": 173165.07, "Plan Rows": 100139, "Plan Width": 18, "Order By": "(embedding <=> '[-0.08946895,-0.094362676,0.092023626,-0.0040844884,0.043233823,0.00062519487,0.1075197,-0.035732996,0.013746996,-0.024103122,0.0021706754,-0.04720602,0.04787615,-0.02043222,0.058315646,0.081252135,-0.05375015,-0.030799072,-0.0077205584,0.035735283,-0.04474784,0.01614922,-0.06905586,0.04194401,-0.08832908,0.007873556,0.0044992934,0.021968655,0.010953364,0.07212316,0.023185596,0.001176476,0.04725166,-0.03580423,-0.032344446,0.004752835,-0.056179125,-0.0353235,-0.037251893,-0.005143068,-0.027599342,0.024403147,0.031338032,0.0056410627,0.008425455,-0.0087741045,-0.039870497,-0.011831836,0.042621233,0.048142947,0.01603842,0.06745542,-0.038018145,0.064212486,-0.009636566,0.01607475,-0.0027868534,-0.03761097,-0.011889431,-0.01084577,0.083403684,0.028946918,0.061870236,-0.045382615,0.03812586,0.060426526,-0.017003024,0.02328925,-0.093020774,-0.024072766,0.042720392,-0.022444837,-0.011597358,0.01209231,-0.005953629,0.029691145,-0.011711495,-0.053345133,-0.081757374,-0.049303155,0.032193717,-0.038507868,-0.0060101775,-0.031470332,-0.07362443,0.03575578,0.0245484,0.01985519,-0.15116754,0.050386053,-0.0389634,0.07548389,-0.09264168,-0.012602816,-0.0035539444,-0.046416666,0.015100493,0.100691915,-0.037529,0.04600518,-0.021859422,0.014675989,-0.097722545,-0.053200893,-0.017040461,-0.01629616,-0.012339222,0.06543302,0.07252074,-0.044301156,0.03054835,-0.038570553,0.019310065,-0.005527747,-0.029386865,-0.058248505,0.065443166,0.028506057,0.0878254,0.058387812,0.081721246,-0.0005962058,0.0569388,0.00960514,0.011667701,0.014456492,-0.07218306,-3.0444767e-33,0.06687367,-0.019213954,0.058202963,0.025171444,-0.028259544,-0.04171671,0.03127722,0.04865431,-0.0721459,-0.050143186,-0.029907746,0.06248994,-0.05096076,-0.065192886,0.010210447,-0.077256374,-0.043370593,-0.0901922,-0.066522665,0.026072454,-0.022315364,0.075375885,-0.029968103,-0.0050771856,-0.0046955296,0.0065650316,-0.06791897,0.07368851,0.056069646,-0.021026025,-0.011352883,0.038811427,-0.090847075,-0.056490287,-0.039881036,0.032885592,-0.029586185,0.04753608,0.006341778,-0.0034275919,0.03681623,0.09326265,-0.0532089,0.032396164,0.012605483,0.045778517,-0.0006472902,0.029721813,0.0734947,-0.037117425,-0.09356724,-0.0071634455,-0.032400057,0.010453378,0.06286513,0.006053158,0.060524322,0.091866635,0.063640274,0.055919155,0.053222507,-0.019174587,0.089296214,-0.058922265,0.013745364,-0.022907477,0.028347451,-0.040759467,-0.06820364,-0.061398294,-0.046857864,0.0060627847,0.010442015,-0.07662317,0.031950947,-0.107813306,-0.0016125626,-0.033413693,-0.09989722,-0.03580293,-0.036118146,-0.08276185,0.1280713,0.07596603,0.049225595,-0.026312606,0.0031930343,-0.0012469466,0.000872793,-0.040353082,-0.034516506,-0.036047976,0.004999689,-0.01570079,-0.09783203,1.4186912e-33,-0.0130903,0.04742377,0.02889633,0.09809088,0.043639127,-0.0747531,0.03402747,0.029147558,0.019848268,0.111865185,0.04845458,-0.016222177,0.025501896,0.09813583,-0.002833195,0.0038929228,0.048537854,0.014111116,0.036570966,-0.017512968,-0.079755045,0.043452248,0.10843892,0.061968718,-0.015678233,0.0060376576,0.010718569,-0.03907552,0.040307663,0.04553227,0.041731495,0.083179526,-0.0844024,-0.023447752,0.014485363,-0.00788756,-0.11410207,0.039309476,0.011220049,0.12889338,-0.07970701,0.010658111,-0.06657531,-0.022037188,-0.043459363,-0.030291978,0.05803613,0.06109021,-0.09415837,0.030727232,0.011858702,0.013526454,-0.049580045,-0.052486703,0.013218615,0.0050231596,0.07660412,-0.0046343356,-0.0692584,-0.03353194,0.022288544,0.022836983,0.01361394,-0.036331616,0.008194689,0.09226991,0.08229875,-0.047639843,-0.005033471,0.0066081914,-0.106666416,-0.06385503,0.006385333,0.06318358,0.03146985,-0.060764216,-0.043287244,-0.042385958,-0.07963737,-0.043478705,-0.031840656,0.060027804,0.027829666,0.03728562,-0.07014622,0.0649568,-0.052579004,-0.0034938955,-0.15390792,-0.04426162,0.028217599,0.05593145,0.023251446,0.029115746,-0.040321343,-1.8832651e-08,-0.037536032,0.0005056563,0.013919978,-0.038129315,0.061486673,0.005534046,-0.010768211,-0.16527015,0.061414365,0.019963786,-0.019666849,-0.059065994,0.00083181454,-0.016695915,0.01090875,0.02685501,-0.0008743374,-0.060967337,-0.049258173,0.012731965,-0.03973743,-0.0018210232,-0.03797001,0.00035104176,0.04477184,0.04185603,-0.03215796,-0.09103283,0.0467481,0.017810425,0.037437033,0.054782324,-0.0059442874,0.022557084,0.053393807,-0.03746794,-0.07080435,0.10548834,-0.04215435,0.042547524,0.05064807,0.07560324,0.04072184,-0.0011278372,0.00702018,-0.070807114,0.040035766,0.051417697,0.011316928,-0.028955372,0.04197099,-0.0026867494,-0.056857754,0.0761941,0.030621601,-0.052413527,-0.021492323,0.0957355,0.1013111,-0.082977295,-0.036080915,-0.03327806,0.0029404552,-0.03586191]'::vector)", "Filter": "(cohort = 'static'::text)" } ] } } ] } }, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "eligible": true, "protocol": "mixed-mutations-v2" }, "examples": [ { "event": 0, "operation": "insert", "doc_id": "mutation-000000", "scheduled_s": 0.0, "submitted_s": 0.00020991708151996136, "schedule_delay_ms": 0.20991708151996136, "precondition": true, "status": "success", "ack_ms": 130.37874992005527, "write_started_ms": 0.8697499288246036, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 131.61899999249727, "stable_since_ms": 131.61899999249727, "confirmed_ms": 183.86449990794063, "consecutive": 3, "regressions": 0, "requests": 14 }, "search": { "first_ms": 143.02212500479072, "stable_since_ms": 143.02212500479072, "confirmed_ms": 201.98274997528642, "consecutive": 3, "regressions": 0, "requests": 13 }, "first_observed_ms": 143.02212500479072, "confirmed_ms": 201.98274997528642, "search_minus_ack_ms": 12.643375084735453 }, { "event": 1, "operation": "update", "doc_id": "mutation-000001", "scheduled_s": 0.1, "submitted_s": 0.10048704198561609, "schedule_delay_ms": 0.48704198561608236, "precondition": true, "status": "success", "ack_ms": 30.25879105553031, "write_started_ms": 0.07870804984122515, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 32.84883301239461, "stable_since_ms": 32.84883301239461, "confirmed_ms": 85.13804106041789, "consecutive": 3, "regressions": 0, "requests": 6 }, "search": { "first_ms": 42.567500029690564, "stable_since_ms": 42.567500029690564, "confirmed_ms": 102.50558308325708, "consecutive": 3, "regressions": 0, "requests": 6 }, "first_observed_ms": 42.567500029690564, "confirmed_ms": 102.50558308325708, "search_minus_ack_ms": 12.308708974160254 }, { "event": 2, "operation": "delete", "doc_id": "mutation-000002", "scheduled_s": 0.2, "submitted_s": 0.20040179206989706, "schedule_delay_ms": 0.4017920698970445, "precondition": true, "status": "success", "ack_ms": 2.6576659874990582, "write_started_ms": 1.8107499927282333, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 12.24695798009634, "stable_since_ms": 12.24695798009634, "confirmed_ms": 64.83870791271329, "consecutive": 3, "regressions": 0, "requests": 3 }, "search": { "first_ms": 15.768957906402647, "stable_since_ms": 15.768957906402647, "confirmed_ms": 73.03187495563179, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 15.768957906402647, "confirmed_ms": 73.03187495563179, "search_minus_ack_ms": 13.11129191890359 } ] }, { "run": "fb2_429e9a_034", "trial": 3, "backend": "qdrant", "scenario": "nq-100k", "status": "complete", "eligible": true, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "summary": { "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "wall_s": 30.065831167041324, "operations": { "insert": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 14.0810840530321, "p50": 69.05475008534268, "p95": 105.23606811184435, "p99": 113.34618707187474, "max": 114.62279199622571 }, "confirmed_ms": { "min": 108.35095902439207, "p50": 214.55739601515234, "p95": 233.09823964373203, "p99": 259.5851408469025, "max": 261.17070799227804 }, "ack_ms": { "min": 3.2779170433059335, "p50": 9.291187510825694, "p95": 13.217069098027423, "p99": 13.996319207362836, "max": 15.071584028191864 }, "search_minus_ack_ms": { "min": 2.7882909635081887, "p50": 59.32120903162286, "p95": 98.11179720563813, "p99": 103.61428083735518, "max": 103.91850001178682 }, "regressions": 0, "confirmed_within_100ms": 0.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "update": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 17.690167063847184, "p50": 67.328687524423, "p95": 74.27814853726886, "p99": 89.79292444884784, "max": 104.60237506777048 }, "confirmed_ms": { "min": 113.83358296006918, "p50": 212.79268752550706, "p95": 224.4344085920602, "p99": 229.00197905604733, "max": 237.74070909712464 }, "ack_ms": { "min": 4.673707997426391, "p50": 9.869604022242129, "p95": 14.301708224229515, "p99": 15.146239864407205, "max": 34.18204199988395 }, "search_minus_ack_ms": { "min": 1.156749902293086, "p50": 57.2042494895868, "p95": 59.99224314582534, "p99": 62.185381981544374, "max": 95.95329209696501 }, "regressions": 0, "confirmed_within_100ms": 0.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "delete": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 8.126125088892877, "p50": 17.923292005434632, "p95": 47.99626526655628, "p99": 71.6105474426877, "max": 76.37962501030415 }, "confirmed_ms": { "min": 105.20929098129272, "p50": 154.67702102614567, "p95": 181.20786278159355, "p99": 207.18691613292327, "max": 218.73270790092647 }, "ack_ms": { "min": 3.3303340896964073, "p50": 7.7619164949283, "p95": 13.441701978445051, "p99": 16.245964184636257, "max": 17.60366593953222 }, "search_minus_ack_ms": { "min": -2.902582986280322, "p50": 0.20277098519727588, "p95": 40.67086250288412, "p99": 58.238791264593644, "max": 66.80637504905462 }, "regressions": 0, "confirmed_within_100ms": 0.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 } }, "total_operations": 300, "statuses": { "success": 300, "timeout": 0, "error": 0, "invalid_precondition": 0, "client_overload": 0 }, "query": { "count": 300, "success": 300, "ann_recall_at_10": 0.989, "ndcg_at_10": 0.7931268675492565, "latency_ms": { "min": 3.1782500445842743, "p50": 9.40662540961057, "p95": 47.172491630772136, "p99": 51.375543731264756, "max": 54.36620896216482 }, "dispatch_delay_ms": { "min": 0.05749997217208147, "p50": 1.1718545574694872, "p95": 2.253492374438793, "p99": 2.6835004345048206, "max": 4.270834033377469 } }, "diagnostics": { "schedule_delay_ms": { "min": 0.001417053863583817, "p50": 0.35241703735700014, "p95": 0.6423315941352745, "p99": 1.2771323956556202, "max": 2.001709071919322 }, "probe_dispatch_ms": { "min": 0.019332976080477238, "p50": 0.3982499474659562, "p95": 1.6458748374134293, "p99": 2.654040865600108, "max": 4.7720830189064145 }, "probe_service_ms": { "min": 0.8865419076755643, "p50": 6.649040966294706, "p95": 49.64432930573821, "p99": 51.85736790066585, "max": 57.953375042416155 }, "write_service_ms": { "min": 2.9853330925107002, "p50": 8.627957955468446, "p95": 13.29959959257394, "p99": 14.654977368190876, "max": 33.91633299179375 } }, "actual_submission_rate": 10.00048923798008, "metadata": { "backend": "qdrant", "collection": "fb2_429e9a_034", "dataset": { "dataset": "nq", "source": "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/nq.zip", "source_sha256": "2553bf7bfbab47b1436ca00a34bce57320e18e611fd00999a2a3a1b4714be752", "source_md5": "d4d3d2e48787a744b6f6e691ff534307", "records": 124000, "queries": 100, "relevant_documents_retained": 120, "seed": 20261028, "license": "CC-BY-SA-3.0", "license_source": "https://raw.githubusercontent.com/google-research-datasets/natural-questions/master/nq_open/README.md", "attribution": "BEIR and original dataset authors; Wikipedia contributors for NQ, SciFact authors and cited article authors for SciFact.", "transformation": "200-word title+text prefix; seeded reservoir of documents with relevant documents for selected test queries retained at front. Simulated mutations; no historical replay.", "model": "sentence-transformers/all-MiniLM-L6-v2", "model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "dimensions": 384, "normalized": true, "max_tokens": 256, "device": "mps", "files": { "corpus.jsonl": "9dc487d785e6ee493eccaf764daa3bf8720a8f4fc8c406246065e1ad33a91f5d", "queries.json": "5603f52876ada5d36118b65dea490f1d43900bc159e1804e637796eb9115e999", "vectors.npy": "cb7421eb6ca5964231403f0a922843088f6fd0e01cbf67f2a1076cc7101005db", "query_vectors.npy": "5e666f8d966d8db6b0fd93d4963234d2dffa0ee3910b8fad069a8ee201e8b5d4", "UPSTREAM-LICENSE.md": "8b675521da3fa142ac6bda38c9bcfcc8cff0e2d6128be53a062ed55fb53f7b87" } }, "base_count": 100000, "profile": { "dataset": "nq", "base_count": 100000, "rate": 10, "query_rate": 10, "arrival": "steady" }, "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "preloaded_mutable": 200, "started_utc": "2026-10-29T03:19:59Z", "setup_s": 43.17075745796319, "backend_before": { "version": "1.20.0", "collection": { "status": "green", "optimizer_status": "ok", "indexed_vectors_count": 100200, "points_count": 100200, "segments_count": 2, "config": { "params": { "vectors": { "size": 384, "distance": "Cosine" }, "shard_number": 1, "replication_factor": 1, "write_consistency_factor": 1, "on_disk_payload": true }, "hnsw_config": { "m": 16, "ef_construct": 128, "full_scan_threshold": 10, "max_indexing_threads": 0, "on_disk": false }, "optimizer_config": { "deleted_threshold": 0.2, "vacuum_min_vector_number": 1000, "default_segment_number": 0, "max_segment_size": null, "memmap_threshold": null, "indexing_threshold": 1024, "flush_interval_sec": 5, "max_optimization_threads": null, "prevent_unoptimized": null }, "wal_config": { "wal_capacity_mb": 32, "wal_segments_ahead": 0, "wal_retain_closed": 1 }, "quantization_config": null }, "payload_schema": { "cohort": { "data_type": "keyword", "points": 100200 } }, "update_queue": { "length": 0 }, "created_at": "2026-10-29T03:19:59.272Z" }, "ack": "wait=true", "transport": "HTTP REST, pooled httpx" }, "count_before": 100200, "idle_duration_s": 10.0, "idle_queries": { "count": 100, "ann_recall": 0.9890000000000001, "ndcg": 0.7931268675492565, "latency_ms": { "min": 4.189541912637651, "p50": 10.929604002740234, "p95": 13.227875612210482, "p99": 14.490182435838664, "max": 45.16933299601078 } }, "exact_ndcg": 0.7931268675492565, "resources_before": { "usage_usec": 323995101, "user_usec": 306061143, "system_usec": 17933957, "nice_usec": 231779000, "nr_periods": 6693, "nr_throttled": 548, "throttled_usec": 2506490, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 638844928, "pids": 25 }, "resources_after": { "usage_usec": 331221140, "user_usec": 312468438, "system_usec": 18752701, "nice_usec": 231779000, "nr_periods": 6995, "nr_throttled": 548, "throttled_usec": 2506490, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 622542848, "pids": 33 }, "count_after": 100200, "backend_after": { "version": "1.20.0", "collection": { "status": "green", "optimizer_status": "ok", "indexed_vectors_count": 100200, "points_count": 100200, "segments_count": 2, "config": { "params": { "vectors": { "size": 384, "distance": "Cosine" }, "shard_number": 1, "replication_factor": 1, "write_consistency_factor": 1, "on_disk_payload": true }, "hnsw_config": { "m": 16, "ef_construct": 128, "full_scan_threshold": 10, "max_indexing_threads": 0, "on_disk": false }, "optimizer_config": { "deleted_threshold": 0.2, "vacuum_min_vector_number": 1000, "default_segment_number": 0, "max_segment_size": null, "memmap_threshold": null, "indexing_threshold": 1024, "flush_interval_sec": 5, "max_optimization_threads": null, "prevent_unoptimized": null }, "wal_config": { "wal_capacity_mb": 32, "wal_segments_ahead": 0, "wal_retain_closed": 1 }, "quantization_config": null }, "payload_schema": { "cohort": { "data_type": "keyword", "points": 100200 } }, "update_queue": { "length": 0 }, "created_at": "2026-10-29T03:19:59.272Z" }, "ack": "wait=true", "transport": "HTTP REST, pooled httpx" } }, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "eligible": true, "protocol": "mixed-mutations-v2" }, "examples": [ { "event": 0, "operation": "insert", "doc_id": "mutation-000000", "scheduled_s": 0.0, "submitted_s": 0.002001709071919322, "schedule_delay_ms": 2.001709071919322, "precondition": true, "status": "success", "ack_ms": 6.767332903109491, "write_started_ms": 1.3840829487890005, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 14.627082971855998, "stable_since_ms": 14.627082971855998, "confirmed_ms": 69.63299994822592, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 63.920707907527685, "stable_since_ms": 63.920707907527685, "confirmed_ms": 206.78766595665365, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 63.920707907527685, "confirmed_ms": 206.78766595665365, "search_minus_ack_ms": 57.153375004418194 }, { "event": 1, "operation": "update", "doc_id": "mutation-000001", "scheduled_s": 0.1, "submitted_s": 0.10064308403525501, "schedule_delay_ms": 0.6430840352550093, "precondition": true, "status": "success", "ack_ms": 11.721999966539443, "write_started_ms": 1.4120410196483135, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 22.15687499847263, "stable_since_ms": 22.15687499847263, "confirmed_ms": 76.74399996176362, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 68.94291599746794, "stable_since_ms": 68.94291599746794, "confirmed_ms": 213.9594160253182, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 68.94291599746794, "confirmed_ms": 213.9594160253182, "search_minus_ack_ms": 57.22091603092849 }, { "event": 2, "operation": "delete", "doc_id": "mutation-000002", "scheduled_s": 0.2, "submitted_s": 0.20055879198480397, "schedule_delay_ms": 0.5587919848039635, "precondition": true, "status": "success", "ack_ms": 5.190166993997991, "write_started_ms": 0.804250012151897, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 16.281750053167343, "stable_since_ms": 16.281750053167343, "confirmed_ms": 71.58745802007616, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 60.65312505234033, "stable_since_ms": 60.65312505234033, "confirmed_ms": 202.0857500610873, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 60.65312505234033, "confirmed_ms": 202.0857500610873, "search_minus_ack_ms": 55.46295805834234 } ] }, { "run": "fb2_429e9a_035", "trial": 5, "backend": "pgvector", "scenario": "nq-10k", "status": "complete", "eligible": true, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "summary": { "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "wall_s": 30.001529874978587, "operations": { "insert": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 15.970083069987595, "p50": 22.473000455647707, "p95": 26.53502916218713, "p99": 30.606993528781473, "max": 33.37004198692739 }, "confirmed_ms": { "min": 72.0662500243634, "p50": 86.63533348590136, "p95": 91.47805799148045, "p99": 93.98955727228895, "max": 94.91929190699011 }, "ack_ms": { "min": 5.140624940395355, "p50": 9.969104488845915, "p95": 12.914806621847674, "p99": 16.790158296935278, "max": 19.949084031395614 }, "search_minus_ack_ms": { "min": 10.343375033698976, "p50": 12.962104461621493, "p95": 14.899997640168294, "p99": 15.775158788310367, "max": 16.611792030744255 }, "regressions": 0, "confirmed_within_100ms": 1.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "update": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 15.223084017634392, "p50": 23.289333505090326, "p95": 26.793889555847272, "p99": 27.47223884449344, "max": 34.380541066639125 }, "confirmed_ms": { "min": 70.26587496511638, "p50": 87.42714545223862, "p95": 92.5614287960343, "p99": 94.00744041544387, "max": 95.72166705038399 }, "ack_ms": { "min": 3.7324170116335154, "p50": 10.187728970777243, "p95": 12.323985452530906, "p99": 13.697172813117565, "max": 25.466375052928925 }, "search_minus_ack_ms": { "min": 8.9141660137102, "p50": 12.927770498208702, "p95": 15.652768797008319, "p99": 16.801887390902266, "max": 17.095958930440247 }, "regressions": 0, "confirmed_within_100ms": 1.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "delete": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 12.114624958485365, "p50": 14.092208002693951, "p95": 24.235108389984816, "p99": 25.390477696200836, "max": 25.879207998514175 }, "confirmed_ms": { "min": 63.200625008903444, "p50": 72.30139552848414, "p95": 89.95891908416525, "p99": 91.12695192801766, "max": 91.47700003813952 }, "ack_ms": { "min": 1.5181670896708965, "p50": 2.6527289883233607, "p95": 4.138873307965696, "p99": 4.6108446398284295, "max": 4.842958995141089 }, "search_minus_ack_ms": { "min": 1.3729160418733954, "p50": 4.164042009506375, "p95": 20.93856257852167, "p99": 21.46894426667132, "max": 22.420458029955626 }, "regressions": 0, "confirmed_within_100ms": 1.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 } }, "total_operations": 300, "statuses": { "success": 300, "timeout": 0, "error": 0, "invalid_precondition": 0, "client_overload": 0 }, "query": { "count": 300, "success": 300, "ann_recall_at_10": 0.999, "ndcg_at_10": 0.9481505328688746, "latency_ms": { "min": 1.8070000223815441, "p50": 5.885291495360434, "p95": 7.819029240636155, "p99": 12.784633799456053, "max": 21.602958091534674 }, "dispatch_delay_ms": { "min": 0.04900002386420965, "p50": 1.1706044897437096, "p95": 2.0256608026102185, "p99": 2.206650486914441, "max": 3.0902500730007887 } }, "diagnostics": { "schedule_delay_ms": { "min": 0.004125037230573447, "p50": 0.3329794853925705, "p95": 0.6167812307831255, "p99": 0.6421143920162553, "max": 0.6933340104300356 }, "probe_dispatch_ms": { "min": 0.0301670515909791, "p50": 0.4355834680609405, "p95": 1.8638543318957081, "p99": 2.4613328685518354, "max": 2.9512910405173898 }, "probe_service_ms": { "min": 0.3385830204933882, "p50": 2.8432710096240044, "p95": 7.702381268609314, "p99": 8.771450360072777, "max": 17.89645804092288 }, "write_service_ms": { "min": 1.2087089708074927, "p50": 8.305437513627112, "p95": 11.80671287002042, "p99": 15.632829564856356, "max": 25.02408402506262 } }, "actual_submission_rate": 10.000006396297943, "metadata": { "backend": "pgvector", "collection": "fb2_429e9a_035", "dataset": { "dataset": "nq", "source": "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/nq.zip", "source_sha256": "2553bf7bfbab47b1436ca00a34bce57320e18e611fd00999a2a3a1b4714be752", "source_md5": "d4d3d2e48787a744b6f6e691ff534307", "records": 124000, "queries": 100, "relevant_documents_retained": 120, "seed": 20261028, "license": "CC-BY-SA-3.0", "license_source": "https://raw.githubusercontent.com/google-research-datasets/natural-questions/master/nq_open/README.md", "attribution": "BEIR and original dataset authors; Wikipedia contributors for NQ, SciFact authors and cited article authors for SciFact.", "transformation": "200-word title+text prefix; seeded reservoir of documents with relevant documents for selected test queries retained at front. Simulated mutations; no historical replay.", "model": "sentence-transformers/all-MiniLM-L6-v2", "model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "dimensions": 384, "normalized": true, "max_tokens": 256, "device": "mps", "files": { "corpus.jsonl": "9dc487d785e6ee493eccaf764daa3bf8720a8f4fc8c406246065e1ad33a91f5d", "queries.json": "5603f52876ada5d36118b65dea490f1d43900bc159e1804e637796eb9115e999", "vectors.npy": "cb7421eb6ca5964231403f0a922843088f6fd0e01cbf67f2a1076cc7101005db", "query_vectors.npy": "5e666f8d966d8db6b0fd93d4963234d2dffa0ee3910b8fad069a8ee201e8b5d4", "UPSTREAM-LICENSE.md": "8b675521da3fa142ac6bda38c9bcfcc8cff0e2d6128be53a062ed55fb53f7b87" } }, "base_count": 10000, "profile": { "dataset": "nq", "base_count": 10000, "rate": 10, "query_rate": 10, "arrival": "steady" }, "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "preloaded_mutable": 200, "started_utc": "2026-10-29T03:21:28Z", "setup_s": 3.682270542019978, "backend_before": { "version": "0.8.7", "postgres": "PostgreSQL 16.15 (Debian 16.15-1.pgdg12+2) on aarch64-unknown-linux-gnu, compiled by gcc (Debian 12.2.0-14+deb12u1) 12.2.0, 64-bit", "ef_search": 128, "ack": "autocommit, READ COMMITTED", "transport": "psycopg, connection per worker", "query_plan": [ { "Plan": { "Node Type": "Limit", "Parallel Aware": false, "Async Capable": false, "Startup Cost": 1340.05, "Total Cost": 1356.29, "Plan Rows": 10, "Plan Width": 18, "Plans": [ { "Node Type": "Index Scan", "Parent Relationship": "Outer", "Parallel Aware": false, "Async Capable": false, "Scan Direction": "Forward", "Index Name": "fb2_429e9a_035_hnsw", "Relation Name": "fb2_429e9a_035", "Alias": "fb2_429e9a_035", "Startup Cost": 1340.05, "Total Cost": 17577.0, "Plan Rows": 10000, "Plan Width": 18, "Order By": "(embedding <=> '[-0.08946895,-0.094362676,0.092023626,-0.0040844884,0.043233823,0.00062519487,0.1075197,-0.035732996,0.013746996,-0.024103122,0.0021706754,-0.04720602,0.04787615,-0.02043222,0.058315646,0.081252135,-0.05375015,-0.030799072,-0.0077205584,0.035735283,-0.04474784,0.01614922,-0.06905586,0.04194401,-0.08832908,0.007873556,0.0044992934,0.021968655,0.010953364,0.07212316,0.023185596,0.001176476,0.04725166,-0.03580423,-0.032344446,0.004752835,-0.056179125,-0.0353235,-0.037251893,-0.005143068,-0.027599342,0.024403147,0.031338032,0.0056410627,0.008425455,-0.0087741045,-0.039870497,-0.011831836,0.042621233,0.048142947,0.01603842,0.06745542,-0.038018145,0.064212486,-0.009636566,0.01607475,-0.0027868534,-0.03761097,-0.011889431,-0.01084577,0.083403684,0.028946918,0.061870236,-0.045382615,0.03812586,0.060426526,-0.017003024,0.02328925,-0.093020774,-0.024072766,0.042720392,-0.022444837,-0.011597358,0.01209231,-0.005953629,0.029691145,-0.011711495,-0.053345133,-0.081757374,-0.049303155,0.032193717,-0.038507868,-0.0060101775,-0.031470332,-0.07362443,0.03575578,0.0245484,0.01985519,-0.15116754,0.050386053,-0.0389634,0.07548389,-0.09264168,-0.012602816,-0.0035539444,-0.046416666,0.015100493,0.100691915,-0.037529,0.04600518,-0.021859422,0.014675989,-0.097722545,-0.053200893,-0.017040461,-0.01629616,-0.012339222,0.06543302,0.07252074,-0.044301156,0.03054835,-0.038570553,0.019310065,-0.005527747,-0.029386865,-0.058248505,0.065443166,0.028506057,0.0878254,0.058387812,0.081721246,-0.0005962058,0.0569388,0.00960514,0.011667701,0.014456492,-0.07218306,-3.0444767e-33,0.06687367,-0.019213954,0.058202963,0.025171444,-0.028259544,-0.04171671,0.03127722,0.04865431,-0.0721459,-0.050143186,-0.029907746,0.06248994,-0.05096076,-0.065192886,0.010210447,-0.077256374,-0.043370593,-0.0901922,-0.066522665,0.026072454,-0.022315364,0.075375885,-0.029968103,-0.0050771856,-0.0046955296,0.0065650316,-0.06791897,0.07368851,0.056069646,-0.021026025,-0.011352883,0.038811427,-0.090847075,-0.056490287,-0.039881036,0.032885592,-0.029586185,0.04753608,0.006341778,-0.0034275919,0.03681623,0.09326265,-0.0532089,0.032396164,0.012605483,0.045778517,-0.0006472902,0.029721813,0.0734947,-0.037117425,-0.09356724,-0.0071634455,-0.032400057,0.010453378,0.06286513,0.006053158,0.060524322,0.091866635,0.063640274,0.055919155,0.053222507,-0.019174587,0.089296214,-0.058922265,0.013745364,-0.022907477,0.028347451,-0.040759467,-0.06820364,-0.061398294,-0.046857864,0.0060627847,0.010442015,-0.07662317,0.031950947,-0.107813306,-0.0016125626,-0.033413693,-0.09989722,-0.03580293,-0.036118146,-0.08276185,0.1280713,0.07596603,0.049225595,-0.026312606,0.0031930343,-0.0012469466,0.000872793,-0.040353082,-0.034516506,-0.036047976,0.004999689,-0.01570079,-0.09783203,1.4186912e-33,-0.0130903,0.04742377,0.02889633,0.09809088,0.043639127,-0.0747531,0.03402747,0.029147558,0.019848268,0.111865185,0.04845458,-0.016222177,0.025501896,0.09813583,-0.002833195,0.0038929228,0.048537854,0.014111116,0.036570966,-0.017512968,-0.079755045,0.043452248,0.10843892,0.061968718,-0.015678233,0.0060376576,0.010718569,-0.03907552,0.040307663,0.04553227,0.041731495,0.083179526,-0.0844024,-0.023447752,0.014485363,-0.00788756,-0.11410207,0.039309476,0.011220049,0.12889338,-0.07970701,0.010658111,-0.06657531,-0.022037188,-0.043459363,-0.030291978,0.05803613,0.06109021,-0.09415837,0.030727232,0.011858702,0.013526454,-0.049580045,-0.052486703,0.013218615,0.0050231596,0.07660412,-0.0046343356,-0.0692584,-0.03353194,0.022288544,0.022836983,0.01361394,-0.036331616,0.008194689,0.09226991,0.08229875,-0.047639843,-0.005033471,0.0066081914,-0.106666416,-0.06385503,0.006385333,0.06318358,0.03146985,-0.060764216,-0.043287244,-0.042385958,-0.07963737,-0.043478705,-0.031840656,0.060027804,0.027829666,0.03728562,-0.07014622,0.0649568,-0.052579004,-0.0034938955,-0.15390792,-0.04426162,0.028217599,0.05593145,0.023251446,0.029115746,-0.040321343,-1.8832651e-08,-0.037536032,0.0005056563,0.013919978,-0.038129315,0.061486673,0.005534046,-0.010768211,-0.16527015,0.061414365,0.019963786,-0.019666849,-0.059065994,0.00083181454,-0.016695915,0.01090875,0.02685501,-0.0008743374,-0.060967337,-0.049258173,0.012731965,-0.03973743,-0.0018210232,-0.03797001,0.00035104176,0.04477184,0.04185603,-0.03215796,-0.09103283,0.0467481,0.017810425,0.037437033,0.054782324,-0.0059442874,0.022557084,0.053393807,-0.03746794,-0.07080435,0.10548834,-0.04215435,0.042547524,0.05064807,0.07560324,0.04072184,-0.0011278372,0.00702018,-0.070807114,0.040035766,0.051417697,0.011316928,-0.028955372,0.04197099,-0.0026867494,-0.056857754,0.0761941,0.030621601,-0.052413527,-0.021492323,0.0957355,0.1013111,-0.082977295,-0.036080915,-0.03327806,0.0029404552,-0.03586191]'::vector)", "Filter": "(cohort = 'static'::text)" } ] } } ] }, "count_before": 10200, "idle_duration_s": 10.0, "idle_queries": { "count": 100, "ann_recall": 0.9990000000000001, "ndcg": 0.9481505328688745, "latency_ms": { "min": 2.9701660387218, "p50": 7.629499537870288, "p95": 8.91219102195464, "p99": 13.44962169532668, "max": 21.0104170255363 } }, "exact_ndcg": 0.9481505328688745, "resources_before": { "usage_usec": 679738750, "user_usec": 634603459, "system_usec": 45135291, "nice_usec": 0, "nr_periods": 8957, "nr_throttled": 2621, "throttled_usec": 276135101, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 1168994304, "pids": 23 }, "resources_after": { "usage_usec": 687069699, "user_usec": 641457111, "system_usec": 45612587, "nice_usec": 0, "nr_periods": 9258, "nr_throttled": 2621, "throttled_usec": 276135101, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 1180991488, "pids": 28 }, "count_after": 10200, "backend_after": { "version": "0.8.7", "postgres": "PostgreSQL 16.15 (Debian 16.15-1.pgdg12+2) on aarch64-unknown-linux-gnu, compiled by gcc (Debian 12.2.0-14+deb12u1) 12.2.0, 64-bit", "ef_search": 128, "ack": "autocommit, READ COMMITTED", "transport": "psycopg, connection per worker", "query_plan": [ { "Plan": { "Node Type": "Limit", "Parallel Aware": false, "Async Capable": false, "Startup Cost": 1358.67, "Total Cost": 1375.09, "Plan Rows": 10, "Plan Width": 18, "Plans": [ { "Node Type": "Index Scan", "Parent Relationship": "Outer", "Parallel Aware": false, "Async Capable": false, "Scan Direction": "Forward", "Index Name": "fb2_429e9a_035_hnsw", "Relation Name": "fb2_429e9a_035", "Alias": "fb2_429e9a_035", "Startup Cost": 1358.67, "Total Cost": 17878.41, "Plan Rows": 10061, "Plan Width": 18, "Order By": "(embedding <=> '[-0.08946895,-0.094362676,0.092023626,-0.0040844884,0.043233823,0.00062519487,0.1075197,-0.035732996,0.013746996,-0.024103122,0.0021706754,-0.04720602,0.04787615,-0.02043222,0.058315646,0.081252135,-0.05375015,-0.030799072,-0.0077205584,0.035735283,-0.04474784,0.01614922,-0.06905586,0.04194401,-0.08832908,0.007873556,0.0044992934,0.021968655,0.010953364,0.07212316,0.023185596,0.001176476,0.04725166,-0.03580423,-0.032344446,0.004752835,-0.056179125,-0.0353235,-0.037251893,-0.005143068,-0.027599342,0.024403147,0.031338032,0.0056410627,0.008425455,-0.0087741045,-0.039870497,-0.011831836,0.042621233,0.048142947,0.01603842,0.06745542,-0.038018145,0.064212486,-0.009636566,0.01607475,-0.0027868534,-0.03761097,-0.011889431,-0.01084577,0.083403684,0.028946918,0.061870236,-0.045382615,0.03812586,0.060426526,-0.017003024,0.02328925,-0.093020774,-0.024072766,0.042720392,-0.022444837,-0.011597358,0.01209231,-0.005953629,0.029691145,-0.011711495,-0.053345133,-0.081757374,-0.049303155,0.032193717,-0.038507868,-0.0060101775,-0.031470332,-0.07362443,0.03575578,0.0245484,0.01985519,-0.15116754,0.050386053,-0.0389634,0.07548389,-0.09264168,-0.012602816,-0.0035539444,-0.046416666,0.015100493,0.100691915,-0.037529,0.04600518,-0.021859422,0.014675989,-0.097722545,-0.053200893,-0.017040461,-0.01629616,-0.012339222,0.06543302,0.07252074,-0.044301156,0.03054835,-0.038570553,0.019310065,-0.005527747,-0.029386865,-0.058248505,0.065443166,0.028506057,0.0878254,0.058387812,0.081721246,-0.0005962058,0.0569388,0.00960514,0.011667701,0.014456492,-0.07218306,-3.0444767e-33,0.06687367,-0.019213954,0.058202963,0.025171444,-0.028259544,-0.04171671,0.03127722,0.04865431,-0.0721459,-0.050143186,-0.029907746,0.06248994,-0.05096076,-0.065192886,0.010210447,-0.077256374,-0.043370593,-0.0901922,-0.066522665,0.026072454,-0.022315364,0.075375885,-0.029968103,-0.0050771856,-0.0046955296,0.0065650316,-0.06791897,0.07368851,0.056069646,-0.021026025,-0.011352883,0.038811427,-0.090847075,-0.056490287,-0.039881036,0.032885592,-0.029586185,0.04753608,0.006341778,-0.0034275919,0.03681623,0.09326265,-0.0532089,0.032396164,0.012605483,0.045778517,-0.0006472902,0.029721813,0.0734947,-0.037117425,-0.09356724,-0.0071634455,-0.032400057,0.010453378,0.06286513,0.006053158,0.060524322,0.091866635,0.063640274,0.055919155,0.053222507,-0.019174587,0.089296214,-0.058922265,0.013745364,-0.022907477,0.028347451,-0.040759467,-0.06820364,-0.061398294,-0.046857864,0.0060627847,0.010442015,-0.07662317,0.031950947,-0.107813306,-0.0016125626,-0.033413693,-0.09989722,-0.03580293,-0.036118146,-0.08276185,0.1280713,0.07596603,0.049225595,-0.026312606,0.0031930343,-0.0012469466,0.000872793,-0.040353082,-0.034516506,-0.036047976,0.004999689,-0.01570079,-0.09783203,1.4186912e-33,-0.0130903,0.04742377,0.02889633,0.09809088,0.043639127,-0.0747531,0.03402747,0.029147558,0.019848268,0.111865185,0.04845458,-0.016222177,0.025501896,0.09813583,-0.002833195,0.0038929228,0.048537854,0.014111116,0.036570966,-0.017512968,-0.079755045,0.043452248,0.10843892,0.061968718,-0.015678233,0.0060376576,0.010718569,-0.03907552,0.040307663,0.04553227,0.041731495,0.083179526,-0.0844024,-0.023447752,0.014485363,-0.00788756,-0.11410207,0.039309476,0.011220049,0.12889338,-0.07970701,0.010658111,-0.06657531,-0.022037188,-0.043459363,-0.030291978,0.05803613,0.06109021,-0.09415837,0.030727232,0.011858702,0.013526454,-0.049580045,-0.052486703,0.013218615,0.0050231596,0.07660412,-0.0046343356,-0.0692584,-0.03353194,0.022288544,0.022836983,0.01361394,-0.036331616,0.008194689,0.09226991,0.08229875,-0.047639843,-0.005033471,0.0066081914,-0.106666416,-0.06385503,0.006385333,0.06318358,0.03146985,-0.060764216,-0.043287244,-0.042385958,-0.07963737,-0.043478705,-0.031840656,0.060027804,0.027829666,0.03728562,-0.07014622,0.0649568,-0.052579004,-0.0034938955,-0.15390792,-0.04426162,0.028217599,0.05593145,0.023251446,0.029115746,-0.040321343,-1.8832651e-08,-0.037536032,0.0005056563,0.013919978,-0.038129315,0.061486673,0.005534046,-0.010768211,-0.16527015,0.061414365,0.019963786,-0.019666849,-0.059065994,0.00083181454,-0.016695915,0.01090875,0.02685501,-0.0008743374,-0.060967337,-0.049258173,0.012731965,-0.03973743,-0.0018210232,-0.03797001,0.00035104176,0.04477184,0.04185603,-0.03215796,-0.09103283,0.0467481,0.017810425,0.037437033,0.054782324,-0.0059442874,0.022557084,0.053393807,-0.03746794,-0.07080435,0.10548834,-0.04215435,0.042547524,0.05064807,0.07560324,0.04072184,-0.0011278372,0.00702018,-0.070807114,0.040035766,0.051417697,0.011316928,-0.028955372,0.04197099,-0.0026867494,-0.056857754,0.0761941,0.030621601,-0.052413527,-0.021492323,0.0957355,0.1013111,-0.082977295,-0.036080915,-0.03327806,0.0029404552,-0.03586191]'::vector)", "Filter": "(cohort = 'static'::text)" } ] } } ] } }, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "eligible": true, "protocol": "mixed-mutations-v2" }, "examples": [ { "event": 0, "operation": "insert", "doc_id": "mutation-000000", "scheduled_s": 0.0, "submitted_s": 0.0003121249610558152, "schedule_delay_ms": 0.3121249610558152, "precondition": true, "status": "success", "ack_ms": 19.949084031395614, "write_started_ms": 1.4490419998764992, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 26.56495908740908, "stable_since_ms": 26.56495908740908, "confirmed_ms": 78.75175005756319, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 30.561375082470477, "stable_since_ms": 30.561375082470477, "confirmed_ms": 85.84462502039969, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 30.561375082470477, "confirmed_ms": 85.84462502039969, "search_minus_ack_ms": 10.612291051074862 }, { "event": 1, "operation": "update", "doc_id": "mutation-000001", "scheduled_s": 0.1, "submitted_s": 0.10008716699667275, "schedule_delay_ms": 0.08716699667274397, "precondition": true, "status": "success", "ack_ms": 7.679375004954636, "write_started_ms": 0.699999975040555, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 13.46720801666379, "stable_since_ms": 13.46720801666379, "confirmed_ms": 67.4689169973135, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 18.775625037960708, "stable_since_ms": 18.775625037960708, "confirmed_ms": 83.14441703259945, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 18.775625037960708, "confirmed_ms": 83.14441703259945, "search_minus_ack_ms": 11.096250033006072 }, { "event": 2, "operation": "delete", "doc_id": "mutation-000002", "scheduled_s": 0.2, "submitted_s": 0.20063900004606694, "schedule_delay_ms": 0.6390000460669287, "precondition": true, "status": "success", "ack_ms": 2.9884999385103583, "write_started_ms": 0.3714999184012413, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 13.836959027685225, "stable_since_ms": 13.836959027685225, "confirmed_ms": 68.45620891544968, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 23.91450002323836, "stable_since_ms": 23.91450002323836, "confirmed_ms": 89.93320900481194, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 23.91450002323836, "confirmed_ms": 89.93320900481194, "search_minus_ack_ms": 20.926000084728003 } ] }, { "run": "fb2_429e9a_036", "trial": 4, "backend": "weaviate", "scenario": "nq-10k", "status": "complete", "eligible": true, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "summary": { "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "wall_s": 30.00125208403915, "operations": { "insert": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 7.5421660440042615, "p50": 17.658416996710002, "p95": 30.248952034162354, "p99": 33.25292714638637, "max": 36.26937500666827 }, "confirmed_ms": { "min": 67.02341698110104, "p50": 82.74920849362388, "p95": 103.42967112665065, "p99": 105.13990148203449, "max": 108.69008395820856 }, "ack_ms": { "min": 4.5081659918650985, "p50": 9.790978976525366, "p95": 13.968550571007654, "p99": 20.354745000368, "max": 23.175750044174492 }, "search_minus_ack_ms": { "min": 0.10325002949684858, "p50": 3.6022920394316316, "p95": 18.046053941361606, "p99": 20.96840542275459, "max": 21.936665987595916 }, "regressions": 0, "confirmed_within_100ms": 0.92, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "update": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 6.801416049711406, "p50": 18.06550001492724, "p95": 34.30093359784223, "p99": 36.95582087500952, "max": 37.152625038288534 }, "confirmed_ms": { "min": 68.2593749370426, "p50": 82.72745844442397, "p95": 103.83039367734455, "p99": 106.43460175488144, "max": 107.57495905272663 }, "ack_ms": { "min": 4.454624955542386, "p50": 10.153833543881774, "p95": 15.66044231876731, "p99": 18.194378030020758, "max": 20.71095898281783 }, "search_minus_ack_ms": { "min": -0.25174999609589577, "p50": 3.623270953539759, "p95": 17.623470444232222, "p99": 22.795784999616448, "max": 23.591249948367476 }, "regressions": 0, "confirmed_within_100ms": 0.85, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "delete": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 5.523583036847413, "p50": 16.1766039673239, "p95": 19.993846188299358, "p99": 23.089710976928497, "max": 28.07712508365512 }, "confirmed_ms": { "min": 63.66266601253301, "p50": 79.78533348068595, "p95": 86.50947088026442, "p99": 95.89545429451391, "max": 96.74895904026926 }, "ack_ms": { "min": 2.8899589087814093, "p50": 5.361770512536168, "p95": 9.46342079550959, "p99": 12.607235242612663, "max": 13.430833001621068 }, "search_minus_ack_ms": { "min": 0.11437491048127413, "p50": 5.189354007598013, "p95": 7.2404665988869965, "p99": 13.835562004242114, "max": 17.801708076149225 }, "regressions": 0, "confirmed_within_100ms": 1.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 } }, "total_operations": 300, "statuses": { "success": 300, "timeout": 0, "error": 0, "invalid_precondition": 0, "client_overload": 0 }, "query": { "count": 300, "success": 300, "ann_recall_at_10": 0.996, "ndcg_at_10": 0.9481505328688746, "latency_ms": { "min": 4.744290956296027, "p50": 10.008291515987366, "p95": 14.564614638220519, "p99": 18.829107491765146, "max": 24.016457959078252 }, "dispatch_delay_ms": { "min": 0.07820897735655308, "p50": 1.4978756080381572, "p95": 2.3711875663138926, "p99": 2.97354531357996, "max": 3.3179590245708823 } }, "diagnostics": { "schedule_delay_ms": { "min": 0.002500088886137064, "p50": 0.34656258067045087, "p95": 0.6205629359461007, "p99": 0.6430535854292073, "max": 0.8390841074286115 }, "probe_dispatch_ms": { "min": 0.03295892383903265, "p50": 0.40468748193234205, "p95": 2.3243893985636523, "p99": 2.8608731355052455, "max": 3.4202919341623783 }, "probe_service_ms": { "min": 0.7955000037327409, "p50": 5.393520928919315, "p95": 11.520581337390466, "p99": 15.516549820313235, "max": 23.387999972328544 }, "write_service_ms": { "min": 2.6247079949826, "p50": 8.31450056284666, "p95": 13.460352050606163, "p99": 17.38235756172797, "max": 22.640416049398482 } }, "actual_submission_rate": 9.999982441516655, "metadata": { "backend": "weaviate", "collection": "fb2_429e9a_036", "dataset": { "dataset": "nq", "source": "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/nq.zip", "source_sha256": "2553bf7bfbab47b1436ca00a34bce57320e18e611fd00999a2a3a1b4714be752", "source_md5": "d4d3d2e48787a744b6f6e691ff534307", "records": 124000, "queries": 100, "relevant_documents_retained": 120, "seed": 20261028, "license": "CC-BY-SA-3.0", "license_source": "https://raw.githubusercontent.com/google-research-datasets/natural-questions/master/nq_open/README.md", "attribution": "BEIR and original dataset authors; Wikipedia contributors for NQ, SciFact authors and cited article authors for SciFact.", "transformation": "200-word title+text prefix; seeded reservoir of documents with relevant documents for selected test queries retained at front. Simulated mutations; no historical replay.", "model": "sentence-transformers/all-MiniLM-L6-v2", "model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "dimensions": 384, "normalized": true, "max_tokens": 256, "device": "mps", "files": { "corpus.jsonl": "9dc487d785e6ee493eccaf764daa3bf8720a8f4fc8c406246065e1ad33a91f5d", "queries.json": "5603f52876ada5d36118b65dea490f1d43900bc159e1804e637796eb9115e999", "vectors.npy": "cb7421eb6ca5964231403f0a922843088f6fd0e01cbf67f2a1076cc7101005db", "query_vectors.npy": "5e666f8d966d8db6b0fd93d4963234d2dffa0ee3910b8fad069a8ee201e8b5d4", "UPSTREAM-LICENSE.md": "8b675521da3fa142ac6bda38c9bcfcc8cff0e2d6128be53a062ed55fb53f7b87" } }, "base_count": 10000, "profile": { "dataset": "nq", "base_count": 10000, "rate": 10, "query_rate": 10, "arrival": "steady" }, "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "preloaded_mutable": 200, "started_utc": "2026-10-29T03:22:16Z", "setup_s": 2.48522116697859, "backend_before": { "version": "1.40.3", "schema": { "class": "Ffb2429e9a036", "invertedIndexConfig": { "bm25": { "b": 0.75, "k1": 1.2 }, "cleanupIntervalSeconds": 60, "stopwords": { "additions": null, "preset": "en", "removals": null }, "usingBlockMaxWAND": true }, "multiTenancyConfig": { "autoTenantActivation": false, "autoTenantCreation": false, "enabled": false }, "properties": [ { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "doc_id", "tokenization": "field" }, { "dataType": [ "int" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": false, "name": "version" }, { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "cohort", "tokenization": "field" }, { "dataType": [ "text" ], "indexFilterable": false, "indexRangeFilters": false, "indexSearchable": false, "name": "text", "tokenization": "word" } ], "shardingConfig": { "actualCount": 1, "actualVirtualCount": 128, "desiredCount": 1, "desiredVirtualCount": 128, "function": "murmur3", "key": "_id", "strategy": "hash", "virtualPerPhysical": 128 }, "vectorIndexConfig": { "bq": { "enabled": false }, "cleanupIntervalSeconds": 300, "distance": "cosine", "dynamicEfFactor": 8, "dynamicEfMax": 500, "dynamicEfMin": 100, "ef": 128, "efConstruction": 128, "filterStrategy": "acorn", "flatSearchCutoff": 0, "maxConnections": 16, "multivector": { "aggregation": "maxSim", "enabled": false, "muvera": { "dprojections": 16, "enabled": false, "ksim": 4, "repetitions": 10 } }, "pq": { "bitCompression": false, "centroids": 256, "enabled": false, "encoder": { "distribution": "log-normal", "type": "kmeans" }, "segments": 0, "trainingLimit": 100000 }, "rq": { "bits": 8, "centering": false, "enabled": false, "rescoreLimit": 20, "trainingLimit": 10000 }, "skip": false, "skipDefaultQuantization": false, "sq": { "enabled": false, "rescoreLimit": 20, "trainingLimit": 100000 }, "trackDefaultQuantization": false, "vectorCacheMaxObjects": 1000000000000 }, "vectorIndexType": "hnsw", "vectorizer": "none", "replicationConfig": { "deletionStrategy": "TimeBasedResolution", "factor": 1, "asyncEnabled": false } }, "shards": [ { "name": "PiflCY4gH0oX", "per_node_status": { "node1": "READY" }, "status": "READY", "vectorQueueSize": 0 } ], "ack": "synchronous indexing, object PUT/POST/DELETE", "transport": "HTTP REST writes + GraphQL search, pooled httpx" }, "count_before": 10200, "idle_duration_s": 10.0, "idle_queries": { "count": 100, "ann_recall": 0.996, "ndcg": 0.9481505328688745, "latency_ms": { "min": 4.655708093196154, "p50": 11.477250489406288, "p95": 13.062333688139915, "p99": 14.193392192246407, "max": 16.773250070400536 } }, "exact_ndcg": 0.9481505328688745, "resources_before": { "usage_usec": 390905405, "user_usec": 381705524, "system_usec": 9199880, "nice_usec": 0, "nr_periods": 10118, "nr_throttled": 487, "throttled_usec": 1573043, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 597676032, "pids": 11 }, "resources_after": { "usage_usec": 400002453, "user_usec": 390415756, "system_usec": 9586697, "nice_usec": 0, "nr_periods": 10419, "nr_throttled": 487, "throttled_usec": 1573043, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 603541504, "pids": 11 }, "count_after": 10200, "backend_after": { "version": "1.40.3", "schema": { "class": "Ffb2429e9a036", "invertedIndexConfig": { "bm25": { "b": 0.75, "k1": 1.2 }, "cleanupIntervalSeconds": 60, "stopwords": { "additions": null, "preset": "en", "removals": null }, "usingBlockMaxWAND": true }, "multiTenancyConfig": { "autoTenantActivation": false, "autoTenantCreation": false, "enabled": false }, "properties": [ { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "doc_id", "tokenization": "field" }, { "dataType": [ "int" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": false, "name": "version" }, { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "cohort", "tokenization": "field" }, { "dataType": [ "text" ], "indexFilterable": false, "indexRangeFilters": false, "indexSearchable": false, "name": "text", "tokenization": "word" } ], "shardingConfig": { "actualCount": 1, "actualVirtualCount": 128, "desiredCount": 1, "desiredVirtualCount": 128, "function": "murmur3", "key": "_id", "strategy": "hash", "virtualPerPhysical": 128 }, "vectorIndexConfig": { "bq": { "enabled": false }, "cleanupIntervalSeconds": 300, "distance": "cosine", "dynamicEfFactor": 8, "dynamicEfMax": 500, "dynamicEfMin": 100, "ef": 128, "efConstruction": 128, "filterStrategy": "acorn", "flatSearchCutoff": 0, "maxConnections": 16, "multivector": { "aggregation": "maxSim", "enabled": false, "muvera": { "dprojections": 16, "enabled": false, "ksim": 4, "repetitions": 10 } }, "pq": { "bitCompression": false, "centroids": 256, "enabled": false, "encoder": { "distribution": "log-normal", "type": "kmeans" }, "segments": 0, "trainingLimit": 100000 }, "rq": { "bits": 8, "centering": false, "enabled": false, "rescoreLimit": 20, "trainingLimit": 10000 }, "skip": false, "skipDefaultQuantization": false, "sq": { "enabled": false, "rescoreLimit": 20, "trainingLimit": 100000 }, "trackDefaultQuantization": false, "vectorCacheMaxObjects": 1000000000000 }, "vectorIndexType": "hnsw", "vectorizer": "none", "replicationConfig": { "deletionStrategy": "TimeBasedResolution", "factor": 1, "asyncEnabled": false } }, "shards": [ { "name": "PiflCY4gH0oX", "per_node_status": { "node1": "READY" }, "status": "READY", "vectorQueueSize": 0 } ], "ack": "synchronous indexing, object PUT/POST/DELETE", "transport": "HTTP REST writes + GraphQL search, pooled httpx" } }, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "eligible": true, "protocol": "mixed-mutations-v2" }, "examples": [ { "event": 0, "operation": "insert", "doc_id": "mutation-000000", "scheduled_s": 0.0, "submitted_s": 0.00019370904192328453, "schedule_delay_ms": 0.19370904192328453, "precondition": true, "status": "success", "ack_ms": 5.332582979463041, "write_started_ms": 1.5682909870520234, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 14.895958011038601, "stable_since_ms": 14.895958011038601, "confirmed_ms": 67.87749996874481, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 6.23599998652935, "stable_since_ms": 6.23599998652935, "confirmed_ms": 67.23820802289993, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 14.895958011038601, "confirmed_ms": 67.87749996874481, "search_minus_ack_ms": 0.9034170070663095 }, { "event": 1, "operation": "update", "doc_id": "mutation-000001", "scheduled_s": 0.1, "submitted_s": 0.1000370840774849, "schedule_delay_ms": 0.03708407748490017, "precondition": true, "status": "success", "ack_ms": 9.881332982331514, "write_started_ms": 0.6045829504728317, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 17.472708015702665, "stable_since_ms": 17.472708015702665, "confirmed_ms": 72.58554094005376, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 9.989415993914008, "stable_since_ms": 9.989415993914008, "confirmed_ms": 77.63616601005197, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 17.472708015702665, "confirmed_ms": 77.63616601005197, "search_minus_ack_ms": 0.10808301158249378 }, { "event": 2, "operation": "delete", "doc_id": "mutation-000002", "scheduled_s": 0.2, "submitted_s": 0.2004526670789346, "schedule_delay_ms": 0.4526670789345988, "precondition": true, "status": "success", "ack_ms": 4.779707989655435, "write_started_ms": 0.6136249285191298, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 18.749875016510487, "stable_since_ms": 18.749875016510487, "confirmed_ms": 73.07933294214308, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 11.191208031959832, "stable_since_ms": 11.191208031959832, "confirmed_ms": 74.13195795379579, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 18.749875016510487, "confirmed_ms": 74.13195795379579, "search_minus_ack_ms": 6.411500042304397 } ] }, { "run": "fb2_429e9a_037", "trial": 4, "backend": "pgvector", "scenario": "nq-10k", "status": "complete", "eligible": true, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "summary": { "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "wall_s": 30.00063800008502, "operations": { "insert": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 15.033249976113439, "p50": 23.223208030685782, "p95": 28.122695168713108, "p99": 35.88781991624274, "max": 36.57941601704806 }, "confirmed_ms": { "min": 71.30645797587931, "p50": 87.45822904165834, "p95": 94.8239017801825, "p99": 100.3272370004561, "max": 103.86104101780802 }, "ack_ms": { "min": 4.223458003252745, "p50": 10.20927052013576, "p95": 14.343218400608748, "p99": 20.929262327263135, "max": 21.540916990488768 }, "search_minus_ack_ms": { "min": 10.012582992203534, "p50": 13.12693755608052, "p95": 15.722474956419319, "p99": 17.151613407768323, "max": 19.790500053204596 }, "regressions": 0, "confirmed_within_100ms": 0.98, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "update": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 16.63120905868709, "p50": 22.922332980670035, "p95": 25.98592469585128, "p99": 26.48687750683167, "max": 28.219625004567206 }, "confirmed_ms": { "min": 73.44058400485665, "p50": 87.480541493278, "p95": 93.3352726220619, "p99": 94.37738758744673, "max": 96.27616603393108 }, "ack_ms": { "min": 4.398541990667582, "p50": 9.993291459977627, "p95": 12.443320488091558, "p99": 13.696170400362462, "max": 13.713042018935084 }, "search_minus_ack_ms": { "min": 8.103625033982098, "p50": 12.982749962247908, "p95": 15.529972937656567, "p99": 16.63358000572771, "max": 16.876624897122383 }, "regressions": 0, "confirmed_within_100ms": 1.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "delete": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 12.240167008712888, "p50": 14.046021562535316, "p95": 24.956264585489407, "p99": 25.736577551579106, "max": 26.107292040251195 }, "confirmed_ms": { "min": 64.47658408433199, "p50": 73.57960403896868, "p95": 90.4052707424853, "p99": 92.82444330397993, "max": 93.85833295527846 }, "ack_ms": { "min": 1.3184589333832264, "p50": 2.733167028054595, "p95": 4.19245621887967, "p99": 8.01049274858088, "max": 9.342125034891069 }, "search_minus_ack_ms": { "min": 1.217417069710791, "p50": 4.760708543471992, "p95": 20.7654771162197, "p99": 21.465650505851954, "max": 21.472332999110222 }, "regressions": 0, "confirmed_within_100ms": 1.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 } }, "total_operations": 300, "statuses": { "success": 300, "timeout": 0, "error": 0, "invalid_precondition": 0, "client_overload": 0 }, "query": { "count": 300, "success": 300, "ann_recall_at_10": 0.9979999999999999, "ndcg_at_10": 0.9481505328688746, "latency_ms": { "min": 2.0871669985353947, "p50": 6.039624975528568, "p95": 8.229849993949758, "p99": 9.647131589008454, "max": 20.64737502951175 }, "dispatch_delay_ms": { "min": 0.13833306729793549, "p50": 0.995624577626586, "p95": 1.8920494127087297, "p99": 2.1798203850630666, "max": 4.411416128277779 } }, "diagnostics": { "schedule_delay_ms": { "min": 0.003708107397137894, "p50": 0.3271665540518498, "p95": 0.5949347780552117, "p99": 0.6358363670311107, "max": 2.6111660525200975 }, "probe_dispatch_ms": { "min": 0.021499930880963802, "p50": 0.42333302553743124, "p95": 1.6294909873977288, "p99": 2.4341438198462115, "max": 4.69879200682044 }, "probe_service_ms": { "min": 0.39779103826731443, "p50": 2.9024590039625764, "p95": 7.822408829815686, "p99": 9.405795135535298, "max": 19.845166942104697 }, "write_service_ms": { "min": 1.078665954992175, "p50": 8.58587451511994, "p95": 12.123647914268078, "p99": 16.62507806904612, "max": 20.713084028102458 } }, "actual_submission_rate": 9.999876185809876, "metadata": { "backend": "pgvector", "collection": "fb2_429e9a_037", "dataset": { "dataset": "nq", "source": "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/nq.zip", "source_sha256": "2553bf7bfbab47b1436ca00a34bce57320e18e611fd00999a2a3a1b4714be752", "source_md5": "d4d3d2e48787a744b6f6e691ff534307", "records": 124000, "queries": 100, "relevant_documents_retained": 120, "seed": 20261028, "license": "CC-BY-SA-3.0", "license_source": "https://raw.githubusercontent.com/google-research-datasets/natural-questions/master/nq_open/README.md", "attribution": "BEIR and original dataset authors; Wikipedia contributors for NQ, SciFact authors and cited article authors for SciFact.", "transformation": "200-word title+text prefix; seeded reservoir of documents with relevant documents for selected test queries retained at front. Simulated mutations; no historical replay.", "model": "sentence-transformers/all-MiniLM-L6-v2", "model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "dimensions": 384, "normalized": true, "max_tokens": 256, "device": "mps", "files": { "corpus.jsonl": "9dc487d785e6ee493eccaf764daa3bf8720a8f4fc8c406246065e1ad33a91f5d", "queries.json": "5603f52876ada5d36118b65dea490f1d43900bc159e1804e637796eb9115e999", "vectors.npy": "cb7421eb6ca5964231403f0a922843088f6fd0e01cbf67f2a1076cc7101005db", "query_vectors.npy": "5e666f8d966d8db6b0fd93d4963234d2dffa0ee3910b8fad069a8ee201e8b5d4", "UPSTREAM-LICENSE.md": "8b675521da3fa142ac6bda38c9bcfcc8cff0e2d6128be53a062ed55fb53f7b87" } }, "base_count": 10000, "profile": { "dataset": "nq", "base_count": 10000, "rate": 10, "query_rate": 10, "arrival": "steady" }, "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "preloaded_mutable": 200, "started_utc": "2026-10-29T03:23:03Z", "setup_s": 3.6066815410740674, "backend_before": { "version": "0.8.7", "postgres": "PostgreSQL 16.15 (Debian 16.15-1.pgdg12+2) on aarch64-unknown-linux-gnu, compiled by gcc (Debian 12.2.0-14+deb12u1) 12.2.0, 64-bit", "ef_search": 128, "ack": "autocommit, READ COMMITTED", "transport": "psycopg, connection per worker", "query_plan": [ { "Plan": { "Node Type": "Limit", "Parallel Aware": false, "Async Capable": false, "Startup Cost": 1340.05, "Total Cost": 1356.31, "Plan Rows": 10, "Plan Width": 18, "Plans": [ { "Node Type": "Index Scan", "Parent Relationship": "Outer", "Parallel Aware": false, "Async Capable": false, "Scan Direction": "Forward", "Index Name": "fb2_429e9a_037_hnsw", "Relation Name": "fb2_429e9a_037", "Alias": "fb2_429e9a_037", "Startup Cost": 1340.05, "Total Cost": 17601.0, "Plan Rows": 10000, "Plan Width": 18, "Order By": "(embedding <=> '[-0.08946895,-0.094362676,0.092023626,-0.0040844884,0.043233823,0.00062519487,0.1075197,-0.035732996,0.013746996,-0.024103122,0.0021706754,-0.04720602,0.04787615,-0.02043222,0.058315646,0.081252135,-0.05375015,-0.030799072,-0.0077205584,0.035735283,-0.04474784,0.01614922,-0.06905586,0.04194401,-0.08832908,0.007873556,0.0044992934,0.021968655,0.010953364,0.07212316,0.023185596,0.001176476,0.04725166,-0.03580423,-0.032344446,0.004752835,-0.056179125,-0.0353235,-0.037251893,-0.005143068,-0.027599342,0.024403147,0.031338032,0.0056410627,0.008425455,-0.0087741045,-0.039870497,-0.011831836,0.042621233,0.048142947,0.01603842,0.06745542,-0.038018145,0.064212486,-0.009636566,0.01607475,-0.0027868534,-0.03761097,-0.011889431,-0.01084577,0.083403684,0.028946918,0.061870236,-0.045382615,0.03812586,0.060426526,-0.017003024,0.02328925,-0.093020774,-0.024072766,0.042720392,-0.022444837,-0.011597358,0.01209231,-0.005953629,0.029691145,-0.011711495,-0.053345133,-0.081757374,-0.049303155,0.032193717,-0.038507868,-0.0060101775,-0.031470332,-0.07362443,0.03575578,0.0245484,0.01985519,-0.15116754,0.050386053,-0.0389634,0.07548389,-0.09264168,-0.012602816,-0.0035539444,-0.046416666,0.015100493,0.100691915,-0.037529,0.04600518,-0.021859422,0.014675989,-0.097722545,-0.053200893,-0.017040461,-0.01629616,-0.012339222,0.06543302,0.07252074,-0.044301156,0.03054835,-0.038570553,0.019310065,-0.005527747,-0.029386865,-0.058248505,0.065443166,0.028506057,0.0878254,0.058387812,0.081721246,-0.0005962058,0.0569388,0.00960514,0.011667701,0.014456492,-0.07218306,-3.0444767e-33,0.06687367,-0.019213954,0.058202963,0.025171444,-0.028259544,-0.04171671,0.03127722,0.04865431,-0.0721459,-0.050143186,-0.029907746,0.06248994,-0.05096076,-0.065192886,0.010210447,-0.077256374,-0.043370593,-0.0901922,-0.066522665,0.026072454,-0.022315364,0.075375885,-0.029968103,-0.0050771856,-0.0046955296,0.0065650316,-0.06791897,0.07368851,0.056069646,-0.021026025,-0.011352883,0.038811427,-0.090847075,-0.056490287,-0.039881036,0.032885592,-0.029586185,0.04753608,0.006341778,-0.0034275919,0.03681623,0.09326265,-0.0532089,0.032396164,0.012605483,0.045778517,-0.0006472902,0.029721813,0.0734947,-0.037117425,-0.09356724,-0.0071634455,-0.032400057,0.010453378,0.06286513,0.006053158,0.060524322,0.091866635,0.063640274,0.055919155,0.053222507,-0.019174587,0.089296214,-0.058922265,0.013745364,-0.022907477,0.028347451,-0.040759467,-0.06820364,-0.061398294,-0.046857864,0.0060627847,0.010442015,-0.07662317,0.031950947,-0.107813306,-0.0016125626,-0.033413693,-0.09989722,-0.03580293,-0.036118146,-0.08276185,0.1280713,0.07596603,0.049225595,-0.026312606,0.0031930343,-0.0012469466,0.000872793,-0.040353082,-0.034516506,-0.036047976,0.004999689,-0.01570079,-0.09783203,1.4186912e-33,-0.0130903,0.04742377,0.02889633,0.09809088,0.043639127,-0.0747531,0.03402747,0.029147558,0.019848268,0.111865185,0.04845458,-0.016222177,0.025501896,0.09813583,-0.002833195,0.0038929228,0.048537854,0.014111116,0.036570966,-0.017512968,-0.079755045,0.043452248,0.10843892,0.061968718,-0.015678233,0.0060376576,0.010718569,-0.03907552,0.040307663,0.04553227,0.041731495,0.083179526,-0.0844024,-0.023447752,0.014485363,-0.00788756,-0.11410207,0.039309476,0.011220049,0.12889338,-0.07970701,0.010658111,-0.06657531,-0.022037188,-0.043459363,-0.030291978,0.05803613,0.06109021,-0.09415837,0.030727232,0.011858702,0.013526454,-0.049580045,-0.052486703,0.013218615,0.0050231596,0.07660412,-0.0046343356,-0.0692584,-0.03353194,0.022288544,0.022836983,0.01361394,-0.036331616,0.008194689,0.09226991,0.08229875,-0.047639843,-0.005033471,0.0066081914,-0.106666416,-0.06385503,0.006385333,0.06318358,0.03146985,-0.060764216,-0.043287244,-0.042385958,-0.07963737,-0.043478705,-0.031840656,0.060027804,0.027829666,0.03728562,-0.07014622,0.0649568,-0.052579004,-0.0034938955,-0.15390792,-0.04426162,0.028217599,0.05593145,0.023251446,0.029115746,-0.040321343,-1.8832651e-08,-0.037536032,0.0005056563,0.013919978,-0.038129315,0.061486673,0.005534046,-0.010768211,-0.16527015,0.061414365,0.019963786,-0.019666849,-0.059065994,0.00083181454,-0.016695915,0.01090875,0.02685501,-0.0008743374,-0.060967337,-0.049258173,0.012731965,-0.03973743,-0.0018210232,-0.03797001,0.00035104176,0.04477184,0.04185603,-0.03215796,-0.09103283,0.0467481,0.017810425,0.037437033,0.054782324,-0.0059442874,0.022557084,0.053393807,-0.03746794,-0.07080435,0.10548834,-0.04215435,0.042547524,0.05064807,0.07560324,0.04072184,-0.0011278372,0.00702018,-0.070807114,0.040035766,0.051417697,0.011316928,-0.028955372,0.04197099,-0.0026867494,-0.056857754,0.0761941,0.030621601,-0.052413527,-0.021492323,0.0957355,0.1013111,-0.082977295,-0.036080915,-0.03327806,0.0029404552,-0.03586191]'::vector)", "Filter": "(cohort = 'static'::text)" } ] } } ] }, "count_before": 10200, "idle_duration_s": 10.0, "idle_queries": { "count": 100, "ann_recall": 0.998, "ndcg": 0.9481505328688745, "latency_ms": { "min": 2.1656249882653356, "p50": 7.801625528372824, "p95": 9.081968385726212, "p99": 10.263213368598421, "max": 19.75537498947233 } }, "exact_ndcg": 0.9481505328688745, "resources_before": { "usage_usec": 692411848, "user_usec": 646526326, "system_usec": 45885521, "nice_usec": 0, "nr_periods": 9425, "nr_throttled": 2638, "throttled_usec": 277805317, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 1169432576, "pids": 23 }, "resources_after": { "usage_usec": 699987441, "user_usec": 653601561, "system_usec": 46385880, "nice_usec": 0, "nr_periods": 9726, "nr_throttled": 2638, "throttled_usec": 277805317, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 1181118464, "pids": 28 }, "count_after": 10200, "backend_after": { "version": "0.8.7", "postgres": "PostgreSQL 16.15 (Debian 16.15-1.pgdg12+2) on aarch64-unknown-linux-gnu, compiled by gcc (Debian 12.2.0-14+deb12u1) 12.2.0, 64-bit", "ef_search": 128, "ack": "autocommit, READ COMMITTED", "transport": "psycopg, connection per worker", "query_plan": [ { "Plan": { "Node Type": "Limit", "Parallel Aware": false, "Async Capable": false, "Startup Cost": 1352.0, "Total Cost": 1368.35, "Plan Rows": 10, "Plan Width": 18, "Plans": [ { "Node Type": "Index Scan", "Parent Relationship": "Outer", "Parallel Aware": false, "Async Capable": false, "Scan Direction": "Forward", "Index Name": "fb2_429e9a_037_hnsw", "Relation Name": "fb2_429e9a_037", "Alias": "fb2_429e9a_037", "Startup Cost": 1352.0, "Total Cost": 17887.69, "Plan Rows": 10118, "Plan Width": 18, "Order By": "(embedding <=> '[-0.08946895,-0.094362676,0.092023626,-0.0040844884,0.043233823,0.00062519487,0.1075197,-0.035732996,0.013746996,-0.024103122,0.0021706754,-0.04720602,0.04787615,-0.02043222,0.058315646,0.081252135,-0.05375015,-0.030799072,-0.0077205584,0.035735283,-0.04474784,0.01614922,-0.06905586,0.04194401,-0.08832908,0.007873556,0.0044992934,0.021968655,0.010953364,0.07212316,0.023185596,0.001176476,0.04725166,-0.03580423,-0.032344446,0.004752835,-0.056179125,-0.0353235,-0.037251893,-0.005143068,-0.027599342,0.024403147,0.031338032,0.0056410627,0.008425455,-0.0087741045,-0.039870497,-0.011831836,0.042621233,0.048142947,0.01603842,0.06745542,-0.038018145,0.064212486,-0.009636566,0.01607475,-0.0027868534,-0.03761097,-0.011889431,-0.01084577,0.083403684,0.028946918,0.061870236,-0.045382615,0.03812586,0.060426526,-0.017003024,0.02328925,-0.093020774,-0.024072766,0.042720392,-0.022444837,-0.011597358,0.01209231,-0.005953629,0.029691145,-0.011711495,-0.053345133,-0.081757374,-0.049303155,0.032193717,-0.038507868,-0.0060101775,-0.031470332,-0.07362443,0.03575578,0.0245484,0.01985519,-0.15116754,0.050386053,-0.0389634,0.07548389,-0.09264168,-0.012602816,-0.0035539444,-0.046416666,0.015100493,0.100691915,-0.037529,0.04600518,-0.021859422,0.014675989,-0.097722545,-0.053200893,-0.017040461,-0.01629616,-0.012339222,0.06543302,0.07252074,-0.044301156,0.03054835,-0.038570553,0.019310065,-0.005527747,-0.029386865,-0.058248505,0.065443166,0.028506057,0.0878254,0.058387812,0.081721246,-0.0005962058,0.0569388,0.00960514,0.011667701,0.014456492,-0.07218306,-3.0444767e-33,0.06687367,-0.019213954,0.058202963,0.025171444,-0.028259544,-0.04171671,0.03127722,0.04865431,-0.0721459,-0.050143186,-0.029907746,0.06248994,-0.05096076,-0.065192886,0.010210447,-0.077256374,-0.043370593,-0.0901922,-0.066522665,0.026072454,-0.022315364,0.075375885,-0.029968103,-0.0050771856,-0.0046955296,0.0065650316,-0.06791897,0.07368851,0.056069646,-0.021026025,-0.011352883,0.038811427,-0.090847075,-0.056490287,-0.039881036,0.032885592,-0.029586185,0.04753608,0.006341778,-0.0034275919,0.03681623,0.09326265,-0.0532089,0.032396164,0.012605483,0.045778517,-0.0006472902,0.029721813,0.0734947,-0.037117425,-0.09356724,-0.0071634455,-0.032400057,0.010453378,0.06286513,0.006053158,0.060524322,0.091866635,0.063640274,0.055919155,0.053222507,-0.019174587,0.089296214,-0.058922265,0.013745364,-0.022907477,0.028347451,-0.040759467,-0.06820364,-0.061398294,-0.046857864,0.0060627847,0.010442015,-0.07662317,0.031950947,-0.107813306,-0.0016125626,-0.033413693,-0.09989722,-0.03580293,-0.036118146,-0.08276185,0.1280713,0.07596603,0.049225595,-0.026312606,0.0031930343,-0.0012469466,0.000872793,-0.040353082,-0.034516506,-0.036047976,0.004999689,-0.01570079,-0.09783203,1.4186912e-33,-0.0130903,0.04742377,0.02889633,0.09809088,0.043639127,-0.0747531,0.03402747,0.029147558,0.019848268,0.111865185,0.04845458,-0.016222177,0.025501896,0.09813583,-0.002833195,0.0038929228,0.048537854,0.014111116,0.036570966,-0.017512968,-0.079755045,0.043452248,0.10843892,0.061968718,-0.015678233,0.0060376576,0.010718569,-0.03907552,0.040307663,0.04553227,0.041731495,0.083179526,-0.0844024,-0.023447752,0.014485363,-0.00788756,-0.11410207,0.039309476,0.011220049,0.12889338,-0.07970701,0.010658111,-0.06657531,-0.022037188,-0.043459363,-0.030291978,0.05803613,0.06109021,-0.09415837,0.030727232,0.011858702,0.013526454,-0.049580045,-0.052486703,0.013218615,0.0050231596,0.07660412,-0.0046343356,-0.0692584,-0.03353194,0.022288544,0.022836983,0.01361394,-0.036331616,0.008194689,0.09226991,0.08229875,-0.047639843,-0.005033471,0.0066081914,-0.106666416,-0.06385503,0.006385333,0.06318358,0.03146985,-0.060764216,-0.043287244,-0.042385958,-0.07963737,-0.043478705,-0.031840656,0.060027804,0.027829666,0.03728562,-0.07014622,0.0649568,-0.052579004,-0.0034938955,-0.15390792,-0.04426162,0.028217599,0.05593145,0.023251446,0.029115746,-0.040321343,-1.8832651e-08,-0.037536032,0.0005056563,0.013919978,-0.038129315,0.061486673,0.005534046,-0.010768211,-0.16527015,0.061414365,0.019963786,-0.019666849,-0.059065994,0.00083181454,-0.016695915,0.01090875,0.02685501,-0.0008743374,-0.060967337,-0.049258173,0.012731965,-0.03973743,-0.0018210232,-0.03797001,0.00035104176,0.04477184,0.04185603,-0.03215796,-0.09103283,0.0467481,0.017810425,0.037437033,0.054782324,-0.0059442874,0.022557084,0.053393807,-0.03746794,-0.07080435,0.10548834,-0.04215435,0.042547524,0.05064807,0.07560324,0.04072184,-0.0011278372,0.00702018,-0.070807114,0.040035766,0.051417697,0.011316928,-0.028955372,0.04197099,-0.0026867494,-0.056857754,0.0761941,0.030621601,-0.052413527,-0.021492323,0.0957355,0.1013111,-0.082977295,-0.036080915,-0.03327806,0.0029404552,-0.03586191]'::vector)", "Filter": "(cohort = 'static'::text)" } ] } } ] } }, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "eligible": true, "protocol": "mixed-mutations-v2" }, "examples": [ { "event": 0, "operation": "insert", "doc_id": "mutation-000000", "scheduled_s": 0.0, "submitted_s": 0.00018341606482863426, "schedule_delay_ms": 0.18341606482863426, "precondition": true, "status": "success", "ack_ms": 19.557874999009073, "write_started_ms": 0.800624955445528, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 27.690374990925193, "stable_since_ms": 27.690374990925193, "confirmed_ms": 80.43520897626877, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 31.896417029201984, "stable_since_ms": 31.896417029201984, "confirmed_ms": 90.09308402892202, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 31.896417029201984, "confirmed_ms": 90.09308402892202, "search_minus_ack_ms": 12.338542030192912 }, { "event": 1, "operation": "update", "doc_id": "mutation-000001", "scheduled_s": 0.1, "submitted_s": 0.10033658309839666, "schedule_delay_ms": 0.33658309839665335, "precondition": true, "status": "success", "ack_ms": 6.6943749552592635, "write_started_ms": 1.0101249208673835, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 13.092457898892462, "stable_since_ms": 13.092457898892462, "confirmed_ms": 66.67862494941801, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 19.060916965827346, "stable_since_ms": 19.060916965827346, "confirmed_ms": 78.09308299329132, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 19.060916965827346, "confirmed_ms": 78.09308299329132, "search_minus_ack_ms": 12.366542010568082 }, { "event": 2, "operation": "delete", "doc_id": "mutation-000002", "scheduled_s": 0.2, "submitted_s": 0.20036025007721037, "schedule_delay_ms": 0.3602500772103556, "precondition": true, "status": "success", "ack_ms": 3.410083008930087, "write_started_ms": 0.24674995802342892, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 13.506916002370417, "stable_since_ms": 13.506916002370417, "confirmed_ms": 67.63420801144093, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 23.38608296122402, "stable_since_ms": 23.38608296122402, "confirmed_ms": 84.91354098077863, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 23.38608296122402, "confirmed_ms": 84.91354098077863, "search_minus_ack_ms": 19.975999952293932 } ] }, { "run": "fb2_429e9a_038", "trial": 4, "backend": "weaviate", "scenario": "scifact-small", "status": "complete", "eligible": true, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "summary": { "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "wall_s": 30.01314545806963, "operations": { "insert": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 7.7179999789223075, "p50": 17.013166972901672, "p95": 31.062885321443897, "p99": 36.558990640332965, "max": 45.45162501744926 }, "confirmed_ms": { "min": 68.16795805934817, "p50": 79.24254197860137, "p95": 103.16983934026212, "p99": 105.05971717997457, "max": 109.51562505215406 }, "ack_ms": { "min": 5.272915936075151, "p50": 9.378458024002612, "p95": 14.025539590511471, "p99": 18.292412196751734, "max": 27.779500000178814 }, "search_minus_ack_ms": { "min": -0.3814580850303173, "p50": 1.913062995299697, "p95": 17.043462430592623, "p99": 19.75837003788911, "max": 20.1786250108853 }, "regressions": 0, "confirmed_within_100ms": 0.94, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "update": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 6.545667070895433, "p50": 17.522604553960264, "p95": 32.10251426207833, "p99": 35.42081585270356, "max": 42.382082901895046 }, "confirmed_ms": { "min": 69.26212494727224, "p50": 79.55735397990793, "p95": 103.17213925882243, "p99": 105.1025433139876, "max": 112.18533292412758 }, "ack_ms": { "min": 4.96720802038908, "p50": 9.642083488870412, "p95": 15.00088091706857, "p99": 18.16547635593456, "max": 26.44616598263383 }, "search_minus_ack_ms": { "min": 0.06666709668934345, "p50": 2.6831874856725335, "p95": 17.626556241884828, "p99": 19.63901727925986, "max": 19.785165903158486 }, "regressions": 0, "confirmed_within_100ms": 0.93, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "delete": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 6.1359170358628035, "p50": 16.784666455350816, "p95": 20.77167949173599, "p99": 22.904902732698282, "max": 29.96545797213912 }, "confirmed_ms": { "min": 68.0857920087874, "p50": 77.8475624974817, "p95": 82.89536254596896, "p99": 99.34899964951916, "max": 111.34449997916818 }, "ack_ms": { "min": 2.4019170086830854, "p50": 5.424958013463765, "p95": 10.026555607328191, "p99": 10.493189155822623, "max": 11.056415969505906 }, "search_minus_ack_ms": { "min": 0.5767090478911996, "p50": 4.39900002675131, "p95": 8.589222980663179, "p99": 12.221963246120174, "max": 18.909042002633214 }, "regressions": 0, "confirmed_within_100ms": 0.99, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 } }, "total_operations": 300, "statuses": { "success": 300, "timeout": 0, "error": 0, "invalid_precondition": 0, "client_overload": 0 }, "query": { "count": 300, "success": 300, "ann_recall_at_10": 1.0, "ndcg_at_10": 0.6567157762068453, "latency_ms": { "min": 4.549582954496145, "p50": 9.785500005818903, "p95": 15.721488045528533, "p99": 18.810131651116503, "max": 26.23854100238532 }, "dispatch_delay_ms": { "min": 0.1308330101892352, "p50": 1.6079790075309575, "p95": 2.541893534362316, "p99": 2.859169755829494, "max": 3.517917008139193 } }, "diagnostics": { "schedule_delay_ms": { "min": 0.005875038914027186, "p50": 0.30722907977231984, "p95": 0.6219042441800937, "p99": 0.6429387093523342, "max": 0.680125039070667 }, "probe_dispatch_ms": { "min": 0.03458303399384022, "p50": 0.4353959229774773, "p95": 2.404289389960467, "p99": 2.9748800094239414, "max": 3.607250051572919 }, "probe_service_ms": { "min": 0.9123750496655703, "p50": 5.619000003207475, "p95": 11.095112498151135, "p99": 14.647325279656798, "max": 35.1062499685213 }, "write_service_ms": { "min": 2.286541974171996, "p50": 7.606312516145408, "p95": 13.300520560005682, "p99": 17.40063823061064, "max": 26.983957970514894 } }, "actual_submission_rate": 10.000010242487475, "metadata": { "backend": "weaviate", "collection": "fb2_429e9a_038", "dataset": { "dataset": "scifact", "source": "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/scifact.zip", "source_sha256": "536e14446a0ba56ed1398ab1055f39fe852686ecad24a6306c80c490fa8e0165", "source_md5": "5f7d1de60b170fc8027bb7898e2efca1", "records": 5183, "queries": 100, "relevant_documents_retained": 107, "seed": 20261028, "license": "Corpus: ODC-By-1.0; claims/evidence: CC-BY-4.0", "license_source": "https://raw.githubusercontent.com/allenai/scifact/master/LICENSE.md", "attribution": "BEIR and original dataset authors; Wikipedia contributors for NQ, SciFact authors and cited article authors for SciFact.", "transformation": "200-word title+text prefix; seeded reservoir of documents with relevant documents for selected test queries retained at front. Simulated mutations; no historical replay.", "model": "sentence-transformers/all-MiniLM-L6-v2", "model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "dimensions": 384, "normalized": true, "max_tokens": 256, "device": "mps", "files": { "corpus.jsonl": "189d58f273468352c9f73c6333c5120dcfb7ba1695984f073f0ebb6c100ed0cd", "queries.json": "c8b333c73353a8c4aeb7d6b63067c9a241aa5d39a0314f0c315ec63019ee9558", "vectors.npy": "b6b21404977151ce565e7d6d57d9931a4679e1ecf4aa44fcd9b29eb54955d09e", "query_vectors.npy": "02324eb7c3b3d2e2f0ec3208453e2ec0efd5d61d9fc845b1f8b7950e913f2730", "UPSTREAM-LICENSE.md": "56ee69bc639cbab8f0b2e4b67091ec5e9390aa99a1d65f0a35e4274686c9a806" } }, "base_count": 4000, "profile": { "dataset": "scifact", "base_count": 4000, "rate": 10, "query_rate": 10, "arrival": "steady" }, "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "preloaded_mutable": 200, "started_utc": "2026-10-29T03:23:51Z", "setup_s": 1.2998492079786956, "backend_before": { "version": "1.40.3", "schema": { "class": "Ffb2429e9a038", "invertedIndexConfig": { "bm25": { "b": 0.75, "k1": 1.2 }, "cleanupIntervalSeconds": 60, "stopwords": { "additions": null, "preset": "en", "removals": null }, "usingBlockMaxWAND": true }, "multiTenancyConfig": { "autoTenantActivation": false, "autoTenantCreation": false, "enabled": false }, "properties": [ { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "doc_id", "tokenization": "field" }, { "dataType": [ "int" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": false, "name": "version" }, { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "cohort", "tokenization": "field" }, { "dataType": [ "text" ], "indexFilterable": false, "indexRangeFilters": false, "indexSearchable": false, "name": "text", "tokenization": "word" } ], "shardingConfig": { "actualCount": 1, "actualVirtualCount": 128, "desiredCount": 1, "desiredVirtualCount": 128, "function": "murmur3", "key": "_id", "strategy": "hash", "virtualPerPhysical": 128 }, "vectorIndexConfig": { "bq": { "enabled": false }, "cleanupIntervalSeconds": 300, "distance": "cosine", "dynamicEfFactor": 8, "dynamicEfMax": 500, "dynamicEfMin": 100, "ef": 128, "efConstruction": 128, "filterStrategy": "acorn", "flatSearchCutoff": 0, "maxConnections": 16, "multivector": { "aggregation": "maxSim", "enabled": false, "muvera": { "dprojections": 16, "enabled": false, "ksim": 4, "repetitions": 10 } }, "pq": { "bitCompression": false, "centroids": 256, "enabled": false, "encoder": { "distribution": "log-normal", "type": "kmeans" }, "segments": 0, "trainingLimit": 100000 }, "rq": { "bits": 8, "centering": false, "enabled": false, "rescoreLimit": 20, "trainingLimit": 10000 }, "skip": false, "skipDefaultQuantization": false, "sq": { "enabled": false, "rescoreLimit": 20, "trainingLimit": 100000 }, "trackDefaultQuantization": false, "vectorCacheMaxObjects": 1000000000000 }, "vectorIndexType": "hnsw", "vectorizer": "none", "replicationConfig": { "deletionStrategy": "TimeBasedResolution", "factor": 1, "asyncEnabled": false } }, "shards": [ { "name": "0aV1iRXKJ6Mw", "per_node_status": { "node1": "READY" }, "status": "READY", "vectorQueueSize": 0 } ], "ack": "synchronous indexing, object PUT/POST/DELETE", "transport": "HTTP REST writes + GraphQL search, pooled httpx" }, "count_before": 4200, "idle_duration_s": 10.0, "idle_queries": { "count": 100, "ann_recall": 1.0, "ndcg": 0.6567157762068453, "latency_ms": { "min": 4.775625071488321, "p50": 10.011833452153951, "p95": 12.831679248483852, "p99": 14.655433137668298, "max": 14.987000031396747 } }, "exact_ndcg": 0.6567157762068453, "resources_before": { "usage_usec": 402929973, "user_usec": 393203347, "system_usec": 9726625, "nice_usec": 0, "nr_periods": 10695, "nr_throttled": 487, "throttled_usec": 1573043, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 556343296, "pids": 11 }, "resources_after": { "usage_usec": 411014214, "user_usec": 400860548, "system_usec": 10153665, "nice_usec": 0, "nr_periods": 10996, "nr_throttled": 487, "throttled_usec": 1573043, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 551976960, "pids": 11 }, "count_after": 4200, "backend_after": { "version": "1.40.3", "schema": { "class": "Ffb2429e9a038", "invertedIndexConfig": { "bm25": { "b": 0.75, "k1": 1.2 }, "cleanupIntervalSeconds": 60, "stopwords": { "additions": null, "preset": "en", "removals": null }, "usingBlockMaxWAND": true }, "multiTenancyConfig": { "autoTenantActivation": false, "autoTenantCreation": false, "enabled": false }, "properties": [ { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "doc_id", "tokenization": "field" }, { "dataType": [ "int" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": false, "name": "version" }, { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "cohort", "tokenization": "field" }, { "dataType": [ "text" ], "indexFilterable": false, "indexRangeFilters": false, "indexSearchable": false, "name": "text", "tokenization": "word" } ], "shardingConfig": { "actualCount": 1, "actualVirtualCount": 128, "desiredCount": 1, "desiredVirtualCount": 128, "function": "murmur3", "key": "_id", "strategy": "hash", "virtualPerPhysical": 128 }, "vectorIndexConfig": { "bq": { "enabled": false }, "cleanupIntervalSeconds": 300, "distance": "cosine", "dynamicEfFactor": 8, "dynamicEfMax": 500, "dynamicEfMin": 100, "ef": 128, "efConstruction": 128, "filterStrategy": "acorn", "flatSearchCutoff": 0, "maxConnections": 16, "multivector": { "aggregation": "maxSim", "enabled": false, "muvera": { "dprojections": 16, "enabled": false, "ksim": 4, "repetitions": 10 } }, "pq": { "bitCompression": false, "centroids": 256, "enabled": false, "encoder": { "distribution": "log-normal", "type": "kmeans" }, "segments": 0, "trainingLimit": 100000 }, "rq": { "bits": 8, "centering": false, "enabled": false, "rescoreLimit": 20, "trainingLimit": 10000 }, "skip": false, "skipDefaultQuantization": false, "sq": { "enabled": false, "rescoreLimit": 20, "trainingLimit": 100000 }, "trackDefaultQuantization": false, "vectorCacheMaxObjects": 1000000000000 }, "vectorIndexType": "hnsw", "vectorizer": "none", "replicationConfig": { "deletionStrategy": "TimeBasedResolution", "factor": 1, "asyncEnabled": false } }, "shards": [ { "name": "0aV1iRXKJ6Mw", "per_node_status": { "node1": "READY" }, "status": "READY", "vectorQueueSize": 0 } ], "ack": "synchronous indexing, object PUT/POST/DELETE", "transport": "HTTP REST writes + GraphQL search, pooled httpx" } }, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "eligible": true, "protocol": "mixed-mutations-v2" }, "examples": [ { "event": 0, "operation": "insert", "doc_id": "mutation-000000", "scheduled_s": 0.0, "submitted_s": 0.00028470810502767563, "schedule_delay_ms": 0.28470810502767563, "precondition": true, "status": "success", "ack_ms": 5.387791898101568, "write_started_ms": 1.826333929784596, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 14.846041915006936, "stable_since_ms": 14.846041915006936, "confirmed_ms": 68.8529999461025, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 5.958041991107166, "stable_since_ms": 5.958041991107166, "confirmed_ms": 63.49024991504848, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 14.846041915006936, "confirmed_ms": 68.8529999461025, "search_minus_ack_ms": 0.5702500930055976 }, { "event": 1, "operation": "update", "doc_id": "mutation-000001", "scheduled_s": 0.1, "submitted_s": 0.10020554205402732, "schedule_delay_ms": 0.2055420540273134, "precondition": true, "status": "success", "ack_ms": 7.677291054278612, "write_started_ms": 0.987416016869247, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 8.252208004705608, "stable_since_ms": 8.252208004705608, "confirmed_ms": 67.7208750275895, "consecutive": 3, "regressions": 0, "requests": 3 }, "search": { "first_ms": 8.864625007845461, "stable_since_ms": 8.864625007845461, "confirmed_ms": 73.1315000448376, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 8.864625007845461, "confirmed_ms": 73.1315000448376, "search_minus_ack_ms": 1.1873339535668492 }, { "event": 2, "operation": "delete", "doc_id": "mutation-000002", "scheduled_s": 0.2, "submitted_s": 0.20016004203353077, "schedule_delay_ms": 0.16004203353076063, "precondition": true, "status": "success", "ack_ms": 5.890832981094718, "write_started_ms": 0.9247499983757734, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 19.48374998755753, "stable_since_ms": 19.48374998755753, "confirmed_ms": 74.06029105186462, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 9.213375044055283, "stable_since_ms": 9.213375044055283, "confirmed_ms": 76.55150000937283, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 19.48374998755753, "confirmed_ms": 76.55150000937283, "search_minus_ack_ms": 3.322542062960565 } ] }, { "run": "fb2_429e9a_039", "trial": 5, "backend": "qdrant", "scenario": "nq-100k", "status": "complete", "eligible": true, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "summary": { "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "wall_s": 30.058049833984114, "operations": { "insert": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 14.242124976590276, "p50": 68.3844790328294, "p95": 74.6026917011477, "p99": 111.7676924704574, "max": 111.77437496371567 }, "confirmed_ms": { "min": 111.12295801285654, "p50": 215.8347915392369, "p95": 222.8782527614385, "p99": 255.60483356472105, "max": 257.01979198493063 }, "ack_ms": { "min": 4.024166031740606, "p50": 8.695625059772283, "p95": 12.791349971666932, "p99": 15.804371928097682, "max": 16.294916975311935 }, "search_minus_ack_ms": { "min": 1.7166249454021454, "p50": 59.17320802109316, "p95": 63.04019584786146, "p99": 101.0163721535355, "max": 102.91779192630202 }, "regressions": 0, "confirmed_within_100ms": 0.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "update": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 16.45795803051442, "p50": 67.81733344541863, "p95": 73.06746215908788, "p99": 74.59447253844718, "max": 107.12562501430511 }, "confirmed_ms": { "min": 148.7170410109684, "p50": 214.75558349629864, "p95": 222.98056180006824, "p99": 225.08958788472242, "max": 251.81175000034273 }, "ack_ms": { "min": 5.038624978624284, "p50": 10.119416459929198, "p95": 14.525078481528904, "p99": 17.435893772635616, "max": 18.992834025993943 }, "search_minus_ack_ms": { "min": 1.2928330106660724, "p50": 57.40425054682419, "p95": 60.05726426374167, "p99": 61.43621366238239, "max": 96.42495797015727 }, "regressions": 0, "confirmed_within_100ms": 0.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "delete": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 14.91170795634389, "p50": 17.921896011102945, "p95": 21.21363806654699, "p99": 59.272907904814964, "max": 66.16904202383012 }, "confirmed_ms": { "min": 106.9703329121694, "p50": 155.4539164644666, "p95": 162.42330349050462, "p99": 205.12997380224994, "max": 216.7598749510944 }, "ack_ms": { "min": 3.69554094504565, "p50": 9.204791975207627, "p95": 11.865287064574659, "p99": 15.929068294353785, "max": 16.232833033427596 }, "search_minus_ack_ms": { "min": -3.53562505915761, "p50": 0.1736244885250926, "p95": 3.1940173183102143, "p99": 49.18015953386206, "max": 56.01458402816206 }, "regressions": 0, "confirmed_within_100ms": 0.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 } }, "total_operations": 300, "statuses": { "success": 300, "timeout": 0, "error": 0, "invalid_precondition": 0, "client_overload": 0 }, "query": { "count": 300, "success": 300, "ann_recall_at_10": 0.989, "ndcg_at_10": 0.7931268675492565, "latency_ms": { "min": 3.7090409314259887, "p50": 8.913208963349462, "p95": 12.25551397074014, "p99": 47.699941949686036, "max": 50.52658298518509 }, "dispatch_delay_ms": { "min": 0.14470901805907488, "p50": 1.523333485238254, "p95": 2.3694334318861365, "p99": 2.916936296969647, "max": 12.615167070180178 } }, "diagnostics": { "schedule_delay_ms": { "min": 0.006459024735505636, "p50": 0.3712920355614102, "p95": 0.6614461401491135, "p99": 0.9536559281418979, "max": 9.369500074535964 }, "probe_dispatch_ms": { "min": 0.029832939617335796, "p50": 0.39175001438707113, "p95": 1.9504422321915635, "p99": 2.726010303013026, "max": 5.979417008347809 }, "probe_service_ms": { "min": 0.922541948966682, "p50": 6.34779199026525, "p95": 49.26160839386284, "p99": 50.81403337419032, "max": 55.12379202991724 }, "write_service_ms": { "min": 3.5378329921513796, "p50": 8.950479503255337, "p95": 12.447441287804399, "p99": 15.750495791435242, "max": 18.55012506712228 } }, "actual_submission_rate": 9.9998536670228, "metadata": { "backend": "qdrant", "collection": "fb2_429e9a_039", "dataset": { "dataset": "nq", "source": "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/nq.zip", "source_sha256": "2553bf7bfbab47b1436ca00a34bce57320e18e611fd00999a2a3a1b4714be752", "source_md5": "d4d3d2e48787a744b6f6e691ff534307", "records": 124000, "queries": 100, "relevant_documents_retained": 120, "seed": 20261028, "license": "CC-BY-SA-3.0", "license_source": "https://raw.githubusercontent.com/google-research-datasets/natural-questions/master/nq_open/README.md", "attribution": "BEIR and original dataset authors; Wikipedia contributors for NQ, SciFact authors and cited article authors for SciFact.", "transformation": "200-word title+text prefix; seeded reservoir of documents with relevant documents for selected test queries retained at front. Simulated mutations; no historical replay.", "model": "sentence-transformers/all-MiniLM-L6-v2", "model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "dimensions": 384, "normalized": true, "max_tokens": 256, "device": "mps", "files": { "corpus.jsonl": "9dc487d785e6ee493eccaf764daa3bf8720a8f4fc8c406246065e1ad33a91f5d", "queries.json": "5603f52876ada5d36118b65dea490f1d43900bc159e1804e637796eb9115e999", "vectors.npy": "cb7421eb6ca5964231403f0a922843088f6fd0e01cbf67f2a1076cc7101005db", "query_vectors.npy": "5e666f8d966d8db6b0fd93d4963234d2dffa0ee3910b8fad069a8ee201e8b5d4", "UPSTREAM-LICENSE.md": "8b675521da3fa142ac6bda38c9bcfcc8cff0e2d6128be53a062ed55fb53f7b87" } }, "base_count": 100000, "profile": { "dataset": "nq", "base_count": 100000, "rate": 10, "query_rate": 10, "arrival": "steady" }, "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "preloaded_mutable": 200, "started_utc": "2026-10-29T03:24:36Z", "setup_s": 45.40784354193602, "backend_before": { "version": "1.20.0", "collection": { "status": "green", "optimizer_status": "ok", "indexed_vectors_count": 100200, "points_count": 100200, "segments_count": 2, "config": { "params": { "vectors": { "size": 384, "distance": "Cosine" }, "shard_number": 1, "replication_factor": 1, "write_consistency_factor": 1, "on_disk_payload": true }, "hnsw_config": { "m": 16, "ef_construct": 128, "full_scan_threshold": 10, "max_indexing_threads": 0, "on_disk": false }, "optimizer_config": { "deleted_threshold": 0.2, "vacuum_min_vector_number": 1000, "default_segment_number": 0, "max_segment_size": null, "memmap_threshold": null, "indexing_threshold": 1024, "flush_interval_sec": 5, "max_optimization_threads": null, "prevent_unoptimized": null }, "wal_config": { "wal_capacity_mb": 32, "wal_segments_ahead": 0, "wal_retain_closed": 1 }, "quantization_config": null }, "payload_schema": { "cohort": { "data_type": "keyword", "points": 100200 } }, "update_queue": { "length": 0 }, "created_at": "2026-10-29T03:24:36.873Z" }, "ack": "wait=true", "transport": "HTTP REST, pooled httpx" }, "count_before": 100200, "idle_duration_s": 10.0, "idle_queries": { "count": 100, "ann_recall": 0.9890000000000001, "ndcg": 0.7931268675492565, "latency_ms": { "min": 5.645582918077707, "p50": 10.829999519046396, "p95": 12.572421116055919, "p99": 14.598465102026413, "max": 48.32104197703302 } }, "exact_ndcg": 0.7931268675492565, "resources_before": { "usage_usec": 398918760, "user_usec": 377775792, "system_usec": 21142967, "nice_usec": 292500000, "nr_periods": 7677, "nr_throttled": 641, "throttled_usec": 2818381, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 665292800, "pids": 25 }, "resources_after": { "usage_usec": 405819743, "user_usec": 383966899, "system_usec": 21852844, "nice_usec": 292500000, "nr_periods": 7978, "nr_throttled": 641, "throttled_usec": 2818381, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 630902784, "pids": 32 }, "count_after": 100200, "backend_after": { "version": "1.20.0", "collection": { "status": "green", "optimizer_status": "ok", "indexed_vectors_count": 100200, "points_count": 100200, "segments_count": 2, "config": { "params": { "vectors": { "size": 384, "distance": "Cosine" }, "shard_number": 1, "replication_factor": 1, "write_consistency_factor": 1, "on_disk_payload": true }, "hnsw_config": { "m": 16, "ef_construct": 128, "full_scan_threshold": 10, "max_indexing_threads": 0, "on_disk": false }, "optimizer_config": { "deleted_threshold": 0.2, "vacuum_min_vector_number": 1000, "default_segment_number": 0, "max_segment_size": null, "memmap_threshold": null, "indexing_threshold": 1024, "flush_interval_sec": 5, "max_optimization_threads": null, "prevent_unoptimized": null }, "wal_config": { "wal_capacity_mb": 32, "wal_segments_ahead": 0, "wal_retain_closed": 1 }, "quantization_config": null }, "payload_schema": { "cohort": { "data_type": "keyword", "points": 100200 } }, "update_queue": { "length": 0 }, "created_at": "2026-10-29T03:24:36.873Z" }, "ack": "wait=true", "transport": "HTTP REST, pooled httpx" } }, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "eligible": true, "protocol": "mixed-mutations-v2" }, "examples": [ { "event": 0, "operation": "insert", "doc_id": "mutation-000000", "scheduled_s": 0.0, "submitted_s": 0.0001953750615939498, "schedule_delay_ms": 0.1953750615939498, "precondition": true, "status": "success", "ack_ms": 6.584916962310672, "write_started_ms": 1.4187089400365949, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 14.048084034584463, "stable_since_ms": 14.048084034584463, "confirmed_ms": 68.14391701482236, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 59.5898749306798, "stable_since_ms": 59.5898749306798, "confirmed_ms": 203.00366694573313, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 59.5898749306798, "confirmed_ms": 203.00366694573313, "search_minus_ack_ms": 53.004957968369126 }, { "event": 1, "operation": "update", "doc_id": "mutation-000001", "scheduled_s": 0.1, "submitted_s": 0.10028720903210342, "schedule_delay_ms": 0.28720903210341375, "precondition": true, "status": "success", "ack_ms": 11.081125005148351, "write_started_ms": 1.825332990847528, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 20.390499965287745, "stable_since_ms": 20.390499965287745, "confirmed_ms": 76.9853750243783, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 67.22083303611726, "stable_since_ms": 67.22083303611726, "confirmed_ms": 213.94312498159707, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 67.22083303611726, "confirmed_ms": 213.94312498159707, "search_minus_ack_ms": 56.139708030968904 }, { "event": 2, "operation": "delete", "doc_id": "mutation-000002", "scheduled_s": 0.2, "submitted_s": 0.2001645000418648, "schedule_delay_ms": 0.16450004186480127, "precondition": true, "status": "success", "ack_ms": 8.113541989587247, "write_started_ms": 1.221999991685152, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 17.709334031678736, "stable_since_ms": 17.709334031678736, "confirmed_ms": 73.41558404732496, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 48.70050004683435, "stable_since_ms": 48.70050004683435, "confirmed_ms": 192.69183394499123, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 48.70050004683435, "confirmed_ms": 192.69183394499123, "search_minus_ack_ms": 40.5869580572471 } ] }, { "run": "fb2_429e9a_040", "trial": 5, "backend": "weaviate", "scenario": "nq-10k", "status": "complete", "eligible": true, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "summary": { "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "wall_s": 30.001507417066023, "operations": { "insert": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 7.243832922540605, "p50": 17.934249946847558, "p95": 30.464852188015357, "p99": 31.37587486417033, "max": 31.949167023412883 }, "confirmed_ms": { "min": 67.41524999961257, "p50": 81.48443751269951, "p95": 98.07594587909989, "p99": 104.44370930315927, "max": 104.60470896214247 }, "ack_ms": { "min": 5.002374993637204, "p50": 9.87947901012376, "p95": 13.239064620574936, "p99": 13.874042395036682, "max": 16.72449999023229 }, "search_minus_ack_ms": { "min": -0.34816598054021597, "p50": 3.2800209592096508, "p95": 17.10423125186935, "p99": 18.670887512853373, "max": 19.26195900887251 }, "regressions": 0, "confirmed_within_100ms": 0.97, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "update": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 8.226417005062103, "p50": 18.131770484615117, "p95": 31.76589564536698, "p99": 33.85779492789882, "max": 39.85995892435312 }, "confirmed_ms": { "min": 65.86658291053027, "p50": 82.01743749668822, "p95": 103.14910382730886, "p99": 104.17228416306898, "max": 104.80679094325751 }, "ack_ms": { "min": 2.933999989181757, "p50": 9.594187489710748, "p95": 13.8528729963582, "p99": 15.154400337487537, "max": 30.192542006261647 }, "search_minus_ack_ms": { "min": 0.11324998922646046, "p50": 3.185542009305209, "p95": 17.94865769916214, "p99": 18.584938800195236, "max": 19.209999940358102 }, "regressions": 0, "confirmed_within_100ms": 0.92, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "delete": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 4.396458039991558, "p50": 16.68752048863098, "p95": 22.56528986035846, "p99": 28.588988331612207, "max": 29.718125006183982 }, "confirmed_ms": { "min": 63.65833303425461, "p50": 79.95164603926241, "p95": 84.78239559917711, "p99": 91.23517962638292, "max": 97.20533399377018 }, "ack_ms": { "min": 2.193500055000186, "p50": 5.685646028723568, "p95": 11.045406904304397, "p99": 13.689464229391895, "max": 15.022416948340833 }, "search_minus_ack_ms": { "min": 0.1259589334949851, "p50": 4.852312442380935, "p95": 7.43168993503786, "p99": 15.61516074230896, "max": 18.72066594660282 }, "regressions": 0, "confirmed_within_100ms": 1.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 } }, "total_operations": 300, "statuses": { "success": 300, "timeout": 0, "error": 0, "invalid_precondition": 0, "client_overload": 0 }, "query": { "count": 300, "success": 300, "ann_recall_at_10": 0.996, "ndcg_at_10": 0.9481505328688746, "latency_ms": { "min": 3.658417030237615, "p50": 9.999292029533535, "p95": 14.259504532674333, "p99": 20.371750742197033, "max": 29.605957912281156 }, "dispatch_delay_ms": { "min": 0.145458965562284, "p50": 1.650500576943159, "p95": 2.427089895354584, "p99": 2.990492129465565, "max": 3.3512921072542667 } }, "diagnostics": { "schedule_delay_ms": { "min": 0.0071249669417738914, "p50": 0.27404202846881276, "p95": 0.6305357289964508, "p99": 0.648215822178415, "max": 0.8318339707322764 }, "probe_dispatch_ms": { "min": 0.029041897505521774, "p50": 0.40639599319547415, "p95": 2.501064533134922, "p99": 3.0158826895058155, "max": 9.324499987997115 }, "probe_service_ms": { "min": 0.756374909542501, "p50": 5.5597914615646005, "p95": 11.162173235788941, "p99": 14.505446810508186, "max": 30.308542074635625 }, "write_service_ms": { "min": 1.7622500890865922, "p50": 8.144082967191935, "p95": 12.467814306728542, "p99": 14.078420060686762, "max": 29.34420807287097 } }, "actual_submission_rate": 9.999907247353677, "metadata": { "backend": "weaviate", "collection": "fb2_429e9a_040", "dataset": { "dataset": "nq", "source": "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/nq.zip", "source_sha256": "2553bf7bfbab47b1436ca00a34bce57320e18e611fd00999a2a3a1b4714be752", "source_md5": "d4d3d2e48787a744b6f6e691ff534307", "records": 124000, "queries": 100, "relevant_documents_retained": 120, "seed": 20261028, "license": "CC-BY-SA-3.0", "license_source": "https://raw.githubusercontent.com/google-research-datasets/natural-questions/master/nq_open/README.md", "attribution": "BEIR and original dataset authors; Wikipedia contributors for NQ, SciFact authors and cited article authors for SciFact.", "transformation": "200-word title+text prefix; seeded reservoir of documents with relevant documents for selected test queries retained at front. Simulated mutations; no historical replay.", "model": "sentence-transformers/all-MiniLM-L6-v2", "model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "dimensions": 384, "normalized": true, "max_tokens": 256, "device": "mps", "files": { "corpus.jsonl": "9dc487d785e6ee493eccaf764daa3bf8720a8f4fc8c406246065e1ad33a91f5d", "queries.json": "5603f52876ada5d36118b65dea490f1d43900bc159e1804e637796eb9115e999", "vectors.npy": "cb7421eb6ca5964231403f0a922843088f6fd0e01cbf67f2a1076cc7101005db", "query_vectors.npy": "5e666f8d966d8db6b0fd93d4963234d2dffa0ee3910b8fad069a8ee201e8b5d4", "UPSTREAM-LICENSE.md": "8b675521da3fa142ac6bda38c9bcfcc8cff0e2d6128be53a062ed55fb53f7b87" } }, "base_count": 10000, "profile": { "dataset": "nq", "base_count": 10000, "rate": 10, "query_rate": 10, "arrival": "steady" }, "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "preloaded_mutable": 200, "started_utc": "2026-10-29T03:26:08Z", "setup_s": 2.6157567080808803, "backend_before": { "version": "1.40.3", "schema": { "class": "Ffb2429e9a040", "invertedIndexConfig": { "bm25": { "b": 0.75, "k1": 1.2 }, "cleanupIntervalSeconds": 60, "stopwords": { "additions": null, "preset": "en", "removals": null }, "usingBlockMaxWAND": true }, "multiTenancyConfig": { "autoTenantActivation": false, "autoTenantCreation": false, "enabled": false }, "properties": [ { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "doc_id", "tokenization": "field" }, { "dataType": [ "int" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": false, "name": "version" }, { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "cohort", "tokenization": "field" }, { "dataType": [ "text" ], "indexFilterable": false, "indexRangeFilters": false, "indexSearchable": false, "name": "text", "tokenization": "word" } ], "shardingConfig": { "actualCount": 1, "actualVirtualCount": 128, "desiredCount": 1, "desiredVirtualCount": 128, "function": "murmur3", "key": "_id", "strategy": "hash", "virtualPerPhysical": 128 }, "vectorIndexConfig": { "bq": { "enabled": false }, "cleanupIntervalSeconds": 300, "distance": "cosine", "dynamicEfFactor": 8, "dynamicEfMax": 500, "dynamicEfMin": 100, "ef": 128, "efConstruction": 128, "filterStrategy": "acorn", "flatSearchCutoff": 0, "maxConnections": 16, "multivector": { "aggregation": "maxSim", "enabled": false, "muvera": { "dprojections": 16, "enabled": false, "ksim": 4, "repetitions": 10 } }, "pq": { "bitCompression": false, "centroids": 256, "enabled": false, "encoder": { "distribution": "log-normal", "type": "kmeans" }, "segments": 0, "trainingLimit": 100000 }, "rq": { "bits": 8, "centering": false, "enabled": false, "rescoreLimit": 20, "trainingLimit": 10000 }, "skip": false, "skipDefaultQuantization": false, "sq": { "enabled": false, "rescoreLimit": 20, "trainingLimit": 100000 }, "trackDefaultQuantization": false, "vectorCacheMaxObjects": 1000000000000 }, "vectorIndexType": "hnsw", "vectorizer": "none", "replicationConfig": { "deletionStrategy": "TimeBasedResolution", "factor": 1, "asyncEnabled": false } }, "shards": [ { "name": "QXX9q5JLAl7w", "per_node_status": { "node1": "READY" }, "status": "READY", "vectorQueueSize": 0 } ], "ack": "synchronous indexing, object PUT/POST/DELETE", "transport": "HTTP REST writes + GraphQL search, pooled httpx" }, "count_before": 10200, "idle_duration_s": 10.0, "idle_queries": { "count": 100, "ann_recall": 0.996, "ndcg": 0.9481505328688745, "latency_ms": { "min": 3.557374933734536, "p50": 9.965437580831349, "p95": 12.129691627342254, "p99": 12.847484922967855, "max": 13.633833965286613 } }, "exact_ndcg": 0.9481505328688745, "resources_before": { "usage_usec": 416680917, "user_usec": 406351568, "system_usec": 10329348, "nice_usec": 0, "nr_periods": 11361, "nr_throttled": 488, "throttled_usec": 1573084, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 575643648, "pids": 11 }, "resources_after": { "usage_usec": 425390527, "user_usec": 414660183, "system_usec": 10730343, "nice_usec": 0, "nr_periods": 11662, "nr_throttled": 488, "throttled_usec": 1573084, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 589811712, "pids": 11 }, "count_after": 10200, "backend_after": { "version": "1.40.3", "schema": { "class": "Ffb2429e9a040", "invertedIndexConfig": { "bm25": { "b": 0.75, "k1": 1.2 }, "cleanupIntervalSeconds": 60, "stopwords": { "additions": null, "preset": "en", "removals": null }, "usingBlockMaxWAND": true }, "multiTenancyConfig": { "autoTenantActivation": false, "autoTenantCreation": false, "enabled": false }, "properties": [ { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "doc_id", "tokenization": "field" }, { "dataType": [ "int" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": false, "name": "version" }, { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "cohort", "tokenization": "field" }, { "dataType": [ "text" ], "indexFilterable": false, "indexRangeFilters": false, "indexSearchable": false, "name": "text", "tokenization": "word" } ], "shardingConfig": { "actualCount": 1, "actualVirtualCount": 128, "desiredCount": 1, "desiredVirtualCount": 128, "function": "murmur3", "key": "_id", "strategy": "hash", "virtualPerPhysical": 128 }, "vectorIndexConfig": { "bq": { "enabled": false }, "cleanupIntervalSeconds": 300, "distance": "cosine", "dynamicEfFactor": 8, "dynamicEfMax": 500, "dynamicEfMin": 100, "ef": 128, "efConstruction": 128, "filterStrategy": "acorn", "flatSearchCutoff": 0, "maxConnections": 16, "multivector": { "aggregation": "maxSim", "enabled": false, "muvera": { "dprojections": 16, "enabled": false, "ksim": 4, "repetitions": 10 } }, "pq": { "bitCompression": false, "centroids": 256, "enabled": false, "encoder": { "distribution": "log-normal", "type": "kmeans" }, "segments": 0, "trainingLimit": 100000 }, "rq": { "bits": 8, "centering": false, "enabled": false, "rescoreLimit": 20, "trainingLimit": 10000 }, "skip": false, "skipDefaultQuantization": false, "sq": { "enabled": false, "rescoreLimit": 20, "trainingLimit": 100000 }, "trackDefaultQuantization": false, "vectorCacheMaxObjects": 1000000000000 }, "vectorIndexType": "hnsw", "vectorizer": "none", "replicationConfig": { "deletionStrategy": "TimeBasedResolution", "factor": 1, "asyncEnabled": false } }, "shards": [ { "name": "QXX9q5JLAl7w", "per_node_status": { "node1": "READY" }, "status": "READY", "vectorQueueSize": 0 } ], "ack": "synchronous indexing, object PUT/POST/DELETE", "transport": "HTTP REST writes + GraphQL search, pooled httpx" } }, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "eligible": true, "protocol": "mixed-mutations-v2" }, "examples": [ { "event": 0, "operation": "insert", "doc_id": "mutation-000000", "scheduled_s": 0.0, "submitted_s": 0.00017345906235277653, "schedule_delay_ms": 0.17345906235277653, "precondition": true, "status": "success", "ack_ms": 5.029415944591165, "write_started_ms": 1.3453749706968665, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 14.306665980257094, "stable_since_ms": 14.306665980257094, "confirmed_ms": 67.41524999961257, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 6.641332991421223, "stable_since_ms": 6.641332991421223, "confirmed_ms": 64.7393330000341, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 14.306665980257094, "confirmed_ms": 67.41524999961257, "search_minus_ack_ms": 1.611917046830058 }, { "event": 1, "operation": "update", "doc_id": "mutation-000001", "scheduled_s": 0.1, "submitted_s": 0.10023137496318668, "schedule_delay_ms": 0.23137496318667572, "precondition": true, "status": "success", "ack_ms": 5.914875073358417, "write_started_ms": 0.6800420815125108, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 15.184084069915116, "stable_since_ms": 15.184084069915116, "confirmed_ms": 69.01904207188636, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 7.498792023397982, "stable_since_ms": 7.498792023397982, "confirmed_ms": 71.49908400606364, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 15.184084069915116, "confirmed_ms": 71.49908400606364, "search_minus_ack_ms": 1.5839169500395656 }, { "event": 2, "operation": "delete", "doc_id": "mutation-000002", "scheduled_s": 0.2, "submitted_s": 0.20007408398669213, "schedule_delay_ms": 0.07408398669211946, "precondition": true, "status": "success", "ack_ms": 6.118666031397879, "write_started_ms": 0.7558750221505761, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 20.421791006810963, "stable_since_ms": 20.421791006810963, "confirmed_ms": 74.37716599088162, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 13.446583063341677, "stable_since_ms": 13.446583063341677, "confirmed_ms": 77.01787503901869, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 20.421791006810963, "confirmed_ms": 77.01787503901869, "search_minus_ack_ms": 7.327917031943798 } ] }, { "run": "fb2_429e9a_041", "trial": 5, "backend": "qdrant", "scenario": "nq-100k-burst", "status": "complete", "eligible": true, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "summary": { "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "burst", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "wall_s": 30.064672874985263, "operations": { "insert": { "count": 130, "success": 130, "success_rate": 1.0, "first_observed_ms": { "min": 16.299833077937365, "p50": 69.75220801541582, "p95": 104.82638475950807, "p99": 113.92065174062736, "max": 119.19220897834748 }, "confirmed_ms": { "min": 146.8923749634996, "p50": 218.56949949869886, "p95": 251.59784343559295, "p99": 260.63015284482384, "max": 263.0154170328751 }, "ack_ms": { "min": 2.6755830040201545, "p50": 8.31754197133705, "p95": 12.936773110413924, "p99": 13.670839676633474, "max": 17.296125064603984 }, "search_minus_ack_ms": { "min": 2.1576250437647104, "p50": 59.86214545555413, "p95": 99.11755367647856, "p99": 102.30614044121465, "max": 109.354083891958 }, "regressions": 0, "confirmed_within_100ms": 0.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "update": { "count": 130, "success": 130, "success_rate": 1.0, "first_observed_ms": { "min": 17.00470899231732, "p50": 69.79702058015391, "p95": 104.7482599853538, "p99": 108.13189459266141, "max": 109.34629198163748 }, "confirmed_ms": { "min": 108.43187500722706, "p50": 217.87916700122878, "p95": 248.60607528244145, "p99": 251.27024703193456, "max": 251.8645829986781 }, "ack_ms": { "min": 3.607042017392814, "p50": 10.027458483818918, "p95": 13.805862650042396, "p99": 18.49271965213118, "max": 27.836708934046328 }, "search_minus_ack_ms": { "min": 1.1819580104202032, "p50": 58.32083348650485, "p95": 98.11685807071626, "p99": 99.71972148981877, "max": 101.27225005999207 }, "regressions": 0, "confirmed_within_100ms": 0.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "delete": { "count": 130, "success": 130, "success_rate": 1.0, "first_observed_ms": { "min": 13.551749987527728, "p50": 20.253124996088445, "p95": 51.40850858297199, "p99": 95.31587785109855, "max": 108.05108398199081 }, "confirmed_ms": { "min": 108.50520909298211, "p50": 158.87516649672762, "p95": 201.17622331017628, "p99": 238.6646233336067, "max": 254.62691695429385 }, "ack_ms": { "min": 3.076000022701919, "p50": 9.1099584242329, "p95": 12.18396276817657, "p99": 12.987665437394751, "max": 17.566791968420148 }, "search_minus_ack_ms": { "min": -3.3745840191841125, "p50": 0.4773124819621444, "p95": 43.49155850941315, "p99": 86.18574468884594, "max": 99.45970901753753 }, "regressions": 0, "confirmed_within_100ms": 0.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 } }, "total_operations": 390, "statuses": { "success": 390, "timeout": 0, "error": 0, "invalid_precondition": 0, "client_overload": 0 }, "query": { "count": 300, "success": 300, "ann_recall_at_10": 0.989, "ndcg_at_10": 0.7931268675492565, "latency_ms": { "min": 4.376666969619691, "p50": 9.893770446069539, "p95": 49.616958398837596, "p99": 51.816281222272664, "max": 53.83162503130734 }, "dispatch_delay_ms": { "min": 0.4628748865798116, "p50": 1.510103465989232, "p95": 2.4827871355228126, "p99": 2.7284796885214737, "max": 3.085999982431531 } }, "diagnostics": { "schedule_delay_ms": { "min": 0.010707951151545103, "p50": 0.3687284188336193, "p95": 0.6560097041069745, "p99": 0.8961293655448447, "max": 1.7451659310623313 }, "probe_dispatch_ms": { "min": 0.03416696563363075, "p50": 0.42395846685394645, "p95": 1.8073896644636989, "p99": 2.666324395686388, "max": 4.534709034487605 }, "probe_service_ms": { "min": 0.9468330536037683, "p50": 6.636770907789469, "p95": 49.717587034683675, "p99": 51.58831992652267, "max": 60.2313750423491 }, "write_service_ms": { "min": 2.5262919953092933, "p50": 8.542999508790672, "p95": 12.258248444413766, "p99": 15.55786634329708, "max": 26.987791992723942 } }, "actual_submission_rate": 13.009951588736744, "metadata": { "backend": "qdrant", "collection": "fb2_429e9a_041", "dataset": { "dataset": "nq", "source": "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/nq.zip", "source_sha256": "2553bf7bfbab47b1436ca00a34bce57320e18e611fd00999a2a3a1b4714be752", "source_md5": "d4d3d2e48787a744b6f6e691ff534307", "records": 124000, "queries": 100, "relevant_documents_retained": 120, "seed": 20261028, "license": "CC-BY-SA-3.0", "license_source": "https://raw.githubusercontent.com/google-research-datasets/natural-questions/master/nq_open/README.md", "attribution": "BEIR and original dataset authors; Wikipedia contributors for NQ, SciFact authors and cited article authors for SciFact.", "transformation": "200-word title+text prefix; seeded reservoir of documents with relevant documents for selected test queries retained at front. Simulated mutations; no historical replay.", "model": "sentence-transformers/all-MiniLM-L6-v2", "model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "dimensions": 384, "normalized": true, "max_tokens": 256, "device": "mps", "files": { "corpus.jsonl": "9dc487d785e6ee493eccaf764daa3bf8720a8f4fc8c406246065e1ad33a91f5d", "queries.json": "5603f52876ada5d36118b65dea490f1d43900bc159e1804e637796eb9115e999", "vectors.npy": "cb7421eb6ca5964231403f0a922843088f6fd0e01cbf67f2a1076cc7101005db", "query_vectors.npy": "5e666f8d966d8db6b0fd93d4963234d2dffa0ee3910b8fad069a8ee201e8b5d4", "UPSTREAM-LICENSE.md": "8b675521da3fa142ac6bda38c9bcfcc8cff0e2d6128be53a062ed55fb53f7b87" } }, "base_count": 100000, "profile": { "dataset": "nq", "base_count": 100000, "rate": 10, "query_rate": 10, "arrival": "burst" }, "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "burst", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "preloaded_mutable": 260, "started_utc": "2026-10-29T03:26:55Z", "setup_s": 41.96031241607852, "backend_before": { "version": "1.20.0", "collection": { "status": "green", "optimizer_status": "ok", "indexed_vectors_count": 100260, "points_count": 100260, "segments_count": 2, "config": { "params": { "vectors": { "size": 384, "distance": "Cosine" }, "shard_number": 1, "replication_factor": 1, "write_consistency_factor": 1, "on_disk_payload": true }, "hnsw_config": { "m": 16, "ef_construct": 128, "full_scan_threshold": 10, "max_indexing_threads": 0, "on_disk": false }, "optimizer_config": { "deleted_threshold": 0.2, "vacuum_min_vector_number": 1000, "default_segment_number": 0, "max_segment_size": null, "memmap_threshold": null, "indexing_threshold": 1024, "flush_interval_sec": 5, "max_optimization_threads": null, "prevent_unoptimized": null }, "wal_config": { "wal_capacity_mb": 32, "wal_segments_ahead": 0, "wal_retain_closed": 1 }, "quantization_config": null }, "payload_schema": { "cohort": { "data_type": "keyword", "points": 100260 } }, "update_queue": { "length": 0 }, "created_at": "2026-10-29T03:26:55.323Z" }, "ack": "wait=true", "transport": "HTTP REST, pooled httpx" }, "count_before": 100260, "idle_duration_s": 10.0, "idle_queries": { "count": 100, "ann_recall": 0.9890000000000001, "ndcg": 0.7931268675492565, "latency_ms": { "min": 4.684415995143354, "p50": 10.105458495672792, "p95": 11.685805988963693, "p99": 13.387900326633998, "max": 46.66679201181978 } }, "exact_ndcg": 0.7931268675492565, "resources_before": { "usage_usec": 468314709, "user_usec": 444243988, "system_usec": 24070720, "nice_usec": 348248000, "nr_periods": 8571, "nr_throttled": 808, "throttled_usec": 3221196, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 649797632, "pids": 30 }, "resources_after": { "usage_usec": 476063418, "user_usec": 451149634, "system_usec": 24913783, "nice_usec": 348248000, "nr_periods": 8872, "nr_throttled": 808, "throttled_usec": 3221196, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 625561600, "pids": 32 }, "count_after": 100260, "backend_after": { "version": "1.20.0", "collection": { "status": "green", "optimizer_status": "ok", "indexed_vectors_count": 100260, "points_count": 100260, "segments_count": 2, "config": { "params": { "vectors": { "size": 384, "distance": "Cosine" }, "shard_number": 1, "replication_factor": 1, "write_consistency_factor": 1, "on_disk_payload": true }, "hnsw_config": { "m": 16, "ef_construct": 128, "full_scan_threshold": 10, "max_indexing_threads": 0, "on_disk": false }, "optimizer_config": { "deleted_threshold": 0.2, "vacuum_min_vector_number": 1000, "default_segment_number": 0, "max_segment_size": null, "memmap_threshold": null, "indexing_threshold": 1024, "flush_interval_sec": 5, "max_optimization_threads": null, "prevent_unoptimized": null }, "wal_config": { "wal_capacity_mb": 32, "wal_segments_ahead": 0, "wal_retain_closed": 1 }, "quantization_config": null }, "payload_schema": { "cohort": { "data_type": "keyword", "points": 100260 } }, "update_queue": { "length": 0 }, "created_at": "2026-10-29T03:26:55.323Z" }, "ack": "wait=true", "transport": "HTTP REST, pooled httpx" } }, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "eligible": true, "protocol": "mixed-mutations-v2" }, "examples": [ { "event": 0, "operation": "insert", "doc_id": "mutation-000000", "scheduled_s": 0.0, "submitted_s": 0.0001955829793587327, "schedule_delay_ms": 0.1955829793587327, "precondition": true, "status": "success", "ack_ms": 7.282624952495098, "write_started_ms": 1.899832976050675, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 15.026291948743165, "stable_since_ms": 15.026291948743165, "confirmed_ms": 68.89658293221146, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 60.58729195501655, "stable_since_ms": 60.58729195501655, "confirmed_ms": 200.09412500075996, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 60.58729195501655, "confirmed_ms": 200.09412500075996, "search_minus_ack_ms": 53.304667002521455 }, { "event": 1, "operation": "update", "doc_id": "mutation-000001", "scheduled_s": 0.025, "submitted_s": 0.02536124992184341, "schedule_delay_ms": 0.36124992184340815, "precondition": true, "status": "success", "ack_ms": 4.94383298791945, "write_started_ms": 0.7692499784752727, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 13.899291050620377, "stable_since_ms": 13.899291050620377, "confirmed_ms": 66.96933298371732, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 58.67887497879565, "stable_since_ms": 58.67887497879565, "confirmed_ms": 199.53858305234462, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 58.67887497879565, "confirmed_ms": 199.53858305234462, "search_minus_ack_ms": 53.7350419908762 }, { "event": 2, "operation": "delete", "doc_id": "mutation-000002", "scheduled_s": 0.05, "submitted_s": 0.05017837497871369, "schedule_delay_ms": 0.17837497871368846, "precondition": true, "status": "success", "ack_ms": 3.8072909228503704, "write_started_ms": 0.9161659982055426, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 13.66425002925098, "stable_since_ms": 13.66425002925098, "confirmed_ms": 66.90045795403421, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 44.58641598466784, "stable_since_ms": 44.58641598466784, "confirmed_ms": 186.55062501784414, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 44.58641598466784, "confirmed_ms": 186.55062501784414, "search_minus_ack_ms": 40.77912506181747 } ] }, { "run": "fb2_429e9a_042", "trial": 4, "backend": "qdrant", "scenario": "nq-100k", "status": "complete", "eligible": true, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "summary": { "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "wall_s": 30.07761220796965, "operations": { "insert": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 17.270958051085472, "p50": 67.11252051172778, "p95": 72.10925590479746, "p99": 76.26651913858967, "max": 107.90712502785027 }, "confirmed_ms": { "min": 150.78970894683152, "p50": 211.83572948211804, "p95": 218.97600833908655, "p99": 221.87210117699578, "max": 221.96041699498892 }, "ack_ms": { "min": 4.034416982904077, "p50": 8.186541963368654, "p95": 11.266185744898394, "p99": 11.827582420082765, "max": 12.202832964248955 }, "search_minus_ack_ms": { "min": 0.5377079360187054, "p50": 58.5093125118874, "p95": 62.82433966989629, "p99": 67.192606610479, "max": 100.69812496658415 }, "regressions": 0, "confirmed_within_100ms": 0.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "update": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 16.361249960027635, "p50": 65.90718746883795, "p95": 70.96087315003388, "p99": 73.73010864015671, "max": 80.06033301353455 }, "confirmed_ms": { "min": 106.4621249679476, "p50": 211.65637503145263, "p95": 221.72822562861256, "p99": 222.5001747091301, "max": 227.01375000178814 }, "ack_ms": { "min": 4.161208984442055, "p50": 8.975187491159886, "p95": 12.33306074864231, "p99": 15.191914427559865, "max": 23.54479196947068 }, "search_minus_ack_ms": { "min": 1.0742080630734563, "p50": 56.74983351491392, "p95": 59.36521695111878, "p99": 59.53078129445203, "max": 59.76900004316121 }, "regressions": 0, "confirmed_within_100ms": 0.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "delete": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 4.649624926969409, "p50": 17.64224999351427, "p95": 20.734089275356382, "p99": 49.35823504114532, "max": 66.19091692846268 }, "confirmed_ms": { "min": 106.56945803202689, "p50": 153.17302098264918, "p95": 159.60381933837198, "p99": 175.6969695619774, "max": 191.8062079930678 }, "ack_ms": { "min": 3.331791958771646, "p50": 7.8962709521874785, "p95": 10.933237714925781, "p99": 12.049053382361341, "max": 18.464584019966424 }, "search_minus_ack_ms": { "min": -3.601333941332996, "p50": 0.18456298857927322, "p95": 4.411481326678768, "p99": 41.00824992987336, "max": 57.85062501672655 }, "regressions": 0, "confirmed_within_100ms": 0.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 } }, "total_operations": 300, "statuses": { "success": 300, "timeout": 0, "error": 0, "invalid_precondition": 0, "client_overload": 0 }, "query": { "count": 300, "success": 300, "ann_recall_at_10": 0.994, "ndcg_at_10": 0.7931268675492565, "latency_ms": { "min": 3.5553750349208713, "p50": 8.631625038105994, "p95": 47.87565842270851, "p99": 49.28177650202996, "max": 52.625374984927475 }, "dispatch_delay_ms": { "min": 0.06608304101973772, "p50": 0.9937084978446364, "p95": 2.156652143457905, "p99": 2.522090737475082, "max": 2.7384579880163074 } }, "diagnostics": { "schedule_delay_ms": { "min": 0.0075830146668565135, "p50": 0.3381454851476562, "p95": 0.6170705135446042, "p99": 0.6499608843580162, "max": 0.7941670482978225 }, "probe_dispatch_ms": { "min": 0.019749975763261318, "p50": 0.3955420106649399, "p95": 1.2997920857742429, "p99": 2.4704585317522283, "max": 7.557916920632124 }, "probe_service_ms": { "min": 0.9660000214353204, "p50": 5.6932090083137155, "p95": 48.72299998532981, "p99": 50.22934977896511, "max": 55.42620795313269 }, "write_service_ms": { "min": 2.9756249859929085, "p50": 7.998562476132065, "p95": 11.13231488270685, "p99": 13.152968341018994, "max": 23.14699999988079 } }, "actual_submission_rate": 10.00001085586903, "metadata": { "backend": "qdrant", "collection": "fb2_429e9a_042", "dataset": { "dataset": "nq", "source": "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/nq.zip", "source_sha256": "2553bf7bfbab47b1436ca00a34bce57320e18e611fd00999a2a3a1b4714be752", "source_md5": "d4d3d2e48787a744b6f6e691ff534307", "records": 124000, "queries": 100, "relevant_documents_retained": 120, "seed": 20261028, "license": "CC-BY-SA-3.0", "license_source": "https://raw.githubusercontent.com/google-research-datasets/natural-questions/master/nq_open/README.md", "attribution": "BEIR and original dataset authors; Wikipedia contributors for NQ, SciFact authors and cited article authors for SciFact.", "transformation": "200-word title+text prefix; seeded reservoir of documents with relevant documents for selected test queries retained at front. Simulated mutations; no historical replay.", "model": "sentence-transformers/all-MiniLM-L6-v2", "model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "dimensions": 384, "normalized": true, "max_tokens": 256, "device": "mps", "files": { "corpus.jsonl": "9dc487d785e6ee493eccaf764daa3bf8720a8f4fc8c406246065e1ad33a91f5d", "queries.json": "5603f52876ada5d36118b65dea490f1d43900bc159e1804e637796eb9115e999", "vectors.npy": "cb7421eb6ca5964231403f0a922843088f6fd0e01cbf67f2a1076cc7101005db", "query_vectors.npy": "5e666f8d966d8db6b0fd93d4963234d2dffa0ee3910b8fad069a8ee201e8b5d4", "UPSTREAM-LICENSE.md": "8b675521da3fa142ac6bda38c9bcfcc8cff0e2d6128be53a062ed55fb53f7b87" } }, "base_count": 100000, "profile": { "dataset": "nq", "base_count": 100000, "rate": 10, "query_rate": 10, "arrival": "steady" }, "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "preloaded_mutable": 200, "started_utc": "2026-10-29T03:28:23Z", "setup_s": 43.28050258301664, "backend_before": { "version": "1.20.0", "collection": { "status": "green", "optimizer_status": "ok", "indexed_vectors_count": 100200, "points_count": 100200, "segments_count": 2, "config": { "params": { "vectors": { "size": 384, "distance": "Cosine" }, "shard_number": 1, "replication_factor": 1, "write_consistency_factor": 1, "on_disk_payload": true }, "hnsw_config": { "m": 16, "ef_construct": 128, "full_scan_threshold": 10, "max_indexing_threads": 0, "on_disk": false }, "optimizer_config": { "deleted_threshold": 0.2, "vacuum_min_vector_number": 1000, "default_segment_number": 0, "max_segment_size": null, "memmap_threshold": null, "indexing_threshold": 1024, "flush_interval_sec": 5, "max_optimization_threads": null, "prevent_unoptimized": null }, "wal_config": { "wal_capacity_mb": 32, "wal_segments_ahead": 0, "wal_retain_closed": 1 }, "quantization_config": null }, "payload_schema": { "cohort": { "data_type": "keyword", "points": 100200 } }, "update_queue": { "length": 0 }, "created_at": "2026-10-29T03:28:23.809Z" }, "ack": "wait=true", "transport": "HTTP REST, pooled httpx" }, "count_before": 100200, "idle_duration_s": 10.0, "idle_queries": { "count": 100, "ann_recall": 0.9940000000000001, "ndcg": 0.7931268675492565, "latency_ms": { "min": 2.887125010602176, "p50": 10.269103979226202, "p95": 13.145056326175109, "p99": 17.536924912128747, "max": 46.68083298020065 } }, "exact_ndcg": 0.7931268675492565, "resources_before": { "usage_usec": 539599121, "user_usec": 512533815, "system_usec": 27065306, "nice_usec": 405136000, "nr_periods": 9446, "nr_throttled": 965, "throttled_usec": 3627339, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 630779904, "pids": 25 }, "resources_after": { "usage_usec": 546451375, "user_usec": 518603665, "system_usec": 27847710, "nice_usec": 405136000, "nr_periods": 9748, "nr_throttled": 965, "throttled_usec": 3627339, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 615575552, "pids": 32 }, "count_after": 100200, "backend_after": { "version": "1.20.0", "collection": { "status": "green", "optimizer_status": "ok", "indexed_vectors_count": 100200, "points_count": 100200, "segments_count": 2, "config": { "params": { "vectors": { "size": 384, "distance": "Cosine" }, "shard_number": 1, "replication_factor": 1, "write_consistency_factor": 1, "on_disk_payload": true }, "hnsw_config": { "m": 16, "ef_construct": 128, "full_scan_threshold": 10, "max_indexing_threads": 0, "on_disk": false }, "optimizer_config": { "deleted_threshold": 0.2, "vacuum_min_vector_number": 1000, "default_segment_number": 0, "max_segment_size": null, "memmap_threshold": null, "indexing_threshold": 1024, "flush_interval_sec": 5, "max_optimization_threads": null, "prevent_unoptimized": null }, "wal_config": { "wal_capacity_mb": 32, "wal_segments_ahead": 0, "wal_retain_closed": 1 }, "quantization_config": null }, "payload_schema": { "cohort": { "data_type": "keyword", "points": 100200 } }, "update_queue": { "length": 0 }, "created_at": "2026-10-29T03:28:23.809Z" }, "ack": "wait=true", "transport": "HTTP REST, pooled httpx" } }, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "eligible": true, "protocol": "mixed-mutations-v2" }, "examples": [ { "event": 0, "operation": "insert", "doc_id": "mutation-000000", "scheduled_s": 0.0, "submitted_s": 0.0002635420532897115, "schedule_delay_ms": 0.2635420532897115, "precondition": true, "status": "success", "ack_ms": 7.213208009488881, "write_started_ms": 1.5246659750118852, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 14.983583008870482, "stable_since_ms": 14.983583008870482, "confirmed_ms": 69.53562493436038, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 61.56362500041723, "stable_since_ms": 61.56362500041723, "confirmed_ms": 203.29337497241795, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 61.56362500041723, "confirmed_ms": 203.29337497241795, "search_minus_ack_ms": 54.35041699092835 }, { "event": 1, "operation": "update", "doc_id": "mutation-000001", "scheduled_s": 0.1, "submitted_s": 0.10014870797749609, "schedule_delay_ms": 0.148707977496082, "precondition": true, "status": "success", "ack_ms": 9.67066700104624, "write_started_ms": 0.6074589909985662, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 17.196041997522116, "stable_since_ms": 17.196041997522116, "confirmed_ms": 72.62887503020465, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 65.10212505236268, "stable_since_ms": 65.10212505236268, "confirmed_ms": 208.81866698618978, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 65.10212505236268, "confirmed_ms": 208.81866698618978, "search_minus_ack_ms": 55.43145805131644 }, { "event": 2, "operation": "delete", "doc_id": "mutation-000002", "scheduled_s": 0.2, "submitted_s": 0.20055379206314683, "schedule_delay_ms": 0.5537920631468185, "precondition": true, "status": "success", "ack_ms": 8.504915982484818, "write_started_ms": 0.4775408888235688, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 17.931832931935787, "stable_since_ms": 17.931832931935787, "confirmed_ms": 72.45237496681511, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 8.715000003576279, "stable_since_ms": 8.715000003576279, "confirmed_ms": 151.17545798420906, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 17.931832931935787, "confirmed_ms": 151.17545798420906, "search_minus_ack_ms": 0.21008402109146118 } ] }, { "run": "fb2_429e9a_043", "trial": 2, "backend": "qdrant", "scenario": "nq-100k", "status": "complete", "eligible": true, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "summary": { "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "wall_s": 30.06056025007274, "operations": { "insert": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 16.38404105324298, "p50": 68.79281293367967, "p95": 72.88419574615546, "p99": 109.75270831026138, "max": 112.97845805529505 }, "confirmed_ms": { "min": 109.02949993032962, "p50": 216.4751044474542, "p95": 225.9419329930097, "p99": 254.36550775659273, "max": 259.1264999937266 }, "ack_ms": { "min": 4.882332985289395, "p50": 9.52639599563554, "p95": 12.739377457182853, "p99": 17.472784990677642, "max": 23.441000026650727 }, "search_minus_ack_ms": { "min": 1.5625420492142439, "p50": 58.86849999660626, "p95": 62.873173208208755, "p99": 100.64932637731546, "max": 103.64341700915247 }, "regressions": 0, "confirmed_within_100ms": 0.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "update": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 17.939125071279705, "p50": 68.19702149368823, "p95": 71.59317740588449, "p99": 88.19330224883754, "max": 110.67554098553956 }, "confirmed_ms": { "min": 153.0777910957113, "p50": 215.8417915343307, "p95": 222.3006479733158, "p99": 233.8325449218975, "max": 258.72312497813255 }, "ack_ms": { "min": 4.373415955342352, "p50": 10.723750048782676, "p95": 12.836958945263177, "p99": 17.969227352878082, "max": 18.824916915036738 }, "search_minus_ack_ms": { "min": 1.6132090240716934, "p50": 57.06414597807452, "p95": 59.47632297175005, "p99": 70.28750240569948, "max": 98.19337504450232 }, "regressions": 0, "confirmed_within_100ms": 0.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "delete": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 13.54575005825609, "p50": 18.182146013714373, "p95": 21.42768147168681, "p99": 22.428895842749643, "max": 25.80112498253584 }, "confirmed_ms": { "min": 105.9537089895457, "p50": 156.34291694732383, "p95": 160.63899855944328, "p99": 166.93477054475815, "max": 182.4592079501599 }, "ack_ms": { "min": 3.5266249906271696, "p50": 9.453770471736789, "p95": 11.70656451722607, "p99": 12.87638541078196, "max": 14.845042023807764 }, "search_minus_ack_ms": { "min": -0.6680419901385903, "p50": 0.1881454954855144, "p95": 3.219572879606858, "p99": 4.7151230357122, "max": 10.956082958728075 }, "regressions": 0, "confirmed_within_100ms": 0.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 } }, "total_operations": 300, "statuses": { "success": 300, "timeout": 0, "error": 0, "invalid_precondition": 0, "client_overload": 0 }, "query": { "count": 300, "success": 300, "ann_recall_at_10": 0.985, "ndcg_at_10": 0.7931268675492565, "latency_ms": { "min": 3.043374978005886, "p50": 9.374458517413586, "p95": 12.250166048761457, "p99": 48.96238718298263, "max": 52.39387508481741 }, "dispatch_delay_ms": { "min": 0.18204201478511095, "p50": 1.4640000299550593, "p95": 2.3569643497467045, "p99": 2.5588567613158366, "max": 6.351000047288835 } }, "diagnostics": { "schedule_delay_ms": { "min": 0.015375018119989647, "p50": 0.33268756233084673, "p95": 0.6294330360833267, "p99": 0.8752101690042756, "max": 4.728250019252478 }, "probe_dispatch_ms": { "min": 0.0265840208157897, "p50": 0.3866039915010333, "p95": 1.866393629461525, "p99": 2.654223251156509, "max": 8.51370906457305 }, "probe_service_ms": { "min": 1.0320000583305955, "p50": 6.586542003788054, "p95": 49.40036253537982, "p99": 51.04552873293871, "max": 74.04741598293185 }, "write_service_ms": { "min": 3.3468749606981874, "p50": 9.341958502773196, "p95": 12.108337553218007, "p99": 17.06455171341076, "max": 22.774667013436556 } }, "actual_submission_rate": 9.999881077353944, "metadata": { "backend": "qdrant", "collection": "fb2_429e9a_043", "dataset": { "dataset": "nq", "source": "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/nq.zip", "source_sha256": "2553bf7bfbab47b1436ca00a34bce57320e18e611fd00999a2a3a1b4714be752", "source_md5": "d4d3d2e48787a744b6f6e691ff534307", "records": 124000, "queries": 100, "relevant_documents_retained": 120, "seed": 20261028, "license": "CC-BY-SA-3.0", "license_source": "https://raw.githubusercontent.com/google-research-datasets/natural-questions/master/nq_open/README.md", "attribution": "BEIR and original dataset authors; Wikipedia contributors for NQ, SciFact authors and cited article authors for SciFact.", "transformation": "200-word title+text prefix; seeded reservoir of documents with relevant documents for selected test queries retained at front. Simulated mutations; no historical replay.", "model": "sentence-transformers/all-MiniLM-L6-v2", "model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "dimensions": 384, "normalized": true, "max_tokens": 256, "device": "mps", "files": { "corpus.jsonl": "9dc487d785e6ee493eccaf764daa3bf8720a8f4fc8c406246065e1ad33a91f5d", "queries.json": "5603f52876ada5d36118b65dea490f1d43900bc159e1804e637796eb9115e999", "vectors.npy": "cb7421eb6ca5964231403f0a922843088f6fd0e01cbf67f2a1076cc7101005db", "query_vectors.npy": "5e666f8d966d8db6b0fd93d4963234d2dffa0ee3910b8fad069a8ee201e8b5d4", "UPSTREAM-LICENSE.md": "8b675521da3fa142ac6bda38c9bcfcc8cff0e2d6128be53a062ed55fb53f7b87" } }, "base_count": 100000, "profile": { "dataset": "nq", "base_count": 100000, "rate": 10, "query_rate": 10, "arrival": "steady" }, "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "preloaded_mutable": 200, "started_utc": "2026-10-29T03:29:53Z", "setup_s": 44.10431374993641, "backend_before": { "version": "1.20.0", "collection": { "status": "green", "optimizer_status": "ok", "indexed_vectors_count": 100200, "points_count": 100200, "segments_count": 2, "config": { "params": { "vectors": { "size": 384, "distance": "Cosine" }, "shard_number": 1, "replication_factor": 1, "write_consistency_factor": 1, "on_disk_payload": true }, "hnsw_config": { "m": 16, "ef_construct": 128, "full_scan_threshold": 10, "max_indexing_threads": 0, "on_disk": false }, "optimizer_config": { "deleted_threshold": 0.2, "vacuum_min_vector_number": 1000, "default_segment_number": 0, "max_segment_size": null, "memmap_threshold": null, "indexing_threshold": 1024, "flush_interval_sec": 5, "max_optimization_threads": null, "prevent_unoptimized": null }, "wal_config": { "wal_capacity_mb": 32, "wal_segments_ahead": 0, "wal_retain_closed": 1 }, "quantization_config": null }, "payload_schema": { "cohort": { "data_type": "keyword", "points": 100200 } }, "update_queue": { "length": 0 }, "created_at": "2026-10-29T03:29:53.163Z" }, "ack": "wait=true", "transport": "HTTP REST, pooled httpx" }, "count_before": 100200, "idle_duration_s": 10.0, "idle_queries": { "count": 100, "ann_recall": 0.985, "ndcg": 0.7931268675492565, "latency_ms": { "min": 4.430584027431905, "p50": 10.798436996992677, "p95": 12.67668814980425, "p99": 18.654187453212362, "max": 46.611332916654646 } }, "exact_ndcg": 0.7931268675492565, "resources_before": { "usage_usec": 611860041, "user_usec": 581696552, "system_usec": 30163488, "nice_usec": 463859000, "nr_periods": 10326, "nr_throttled": 1106, "throttled_usec": 4064565, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 641617920, "pids": 29 }, "resources_after": { "usage_usec": 618925001, "user_usec": 588020162, "system_usec": 30904839, "nice_usec": 463859000, "nr_periods": 10628, "nr_throttled": 1106, "throttled_usec": 4064565, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 623820800, "pids": 32 }, "count_after": 100200, "backend_after": { "version": "1.20.0", "collection": { "status": "green", "optimizer_status": "ok", "indexed_vectors_count": 100200, "points_count": 100200, "segments_count": 2, "config": { "params": { "vectors": { "size": 384, "distance": "Cosine" }, "shard_number": 1, "replication_factor": 1, "write_consistency_factor": 1, "on_disk_payload": true }, "hnsw_config": { "m": 16, "ef_construct": 128, "full_scan_threshold": 10, "max_indexing_threads": 0, "on_disk": false }, "optimizer_config": { "deleted_threshold": 0.2, "vacuum_min_vector_number": 1000, "default_segment_number": 0, "max_segment_size": null, "memmap_threshold": null, "indexing_threshold": 1024, "flush_interval_sec": 5, "max_optimization_threads": null, "prevent_unoptimized": null }, "wal_config": { "wal_capacity_mb": 32, "wal_segments_ahead": 0, "wal_retain_closed": 1 }, "quantization_config": null }, "payload_schema": { "cohort": { "data_type": "keyword", "points": 100200 } }, "update_queue": { "length": 0 }, "created_at": "2026-10-29T03:29:53.163Z" }, "ack": "wait=true", "transport": "HTTP REST, pooled httpx" } }, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "eligible": true, "protocol": "mixed-mutations-v2" }, "examples": [ { "event": 0, "operation": "insert", "doc_id": "mutation-000000", "scheduled_s": 0.0, "submitted_s": 0.00025154207833111286, "schedule_delay_ms": 0.25154207833111286, "precondition": true, "status": "success", "ack_ms": 6.911915959790349, "write_started_ms": 1.614999957382679, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 14.67691594734788, "stable_since_ms": 14.67691594734788, "confirmed_ms": 68.46166600007564, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 60.26474991813302, "stable_since_ms": 60.26474991813302, "confirmed_ms": 203.37729097809643, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 60.26474991813302, "confirmed_ms": 203.37729097809643, "search_minus_ack_ms": 53.35283395834267 }, { "event": 1, "operation": "update", "doc_id": "mutation-000001", "scheduled_s": 0.1, "submitted_s": 0.10031387500930578, "schedule_delay_ms": 0.3138750093057696, "precondition": true, "status": "success", "ack_ms": 10.73375006671995, "write_started_ms": 1.1867920402437449, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 20.519957994110882, "stable_since_ms": 20.519957994110882, "confirmed_ms": 77.47958309482783, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 65.23587508127093, "stable_since_ms": 65.23587508127093, "confirmed_ms": 212.77637500315905, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 65.23587508127093, "confirmed_ms": 212.77637500315905, "search_minus_ack_ms": 54.502125014550984 }, { "event": 2, "operation": "delete", "doc_id": "mutation-000002", "scheduled_s": 0.2, "submitted_s": 0.20037450001109391, "schedule_delay_ms": 0.3745000110939034, "precondition": true, "status": "success", "ack_ms": 9.12395806517452, "write_started_ms": 1.2695420300588012, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 18.214917043223977, "stable_since_ms": 18.214917043223977, "confirmed_ms": 73.82183300796896, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 9.322541998699307, "stable_since_ms": 9.322541998699307, "confirmed_ms": 156.2312920577824, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 18.214917043223977, "confirmed_ms": 156.2312920577824, "search_minus_ack_ms": 0.19858393352478743 } ] }, { "run": "fb2_429e9a_044", "trial": 3, "backend": "weaviate", "scenario": "nq-10k", "status": "complete", "eligible": true, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "summary": { "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "wall_s": 30.000890791066922, "operations": { "insert": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 5.356875015422702, "p50": 17.56004203343764, "p95": 30.209300003480163, "p99": 32.43947276961993, "max": 38.929499918594956 }, "confirmed_ms": { "min": 66.13645795732737, "p50": 80.66006301669404, "p95": 103.44337745918892, "p99": 107.51021540723742, "max": 110.7490420108661 }, "ack_ms": { "min": 3.3492500660941005, "p50": 9.375062538310885, "p95": 13.286822225200012, "p99": 14.45705904625357, "max": 16.141749918460846 }, "search_minus_ack_ms": { "min": 0.1431250711902976, "p50": 3.3990624942816794, "p95": 17.746095539769158, "p99": 20.205411531496832, "max": 27.650457923300564 }, "regressions": 0, "confirmed_within_100ms": 0.93, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "update": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 6.537292036227882, "p50": 17.30864599812776, "p95": 30.610112578142434, "p99": 34.177761832252166, "max": 38.9145000372082 }, "confirmed_ms": { "min": 66.30254094488919, "p50": 80.80577047076076, "p95": 99.55286015756428, "p99": 103.62972183851525, "max": 104.80662505142391 }, "ack_ms": { "min": 4.36045799870044, "p50": 9.491499979048967, "p95": 14.030698250280693, "p99": 15.846479698084307, "max": 18.150374991819263 }, "search_minus_ack_ms": { "min": 0.1282920129597187, "p50": 3.2953964546322823, "p95": 17.71248746081255, "p99": 23.106603248743347, "max": 24.622416007332504 }, "regressions": 0, "confirmed_within_100ms": 0.96, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "delete": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 6.4693751046434045, "p50": 16.088604519609362, "p95": 21.343398588942357, "p99": 32.464151143794886, "max": 34.06716708559543 }, "confirmed_ms": { "min": 66.70824997127056, "p50": 78.16222950350493, "p95": 86.32707701181062, "p99": 102.60162339429371, "max": 103.73995907139033 }, "ack_ms": { "min": 2.454417059198022, "p50": 5.436374980490655, "p95": 9.95846518781036, "p99": 12.82264750916511, "max": 13.010499998927116 }, "search_minus_ack_ms": { "min": 0.09545800276100636, "p50": 4.92870801826939, "p95": 12.863806332461527, "p99": 19.469634054694342, "max": 22.654958069324493 }, "regressions": 0, "confirmed_within_100ms": 0.98, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 } }, "total_operations": 300, "statuses": { "success": 300, "timeout": 0, "error": 0, "invalid_precondition": 0, "client_overload": 0 }, "query": { "count": 300, "success": 300, "ann_recall_at_10": 0.996, "ndcg_at_10": 0.9481505328688746, "latency_ms": { "min": 4.033459001220763, "p50": 10.185562539845705, "p95": 14.892701461212711, "p99": 20.772314439527683, "max": 23.50624999962747 }, "dispatch_delay_ms": { "min": 0.08479109965264797, "p50": 1.2313540210016072, "p95": 2.3528289515525103, "p99": 2.719973856583235, "max": 3.3636250300332904 } }, "diagnostics": { "schedule_delay_ms": { "min": 0.01083305105564758, "p50": 0.32745803473499535, "p95": 0.6152160069905932, "p99": 0.6454348403494947, "max": 0.6712910952053619 }, "probe_dispatch_ms": { "min": 0.024416949599981308, "p50": 0.41029200656339526, "p95": 2.0764164510183036, "p99": 2.7824393881019205, "max": 3.8958339719101787 }, "probe_service_ms": { "min": 0.6137919845059514, "p50": 5.522916559129953, "p95": 10.880166490096599, "p99": 13.52921627694741, "max": 35.039166919887066 }, "write_service_ms": { "min": 2.203584066592157, "p50": 7.717479020357132, "p95": 12.414508051006125, "p99": 14.134622107958414, "max": 17.961915931664407 } }, "actual_submission_rate": 10.00001762814526, "metadata": { "backend": "weaviate", "collection": "fb2_429e9a_044", "dataset": { "dataset": "nq", "source": "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/nq.zip", "source_sha256": "2553bf7bfbab47b1436ca00a34bce57320e18e611fd00999a2a3a1b4714be752", "source_md5": "d4d3d2e48787a744b6f6e691ff534307", "records": 124000, "queries": 100, "relevant_documents_retained": 120, "seed": 20261028, "license": "CC-BY-SA-3.0", "license_source": "https://raw.githubusercontent.com/google-research-datasets/natural-questions/master/nq_open/README.md", "attribution": "BEIR and original dataset authors; Wikipedia contributors for NQ, SciFact authors and cited article authors for SciFact.", "transformation": "200-word title+text prefix; seeded reservoir of documents with relevant documents for selected test queries retained at front. Simulated mutations; no historical replay.", "model": "sentence-transformers/all-MiniLM-L6-v2", "model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "dimensions": 384, "normalized": true, "max_tokens": 256, "device": "mps", "files": { "corpus.jsonl": "9dc487d785e6ee493eccaf764daa3bf8720a8f4fc8c406246065e1ad33a91f5d", "queries.json": "5603f52876ada5d36118b65dea490f1d43900bc159e1804e637796eb9115e999", "vectors.npy": "cb7421eb6ca5964231403f0a922843088f6fd0e01cbf67f2a1076cc7101005db", "query_vectors.npy": "5e666f8d966d8db6b0fd93d4963234d2dffa0ee3910b8fad069a8ee201e8b5d4", "UPSTREAM-LICENSE.md": "8b675521da3fa142ac6bda38c9bcfcc8cff0e2d6128be53a062ed55fb53f7b87" } }, "base_count": 10000, "profile": { "dataset": "nq", "base_count": 10000, "rate": 10, "query_rate": 10, "arrival": "steady" }, "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "preloaded_mutable": 200, "started_utc": "2026-10-29T15:40:34Z", "setup_s": 2.5539931670064107, "backend_before": { "version": "1.40.3", "schema": { "class": "Ffb2429e9a044", "invertedIndexConfig": { "bm25": { "b": 0.75, "k1": 1.2 }, "cleanupIntervalSeconds": 60, "stopwords": { "additions": null, "preset": "en", "removals": null }, "usingBlockMaxWAND": true }, "multiTenancyConfig": { "autoTenantActivation": false, "autoTenantCreation": false, "enabled": false }, "properties": [ { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "doc_id", "tokenization": "field" }, { "dataType": [ "int" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": false, "name": "version" }, { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "cohort", "tokenization": "field" }, { "dataType": [ "text" ], "indexFilterable": false, "indexRangeFilters": false, "indexSearchable": false, "name": "text", "tokenization": "word" } ], "shardingConfig": { "actualCount": 1, "actualVirtualCount": 128, "desiredCount": 1, "desiredVirtualCount": 128, "function": "murmur3", "key": "_id", "strategy": "hash", "virtualPerPhysical": 128 }, "vectorIndexConfig": { "bq": { "enabled": false }, "cleanupIntervalSeconds": 300, "distance": "cosine", "dynamicEfFactor": 8, "dynamicEfMax": 500, "dynamicEfMin": 100, "ef": 128, "efConstruction": 128, "filterStrategy": "acorn", "flatSearchCutoff": 0, "maxConnections": 16, "multivector": { "aggregation": "maxSim", "enabled": false, "muvera": { "dprojections": 16, "enabled": false, "ksim": 4, "repetitions": 10 } }, "pq": { "bitCompression": false, "centroids": 256, "enabled": false, "encoder": { "distribution": "log-normal", "type": "kmeans" }, "segments": 0, "trainingLimit": 100000 }, "rq": { "bits": 8, "centering": false, "enabled": false, "rescoreLimit": 20, "trainingLimit": 10000 }, "skip": false, "skipDefaultQuantization": false, "sq": { "enabled": false, "rescoreLimit": 20, "trainingLimit": 100000 }, "trackDefaultQuantization": false, "vectorCacheMaxObjects": 1000000000000 }, "vectorIndexType": "hnsw", "vectorizer": "none", "replicationConfig": { "deletionStrategy": "TimeBasedResolution", "factor": 1, "asyncEnabled": false } }, "shards": [ { "name": "Vu3UChUdv4yl", "per_node_status": { "node1": "READY" }, "status": "READY", "vectorQueueSize": 0 } ], "ack": "synchronous indexing, object PUT/POST/DELETE", "transport": "HTTP REST writes + GraphQL search, pooled httpx" }, "count_before": 10200, "idle_duration_s": 10.0, "idle_queries": { "count": 100, "ann_recall": 0.996, "ndcg": 0.9481505328688745, "latency_ms": { "min": 3.8638749392703176, "p50": 10.016874992288649, "p95": 12.074016651604323, "p99": 13.03879068000243, "max": 14.734124997630715 } }, "exact_ndcg": 0.9481505328688745, "resources_before": { "usage_usec": 5932111, "user_usec": 5684593, "system_usec": 247517, "nice_usec": 0, "nr_periods": 444, "nr_throttled": 4, "throttled_usec": 21159, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 295976960, "pids": 10 }, "resources_after": { "usage_usec": 14608406, "user_usec": 13935390, "system_usec": 673016, "nice_usec": 0, "nr_periods": 745, "nr_throttled": 4, "throttled_usec": 21159, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 306544640, "pids": 10 }, "count_after": 10200, "backend_after": { "version": "1.40.3", "schema": { "class": "Ffb2429e9a044", "invertedIndexConfig": { "bm25": { "b": 0.75, "k1": 1.2 }, "cleanupIntervalSeconds": 60, "stopwords": { "additions": null, "preset": "en", "removals": null }, "usingBlockMaxWAND": true }, "multiTenancyConfig": { "autoTenantActivation": false, "autoTenantCreation": false, "enabled": false }, "properties": [ { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "doc_id", "tokenization": "field" }, { "dataType": [ "int" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": false, "name": "version" }, { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "cohort", "tokenization": "field" }, { "dataType": [ "text" ], "indexFilterable": false, "indexRangeFilters": false, "indexSearchable": false, "name": "text", "tokenization": "word" } ], "shardingConfig": { "actualCount": 1, "actualVirtualCount": 128, "desiredCount": 1, "desiredVirtualCount": 128, "function": "murmur3", "key": "_id", "strategy": "hash", "virtualPerPhysical": 128 }, "vectorIndexConfig": { "bq": { "enabled": false }, "cleanupIntervalSeconds": 300, "distance": "cosine", "dynamicEfFactor": 8, "dynamicEfMax": 500, "dynamicEfMin": 100, "ef": 128, "efConstruction": 128, "filterStrategy": "acorn", "flatSearchCutoff": 0, "maxConnections": 16, "multivector": { "aggregation": "maxSim", "enabled": false, "muvera": { "dprojections": 16, "enabled": false, "ksim": 4, "repetitions": 10 } }, "pq": { "bitCompression": false, "centroids": 256, "enabled": false, "encoder": { "distribution": "log-normal", "type": "kmeans" }, "segments": 0, "trainingLimit": 100000 }, "rq": { "bits": 8, "centering": false, "enabled": false, "rescoreLimit": 20, "trainingLimit": 10000 }, "skip": false, "skipDefaultQuantization": false, "sq": { "enabled": false, "rescoreLimit": 20, "trainingLimit": 100000 }, "trackDefaultQuantization": false, "vectorCacheMaxObjects": 1000000000000 }, "vectorIndexType": "hnsw", "vectorizer": "none", "replicationConfig": { "deletionStrategy": "TimeBasedResolution", "factor": 1, "asyncEnabled": false } }, "shards": [ { "name": "Vu3UChUdv4yl", "per_node_status": { "node1": "READY" }, "status": "READY", "vectorQueueSize": 0 } ], "ack": "synchronous indexing, object PUT/POST/DELETE", "transport": "HTTP REST writes + GraphQL search, pooled httpx" } }, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "eligible": true, "protocol": "mixed-mutations-v2" }, "examples": [ { "event": 0, "operation": "insert", "doc_id": "mutation-000000", "scheduled_s": 0.0, "submitted_s": 0.00017033307813107967, "schedule_delay_ms": 0.17033307813107967, "precondition": true, "status": "success", "ack_ms": 6.127542001195252, "write_started_ms": 1.1067079612985253, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 13.310750015079975, "stable_since_ms": 13.310750015079975, "confirmed_ms": 65.84225001279265, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 18.638417008332908, "stable_since_ms": 18.638417008332908, "confirmed_ms": 75.11924998834729, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 18.638417008332908, "confirmed_ms": 75.11924998834729, "search_minus_ack_ms": 12.510875007137656 }, { "event": 1, "operation": "update", "doc_id": "mutation-000001", "scheduled_s": 0.1, "submitted_s": 0.10061449999921024, "schedule_delay_ms": 0.6144999992102329, "precondition": true, "status": "success", "ack_ms": 4.916083067655563, "write_started_ms": 0.4069580463692546, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 13.539500068873167, "stable_since_ms": 13.539500068873167, "confirmed_ms": 66.65275001432747, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 17.77287502773106, "stable_since_ms": 17.77287502773106, "confirmed_ms": 74.52583301346749, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 17.77287502773106, "confirmed_ms": 74.52583301346749, "search_minus_ack_ms": 12.856791960075498 }, { "event": 2, "operation": "delete", "doc_id": "mutation-000002", "scheduled_s": 0.2, "submitted_s": 0.20060479105450213, "schedule_delay_ms": 0.6047910545021185, "precondition": true, "status": "success", "ack_ms": 5.483041983097792, "write_started_ms": 0.2632089890539646, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 5.6237500393763185, "stable_since_ms": 5.6237500393763185, "confirmed_ms": 60.20979199092835, "consecutive": 3, "regressions": 0, "requests": 3 }, "search": { "first_ms": 7.191624958068132, "stable_since_ms": 7.191624958068132, "confirmed_ms": 69.4190840004012, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 7.191624958068132, "confirmed_ms": 69.4190840004012, "search_minus_ack_ms": 1.7085829749703407 } ] }, { "run": "fb2_429e9a_045", "trial": 1, "backend": "qdrant", "scenario": "scifact-small", "status": "complete", "eligible": true, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "summary": { "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "wall_s": 30.051679500029422, "operations": { "insert": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 19.242167007178068, "p50": 64.60339552722871, "p95": 104.77418363443576, "p99": 105.64234164310619, "max": 106.77754099015146 }, "confirmed_ms": { "min": 109.02687499765307, "p50": 207.94818754075095, "p95": 212.01495390268974, "p99": 214.413339199964, "max": 226.5249160118401 }, "ack_ms": { "min": 5.123750073835254, "p50": 9.289874986279756, "p95": 11.604839173378423, "p99": 24.3977079575416, "max": 29.90041591692716 }, "search_minus_ack_ms": { "min": 1.2318750377744436, "p50": 55.15468801604584, "p95": 95.09928484330885, "p99": 97.37339352373965, "max": 97.91566606145352 }, "regressions": 0, "confirmed_within_100ms": 0.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "update": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 16.393917030654848, "p50": 64.09275002079085, "p95": 67.50240036053583, "p99": 101.19115042383784, "max": 104.11416704300791 }, "confirmed_ms": { "min": 109.70687493681908, "p50": 207.9897709772922, "p95": 212.14049130794592, "p99": 230.2354033698794, "max": 241.51600000914186 }, "ack_ms": { "min": 5.151999997906387, "p50": 8.614937483798712, "p95": 10.26425207965076, "p99": 11.00931206252426, "max": 11.386708007194102 }, "search_minus_ack_ms": { "min": 0.7760830922052264, "p50": 55.38508348399773, "p95": 58.30771599430591, "p99": 94.94291812879966, "max": 97.59137500077486 }, "regressions": 0, "confirmed_within_100ms": 0.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "delete": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 9.871916030533612, "p50": 19.041812454815954, "p95": 59.58017531083897, "p99": 63.633955932455144, "max": 64.53712505754083 }, "confirmed_ms": { "min": 105.1258749794215, "p50": 111.14320845808834, "p95": 202.821593999397, "p99": 206.7007935687434, "max": 209.1265410417691 }, "ack_ms": { "min": 3.5596659872680902, "p50": 7.28118751430884, "p95": 9.516127430833876, "p99": 11.554818846052536, "max": 15.971290995366871 }, "search_minus_ack_ms": { "min": -3.0763751128688455, "p50": 0.23612548829987645, "p95": 53.21290066931397, "p99": 56.06563785113395, "max": 57.08987498655915 }, "regressions": 0, "confirmed_within_100ms": 0.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 } }, "total_operations": 300, "statuses": { "success": 300, "timeout": 0, "error": 0, "invalid_precondition": 0, "client_overload": 0 }, "query": { "count": 300, "success": 300, "ann_recall_at_10": 1.0, "ndcg_at_10": 0.6567157762068453, "latency_ms": { "min": 3.6445000441744924, "p50": 7.254082942381501, "p95": 9.655477327760309, "p99": 48.10023012803867, "max": 49.451541039161384 }, "dispatch_delay_ms": { "min": 0.18400012049824, "p50": 1.409833028446883, "p95": 2.126078598666936, "p99": 2.3496976296883068, "max": 2.8561250073835254 } }, "diagnostics": { "schedule_delay_ms": { "min": 0.00700005330145359, "p50": 0.33754103351402165, "p95": 0.6203583546442459, "p99": 0.6455905532944151, "max": 0.6682910257946872 }, "probe_dispatch_ms": { "min": 0.02975005190819502, "p50": 0.3957089502364397, "p95": 1.6885410295799375, "p99": 2.2296669194474816, "max": 3.1229578889906406 }, "probe_service_ms": { "min": 1.2201250065118074, "p50": 5.794125027023256, "p95": 47.144750016741455, "p99": 48.37141698226333, "max": 69.22449998091906 }, "write_service_ms": { "min": 3.406457952223718, "p50": 7.8642499865964055, "p95": 10.250443132827058, "p99": 12.699056168785294, "max": 29.23437498975545 } }, "actual_submission_rate": 9.999988628797905, "metadata": { "backend": "qdrant", "collection": "fb2_429e9a_045", "dataset": { "dataset": "scifact", "source": "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/scifact.zip", "source_sha256": "536e14446a0ba56ed1398ab1055f39fe852686ecad24a6306c80c490fa8e0165", "source_md5": "5f7d1de60b170fc8027bb7898e2efca1", "records": 5183, "queries": 100, "relevant_documents_retained": 107, "seed": 20261028, "license": "Corpus: ODC-By-1.0; claims/evidence: CC-BY-4.0", "license_source": "https://raw.githubusercontent.com/allenai/scifact/master/LICENSE.md", "attribution": "BEIR and original dataset authors; Wikipedia contributors for NQ, SciFact authors and cited article authors for SciFact.", "transformation": "200-word title+text prefix; seeded reservoir of documents with relevant documents for selected test queries retained at front. Simulated mutations; no historical replay.", "model": "sentence-transformers/all-MiniLM-L6-v2", "model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "dimensions": 384, "normalized": true, "max_tokens": 256, "device": "mps", "files": { "corpus.jsonl": "189d58f273468352c9f73c6333c5120dcfb7ba1695984f073f0ebb6c100ed0cd", "queries.json": "c8b333c73353a8c4aeb7d6b63067c9a241aa5d39a0314f0c315ec63019ee9558", "vectors.npy": "b6b21404977151ce565e7d6d57d9931a4679e1ecf4aa44fcd9b29eb54955d09e", "query_vectors.npy": "02324eb7c3b3d2e2f0ec3208453e2ec0efd5d61d9fc845b1f8b7950e913f2730", "UPSTREAM-LICENSE.md": "56ee69bc639cbab8f0b2e4b67091ec5e9390aa99a1d65f0a35e4274686c9a806" } }, "base_count": 4000, "profile": { "dataset": "scifact", "base_count": 4000, "rate": 10, "query_rate": 10, "arrival": "steady" }, "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "preloaded_mutable": 200, "started_utc": "2026-10-29T15:41:21Z", "setup_s": 1.727110999985598, "backend_before": { "version": "1.20.0", "collection": { "status": "green", "optimizer_status": "ok", "indexed_vectors_count": 4200, "points_count": 4200, "segments_count": 2, "config": { "params": { "vectors": { "size": 384, "distance": "Cosine" }, "shard_number": 1, "replication_factor": 1, "write_consistency_factor": 1, "on_disk_payload": true }, "hnsw_config": { "m": 16, "ef_construct": 128, "full_scan_threshold": 10, "max_indexing_threads": 0, "on_disk": false }, "optimizer_config": { "deleted_threshold": 0.2, "vacuum_min_vector_number": 1000, "default_segment_number": 0, "max_segment_size": null, "memmap_threshold": null, "indexing_threshold": 1024, "flush_interval_sec": 5, "max_optimization_threads": null, "prevent_unoptimized": null }, "wal_config": { "wal_capacity_mb": 32, "wal_segments_ahead": 0, "wal_retain_closed": 1 }, "quantization_config": null }, "payload_schema": { "cohort": { "data_type": "keyword", "points": 4200 } }, "update_queue": { "length": 0 }, "created_at": "2026-10-29T15:41:21.367Z" }, "ack": "wait=true", "transport": "HTTP REST, pooled httpx" }, "count_before": 4200, "idle_duration_s": 10.0, "idle_queries": { "count": 100, "ann_recall": 1.0, "ndcg": 0.6567157762068453, "latency_ms": { "min": 3.623457974754274, "p50": 6.737646472174674, "p95": 7.985187938902527, "p99": 8.865417455090405, "max": 46.75770900212228 } }, "exact_ndcg": 0.6567157762068453, "resources_before": { "usage_usec": 1654932, "user_usec": 1334208, "system_usec": 320723, "nice_usec": 407000, "nr_periods": 231, "nr_throttled": 0, "throttled_usec": 0, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 292028416, "pids": 32 }, "resources_after": { "usage_usec": 6535576, "user_usec": 5498200, "system_usec": 1037376, "nice_usec": 407000, "nr_periods": 532, "nr_throttled": 0, "throttled_usec": 0, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 299266048, "pids": 32 }, "count_after": 4200, "backend_after": { "version": "1.20.0", "collection": { "status": "green", "optimizer_status": "ok", "indexed_vectors_count": 4200, "points_count": 4200, "segments_count": 2, "config": { "params": { "vectors": { "size": 384, "distance": "Cosine" }, "shard_number": 1, "replication_factor": 1, "write_consistency_factor": 1, "on_disk_payload": true }, "hnsw_config": { "m": 16, "ef_construct": 128, "full_scan_threshold": 10, "max_indexing_threads": 0, "on_disk": false }, "optimizer_config": { "deleted_threshold": 0.2, "vacuum_min_vector_number": 1000, "default_segment_number": 0, "max_segment_size": null, "memmap_threshold": null, "indexing_threshold": 1024, "flush_interval_sec": 5, "max_optimization_threads": null, "prevent_unoptimized": null }, "wal_config": { "wal_capacity_mb": 32, "wal_segments_ahead": 0, "wal_retain_closed": 1 }, "quantization_config": null }, "payload_schema": { "cohort": { "data_type": "keyword", "points": 4200 } }, "update_queue": { "length": 0 }, "created_at": "2026-10-29T15:41:21.367Z" }, "ack": "wait=true", "transport": "HTTP REST, pooled httpx" } }, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "eligible": true, "protocol": "mixed-mutations-v2" }, "examples": [ { "event": 0, "operation": "insert", "doc_id": "mutation-000000", "scheduled_s": 0.0, "submitted_s": 0.00020429107826203108, "schedule_delay_ms": 0.20429107826203108, "precondition": true, "status": "success", "ack_ms": 5.8611249551177025, "write_started_ms": 1.5005419263616204, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 14.239999931305647, "stable_since_ms": 14.239999931305647, "confirmed_ms": 68.8289589015767, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 58.08141699526459, "stable_since_ms": 58.08141699526459, "confirmed_ms": 201.78354193922132, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 58.08141699526459, "confirmed_ms": 201.78354193922132, "search_minus_ack_ms": 52.22029204014689 }, { "event": 1, "operation": "update", "doc_id": "mutation-000001", "scheduled_s": 0.1, "submitted_s": 0.10032000008504838, "schedule_delay_ms": 0.32000008504837196, "precondition": true, "status": "success", "ack_ms": 10.133457952179015, "write_started_ms": 1.1070409091189504, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 20.40537493303418, "stable_since_ms": 20.40537493303418, "confirmed_ms": 77.85458292346448, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 62.2434999095276, "stable_since_ms": 62.2434999095276, "confirmed_ms": 204.85466590616852, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 62.2434999095276, "confirmed_ms": 204.85466590616852, "search_minus_ack_ms": 52.110041957348585 }, { "event": 2, "operation": "delete", "doc_id": "mutation-000002", "scheduled_s": 0.2, "submitted_s": 0.20007050002459437, "schedule_delay_ms": 0.07050002459435545, "precondition": true, "status": "success", "ack_ms": 6.588916061446071, "write_started_ms": 1.2322909897193313, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 20.219707977958024, "stable_since_ms": 20.219707977958024, "confirmed_ms": 76.1675409739837, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 7.517582969740033, "stable_since_ms": 7.517582969740033, "confirmed_ms": 108.75037498772144, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 20.219707977958024, "confirmed_ms": 108.75037498772144, "search_minus_ack_ms": 0.9286669082939625 } ] }, { "run": "fb2_429e9a_046", "trial": 2, "backend": "weaviate", "scenario": "scifact-small", "status": "complete", "eligible": true, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "summary": { "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "wall_s": 30.00117699999828, "operations": { "insert": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 6.586041068658233, "p50": 17.222271068021655, "p95": 30.58800835860893, "p99": 32.77391239535065, "max": 35.446457914076746 }, "confirmed_ms": { "min": 64.97837498318404, "p50": 78.10908346436918, "p95": 98.49914374644868, "p99": 103.54546101065354, "max": 104.96887494809926 }, "ack_ms": { "min": 4.359000013209879, "p50": 8.749896078370512, "p95": 13.253747241105884, "p99": 14.675315497443084, "max": 15.238665975630283 }, "search_minus_ack_ms": { "min": -0.22995797917246819, "p50": 2.185916993767023, "p95": 18.41182296629995, "p99": 19.50806640204975, "max": 22.117541986517608 }, "regressions": 0, "confirmed_within_100ms": 0.96, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "update": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 6.668750080280006, "p50": 17.26849947590381, "p95": 29.957671550801024, "p99": 30.707655776059255, "max": 30.74366704095155 }, "confirmed_ms": { "min": 62.8623329102993, "p50": 77.66902050934732, "p95": 96.13994136452675, "p99": 98.62054578959942, "max": 99.39233399927616 }, "ack_ms": { "min": 4.105874919332564, "p50": 8.483625017106533, "p95": 13.209997647209093, "p99": 14.158912880811872, "max": 20.498666912317276 }, "search_minus_ack_ms": { "min": -0.42337493505328894, "p50": 2.7034995146095753, "p95": 17.750364384846762, "p99": 18.64355805562809, "max": 18.719500047154725 }, "regressions": 0, "confirmed_within_100ms": 1.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "delete": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 7.204875000752509, "p50": 16.792792011983693, "p95": 22.411120188189667, "p99": 26.794553366489723, "max": 29.831707943230867 }, "confirmed_ms": { "min": 67.28079193271697, "p50": 76.3279790407978, "p95": 83.23025389690882, "p99": 93.454233763041, "max": 98.05612498894334 }, "ack_ms": { "min": 2.8519589686766267, "p50": 5.295249982737005, "p95": 9.44761430728249, "p99": 11.228790879249587, "max": 14.04195895884186 }, "search_minus_ack_ms": { "min": -0.10258308611810207, "p50": 4.384603991638869, "p95": 12.831260060193012, "p99": 18.364528983365744, "max": 18.631332903169096 }, "regressions": 0, "confirmed_within_100ms": 1.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 } }, "total_operations": 300, "statuses": { "success": 300, "timeout": 0, "error": 0, "invalid_precondition": 0, "client_overload": 0 }, "query": { "count": 300, "success": 300, "ann_recall_at_10": 1.0, "ndcg_at_10": 0.6567157762068453, "latency_ms": { "min": 3.6300410283729434, "p50": 9.45683300960809, "p95": 13.895539566874508, "p99": 18.143013843800873, "max": 20.838874974288046 }, "dispatch_delay_ms": { "min": 0.05004194099456072, "p50": 1.3434374704957008, "p95": 2.2430392215028405, "p99": 2.750858726212754, "max": 3.081333008594811 } }, "diagnostics": { "schedule_delay_ms": { "min": 0.006082933396100998, "p50": 0.3624164499349547, "p95": 0.6129850388976621, "p99": 0.6403888640924427, "max": 0.675916974433477 }, "probe_dispatch_ms": { "min": 0.028749927878379822, "p50": 0.4045204841531813, "p95": 1.7605939938221127, "p99": 2.499190252274275, "max": 4.731541033834219 }, "probe_service_ms": { "min": 0.6507080979645252, "p50": 5.528562469407916, "p95": 10.213833505986258, "p99": 13.118106051115303, "max": 22.918457980267704 }, "write_service_ms": { "min": 2.6942919939756393, "p50": 7.207687478512526, "p95": 11.884718760848047, "p99": 13.222896705847228, "max": 20.215999917127192 } }, "actual_submission_rate": 9.999886469866631, "metadata": { "backend": "weaviate", "collection": "fb2_429e9a_046", "dataset": { "dataset": "scifact", "source": "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/scifact.zip", "source_sha256": "536e14446a0ba56ed1398ab1055f39fe852686ecad24a6306c80c490fa8e0165", "source_md5": "5f7d1de60b170fc8027bb7898e2efca1", "records": 5183, "queries": 100, "relevant_documents_retained": 107, "seed": 20261028, "license": "Corpus: ODC-By-1.0; claims/evidence: CC-BY-4.0", "license_source": "https://raw.githubusercontent.com/allenai/scifact/master/LICENSE.md", "attribution": "BEIR and original dataset authors; Wikipedia contributors for NQ, SciFact authors and cited article authors for SciFact.", "transformation": "200-word title+text prefix; seeded reservoir of documents with relevant documents for selected test queries retained at front. Simulated mutations; no historical replay.", "model": "sentence-transformers/all-MiniLM-L6-v2", "model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "dimensions": 384, "normalized": true, "max_tokens": 256, "device": "mps", "files": { "corpus.jsonl": "189d58f273468352c9f73c6333c5120dcfb7ba1695984f073f0ebb6c100ed0cd", "queries.json": "c8b333c73353a8c4aeb7d6b63067c9a241aa5d39a0314f0c315ec63019ee9558", "vectors.npy": "b6b21404977151ce565e7d6d57d9931a4679e1ecf4aa44fcd9b29eb54955d09e", "query_vectors.npy": "02324eb7c3b3d2e2f0ec3208453e2ec0efd5d61d9fc845b1f8b7950e913f2730", "UPSTREAM-LICENSE.md": "56ee69bc639cbab8f0b2e4b67091ec5e9390aa99a1d65f0a35e4274686c9a806" } }, "base_count": 4000, "profile": { "dataset": "scifact", "base_count": 4000, "rate": 10, "query_rate": 10, "arrival": "steady" }, "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "preloaded_mutable": 200, "started_utc": "2026-10-29T15:42:09Z", "setup_s": 1.2127800420857966, "backend_before": { "version": "1.40.3", "schema": { "class": "Ffb2429e9a046", "invertedIndexConfig": { "bm25": { "b": 0.75, "k1": 1.2 }, "cleanupIntervalSeconds": 60, "stopwords": { "additions": null, "preset": "en", "removals": null }, "usingBlockMaxWAND": true }, "multiTenancyConfig": { "autoTenantActivation": false, "autoTenantCreation": false, "enabled": false }, "properties": [ { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "doc_id", "tokenization": "field" }, { "dataType": [ "int" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": false, "name": "version" }, { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "cohort", "tokenization": "field" }, { "dataType": [ "text" ], "indexFilterable": false, "indexRangeFilters": false, "indexSearchable": false, "name": "text", "tokenization": "word" } ], "shardingConfig": { "actualCount": 1, "actualVirtualCount": 128, "desiredCount": 1, "desiredVirtualCount": 128, "function": "murmur3", "key": "_id", "strategy": "hash", "virtualPerPhysical": 128 }, "vectorIndexConfig": { "bq": { "enabled": false }, "cleanupIntervalSeconds": 300, "distance": "cosine", "dynamicEfFactor": 8, "dynamicEfMax": 500, "dynamicEfMin": 100, "ef": 128, "efConstruction": 128, "filterStrategy": "acorn", "flatSearchCutoff": 0, "maxConnections": 16, "multivector": { "aggregation": "maxSim", "enabled": false, "muvera": { "dprojections": 16, "enabled": false, "ksim": 4, "repetitions": 10 } }, "pq": { "bitCompression": false, "centroids": 256, "enabled": false, "encoder": { "distribution": "log-normal", "type": "kmeans" }, "segments": 0, "trainingLimit": 100000 }, "rq": { "bits": 8, "centering": false, "enabled": false, "rescoreLimit": 20, "trainingLimit": 10000 }, "skip": false, "skipDefaultQuantization": false, "sq": { "enabled": false, "rescoreLimit": 20, "trainingLimit": 100000 }, "trackDefaultQuantization": false, "vectorCacheMaxObjects": 1000000000000 }, "vectorIndexType": "hnsw", "vectorizer": "none", "replicationConfig": { "deletionStrategy": "TimeBasedResolution", "factor": 1, "asyncEnabled": false } }, "shards": [ { "name": "p4VpMBlzDmBw", "per_node_status": { "node1": "READY" }, "status": "READY", "vectorQueueSize": 0 } ], "ack": "synchronous indexing, object PUT/POST/DELETE", "transport": "HTTP REST writes + GraphQL search, pooled httpx" }, "count_before": 4200, "idle_duration_s": 10.0, "idle_queries": { "count": 100, "ann_recall": 1.0, "ndcg": 0.6567157762068453, "latency_ms": { "min": 3.7800410063937306, "p50": 10.079603991471231, "p95": 11.745160381542519, "p99": 14.457965558394806, "max": 17.46574998833239 } }, "exact_ndcg": 0.6567157762068453, "resources_before": { "usage_usec": 17431641, "user_usec": 16626911, "system_usec": 804729, "nice_usec": 0, "nr_periods": 1022, "nr_throttled": 4, "throttled_usec": 21159, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 265740288, "pids": 10 }, "resources_after": { "usage_usec": 25661801, "user_usec": 24398720, "system_usec": 1263080, "nice_usec": 0, "nr_periods": 1323, "nr_throttled": 4, "throttled_usec": 21159, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 273899520, "pids": 10 }, "count_after": 4200, "backend_after": { "version": "1.40.3", "schema": { "class": "Ffb2429e9a046", "invertedIndexConfig": { "bm25": { "b": 0.75, "k1": 1.2 }, "cleanupIntervalSeconds": 60, "stopwords": { "additions": null, "preset": "en", "removals": null }, "usingBlockMaxWAND": true }, "multiTenancyConfig": { "autoTenantActivation": false, "autoTenantCreation": false, "enabled": false }, "properties": [ { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "doc_id", "tokenization": "field" }, { "dataType": [ "int" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": false, "name": "version" }, { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "cohort", "tokenization": "field" }, { "dataType": [ "text" ], "indexFilterable": false, "indexRangeFilters": false, "indexSearchable": false, "name": "text", "tokenization": "word" } ], "shardingConfig": { "actualCount": 1, "actualVirtualCount": 128, "desiredCount": 1, "desiredVirtualCount": 128, "function": "murmur3", "key": "_id", "strategy": "hash", "virtualPerPhysical": 128 }, "vectorIndexConfig": { "bq": { "enabled": false }, "cleanupIntervalSeconds": 300, "distance": "cosine", "dynamicEfFactor": 8, "dynamicEfMax": 500, "dynamicEfMin": 100, "ef": 128, "efConstruction": 128, "filterStrategy": "acorn", "flatSearchCutoff": 0, "maxConnections": 16, "multivector": { "aggregation": "maxSim", "enabled": false, "muvera": { "dprojections": 16, "enabled": false, "ksim": 4, "repetitions": 10 } }, "pq": { "bitCompression": false, "centroids": 256, "enabled": false, "encoder": { "distribution": "log-normal", "type": "kmeans" }, "segments": 0, "trainingLimit": 100000 }, "rq": { "bits": 8, "centering": false, "enabled": false, "rescoreLimit": 20, "trainingLimit": 10000 }, "skip": false, "skipDefaultQuantization": false, "sq": { "enabled": false, "rescoreLimit": 20, "trainingLimit": 100000 }, "trackDefaultQuantization": false, "vectorCacheMaxObjects": 1000000000000 }, "vectorIndexType": "hnsw", "vectorizer": "none", "replicationConfig": { "deletionStrategy": "TimeBasedResolution", "factor": 1, "asyncEnabled": false } }, "shards": [ { "name": "p4VpMBlzDmBw", "per_node_status": { "node1": "READY" }, "status": "READY", "vectorQueueSize": 0 } ], "ack": "synchronous indexing, object PUT/POST/DELETE", "transport": "HTTP REST writes + GraphQL search, pooled httpx" } }, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "eligible": true, "protocol": "mixed-mutations-v2" }, "examples": [ { "event": 0, "operation": "insert", "doc_id": "mutation-000000", "scheduled_s": 0.0, "submitted_s": 0.00021058297716081142, "schedule_delay_ms": 0.21058297716081142, "precondition": true, "status": "success", "ack_ms": 4.46566694881767, "write_started_ms": 0.9039170108735561, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 12.691749958321452, "stable_since_ms": 12.691749958321452, "confirmed_ms": 64.97837498318404, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 5.653708940371871, "stable_since_ms": 5.653708940371871, "confirmed_ms": 62.790249940007925, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 12.691749958321452, "confirmed_ms": 64.97837498318404, "search_minus_ack_ms": 1.1880419915542006 }, { "event": 1, "operation": "update", "doc_id": "mutation-000001", "scheduled_s": 0.1, "submitted_s": 0.1002882500179112, "schedule_delay_ms": 0.2882500179111902, "precondition": true, "status": "success", "ack_ms": 5.389082944020629, "write_started_ms": 0.3687079297378659, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 14.902749913744628, "stable_since_ms": 14.902749913744628, "confirmed_ms": 68.05991695728153, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 7.115707965567708, "stable_since_ms": 7.115707965567708, "confirmed_ms": 70.2020829776302, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 14.902749913744628, "confirmed_ms": 70.2020829776302, "search_minus_ack_ms": 1.726625021547079 }, { "event": 2, "operation": "delete", "doc_id": "mutation-000002", "scheduled_s": 0.2, "submitted_s": 0.2004091669805348, "schedule_delay_ms": 0.40916698053478084, "precondition": true, "status": "success", "ack_ms": 3.065333003178239, "write_started_ms": 0.3710410092025995, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 3.238874953240156, "stable_since_ms": 3.238874953240156, "confirmed_ms": 58.235415956005454, "consecutive": 3, "regressions": 0, "requests": 3 }, "search": { "first_ms": 7.388458005152643, "stable_since_ms": 7.388458005152643, "confirmed_ms": 71.95849996060133, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 7.388458005152643, "confirmed_ms": 71.95849996060133, "search_minus_ack_ms": 4.323125001974404 } ] }, { "run": "fb2_429e9a_047", "trial": 1, "backend": "pgvector", "scenario": "nq-10k", "status": "complete", "eligible": true, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "summary": { "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "wall_s": 30.00075770902913, "operations": { "insert": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 16.267750062979758, "p50": 22.471749980468303, "p95": 25.949552131351084, "p99": 26.82151382556186, "max": 28.324416955001652 }, "confirmed_ms": { "min": 72.37145898398012, "p50": 85.38460405543447, "p95": 89.19969765702263, "p99": 90.14796411036514, "max": 91.06837504077703 }, "ack_ms": { "min": 4.763250006362796, "p50": 9.751103993039578, "p95": 11.649919423507527, "p99": 17.052767203422274, "max": 19.137625000439584 }, "search_minus_ack_ms": { "min": 6.405833992175758, "p50": 12.348104559350759, "p95": 14.980489894514903, "p99": 16.103924645576633, "max": 17.618582933209836 }, "regressions": 0, "confirmed_within_100ms": 1.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "update": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 14.957832987420261, "p50": 22.488437476567924, "p95": 25.759424740681425, "p99": 29.56904832506554, "max": 33.12545805238187 }, "confirmed_ms": { "min": 70.37424994632602, "p50": 86.01508353604004, "p95": 90.40831254096702, "p99": 94.30229656165467, "max": 96.86020796652883 }, "ack_ms": { "min": 4.9359999829903245, "p50": 9.595624986104667, "p95": 12.102104222867638, "p99": 13.855167828733109, "max": 15.327875036746264 }, "search_minus_ack_ms": { "min": 5.339875002391636, "p50": 12.551438005175442, "p95": 16.101152735063806, "p99": 19.03234302764761, "max": 23.5296250320971 }, "regressions": 0, "confirmed_within_100ms": 1.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "delete": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 11.708874953910708, "p50": 14.127479575108737, "p95": 23.484032740816474, "p99": 24.109396885614842, "max": 24.264125037007034 }, "confirmed_ms": { "min": 63.60308302100748, "p50": 72.25129147991538, "p95": 87.45332955149934, "p99": 88.81647989852354, "max": 89.13208392914385 }, "ack_ms": { "min": 1.2106250505894423, "p50": 2.841624431312084, "p95": 4.097185388673097, "p99": 4.699568261858077, "max": 4.805334028787911 }, "search_minus_ack_ms": { "min": 1.561791985295713, "p50": 5.284166487399489, "p95": 19.652185443555936, "p99": 20.851414756616578, "max": 21.032749908044934 }, "regressions": 0, "confirmed_within_100ms": 1.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 } }, "total_operations": 300, "statuses": { "success": 300, "timeout": 0, "error": 0, "invalid_precondition": 0, "client_overload": 0 }, "query": { "count": 300, "success": 300, "ann_recall_at_10": 0.9979999999999999, "ndcg_at_10": 0.9481505328688746, "latency_ms": { "min": 2.376041025854647, "p50": 5.730062490329146, "p95": 6.980877689784393, "p99": 7.923999020131304, "max": 13.95712501835078 }, "dispatch_delay_ms": { "min": 0.08829194121062756, "p50": 1.048937439918518, "p95": 1.825679751345888, "p99": 1.979806930758059, "max": 4.295542021282017 } }, "diagnostics": { "schedule_delay_ms": { "min": 0.00212499871921068, "p50": 0.33402098342705244, "p95": 0.6059253506821349, "p99": 0.6367164468389142, "max": 0.6621249485760927 }, "probe_dispatch_ms": { "min": 0.03424996975809336, "p50": 0.42727100662887096, "p95": 1.5427326725330175, "p99": 2.340237100142989, "max": 2.8425409691408277 }, "probe_service_ms": { "min": 0.35404099617153406, "p50": 2.7414584765210748, "p95": 7.120298570953309, "p99": 7.867339977528906, "max": 18.97054200526327 }, "write_service_ms": { "min": 1.0600420646369457, "p50": 8.8115205289796, "p95": 11.25716611277312, "p99": 14.686081347754225, "max": 18.18112505134195 } }, "actual_submission_rate": 9.999995303706893, "metadata": { "backend": "pgvector", "collection": "fb2_429e9a_047", "dataset": { "dataset": "nq", "source": "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/nq.zip", "source_sha256": "2553bf7bfbab47b1436ca00a34bce57320e18e611fd00999a2a3a1b4714be752", "source_md5": "d4d3d2e48787a744b6f6e691ff534307", "records": 124000, "queries": 100, "relevant_documents_retained": 120, "seed": 20261028, "license": "CC-BY-SA-3.0", "license_source": "https://raw.githubusercontent.com/google-research-datasets/natural-questions/master/nq_open/README.md", "attribution": "BEIR and original dataset authors; Wikipedia contributors for NQ, SciFact authors and cited article authors for SciFact.", "transformation": "200-word title+text prefix; seeded reservoir of documents with relevant documents for selected test queries retained at front. Simulated mutations; no historical replay.", "model": "sentence-transformers/all-MiniLM-L6-v2", "model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "dimensions": 384, "normalized": true, "max_tokens": 256, "device": "mps", "files": { "corpus.jsonl": "9dc487d785e6ee493eccaf764daa3bf8720a8f4fc8c406246065e1ad33a91f5d", "queries.json": "5603f52876ada5d36118b65dea490f1d43900bc159e1804e637796eb9115e999", "vectors.npy": "cb7421eb6ca5964231403f0a922843088f6fd0e01cbf67f2a1076cc7101005db", "query_vectors.npy": "5e666f8d966d8db6b0fd93d4963234d2dffa0ee3910b8fad069a8ee201e8b5d4", "UPSTREAM-LICENSE.md": "8b675521da3fa142ac6bda38c9bcfcc8cff0e2d6128be53a062ed55fb53f7b87" } }, "base_count": 10000, "profile": { "dataset": "nq", "base_count": 10000, "rate": 10, "query_rate": 10, "arrival": "steady" }, "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "preloaded_mutable": 200, "started_utc": "2026-10-29T15:42:54Z", "setup_s": 3.6316290840040892, "backend_before": { "version": "0.8.7", "postgres": "PostgreSQL 16.15 (Debian 16.15-1.pgdg12+2) on aarch64-unknown-linux-gnu, compiled by gcc (Debian 12.2.0-14+deb12u1) 12.2.0, 64-bit", "ef_search": 128, "ack": "autocommit, READ COMMITTED", "transport": "psycopg, connection per worker", "query_plan": [ { "Plan": { "Node Type": "Limit", "Parallel Aware": false, "Async Capable": false, "Startup Cost": 1340.05, "Total Cost": 1356.32, "Plan Rows": 10, "Plan Width": 18, "Plans": [ { "Node Type": "Index Scan", "Parent Relationship": "Outer", "Parallel Aware": false, "Async Capable": false, "Scan Direction": "Forward", "Index Name": "fb2_429e9a_047_hnsw", "Relation Name": "fb2_429e9a_047", "Alias": "fb2_429e9a_047", "Startup Cost": 1340.05, "Total Cost": 17605.0, "Plan Rows": 10000, "Plan Width": 18, "Order By": "(embedding <=> '[-0.08946895,-0.094362676,0.092023626,-0.0040844884,0.043233823,0.00062519487,0.1075197,-0.035732996,0.013746996,-0.024103122,0.0021706754,-0.04720602,0.04787615,-0.02043222,0.058315646,0.081252135,-0.05375015,-0.030799072,-0.0077205584,0.035735283,-0.04474784,0.01614922,-0.06905586,0.04194401,-0.08832908,0.007873556,0.0044992934,0.021968655,0.010953364,0.07212316,0.023185596,0.001176476,0.04725166,-0.03580423,-0.032344446,0.004752835,-0.056179125,-0.0353235,-0.037251893,-0.005143068,-0.027599342,0.024403147,0.031338032,0.0056410627,0.008425455,-0.0087741045,-0.039870497,-0.011831836,0.042621233,0.048142947,0.01603842,0.06745542,-0.038018145,0.064212486,-0.009636566,0.01607475,-0.0027868534,-0.03761097,-0.011889431,-0.01084577,0.083403684,0.028946918,0.061870236,-0.045382615,0.03812586,0.060426526,-0.017003024,0.02328925,-0.093020774,-0.024072766,0.042720392,-0.022444837,-0.011597358,0.01209231,-0.005953629,0.029691145,-0.011711495,-0.053345133,-0.081757374,-0.049303155,0.032193717,-0.038507868,-0.0060101775,-0.031470332,-0.07362443,0.03575578,0.0245484,0.01985519,-0.15116754,0.050386053,-0.0389634,0.07548389,-0.09264168,-0.012602816,-0.0035539444,-0.046416666,0.015100493,0.100691915,-0.037529,0.04600518,-0.021859422,0.014675989,-0.097722545,-0.053200893,-0.017040461,-0.01629616,-0.012339222,0.06543302,0.07252074,-0.044301156,0.03054835,-0.038570553,0.019310065,-0.005527747,-0.029386865,-0.058248505,0.065443166,0.028506057,0.0878254,0.058387812,0.081721246,-0.0005962058,0.0569388,0.00960514,0.011667701,0.014456492,-0.07218306,-3.0444767e-33,0.06687367,-0.019213954,0.058202963,0.025171444,-0.028259544,-0.04171671,0.03127722,0.04865431,-0.0721459,-0.050143186,-0.029907746,0.06248994,-0.05096076,-0.065192886,0.010210447,-0.077256374,-0.043370593,-0.0901922,-0.066522665,0.026072454,-0.022315364,0.075375885,-0.029968103,-0.0050771856,-0.0046955296,0.0065650316,-0.06791897,0.07368851,0.056069646,-0.021026025,-0.011352883,0.038811427,-0.090847075,-0.056490287,-0.039881036,0.032885592,-0.029586185,0.04753608,0.006341778,-0.0034275919,0.03681623,0.09326265,-0.0532089,0.032396164,0.012605483,0.045778517,-0.0006472902,0.029721813,0.0734947,-0.037117425,-0.09356724,-0.0071634455,-0.032400057,0.010453378,0.06286513,0.006053158,0.060524322,0.091866635,0.063640274,0.055919155,0.053222507,-0.019174587,0.089296214,-0.058922265,0.013745364,-0.022907477,0.028347451,-0.040759467,-0.06820364,-0.061398294,-0.046857864,0.0060627847,0.010442015,-0.07662317,0.031950947,-0.107813306,-0.0016125626,-0.033413693,-0.09989722,-0.03580293,-0.036118146,-0.08276185,0.1280713,0.07596603,0.049225595,-0.026312606,0.0031930343,-0.0012469466,0.000872793,-0.040353082,-0.034516506,-0.036047976,0.004999689,-0.01570079,-0.09783203,1.4186912e-33,-0.0130903,0.04742377,0.02889633,0.09809088,0.043639127,-0.0747531,0.03402747,0.029147558,0.019848268,0.111865185,0.04845458,-0.016222177,0.025501896,0.09813583,-0.002833195,0.0038929228,0.048537854,0.014111116,0.036570966,-0.017512968,-0.079755045,0.043452248,0.10843892,0.061968718,-0.015678233,0.0060376576,0.010718569,-0.03907552,0.040307663,0.04553227,0.041731495,0.083179526,-0.0844024,-0.023447752,0.014485363,-0.00788756,-0.11410207,0.039309476,0.011220049,0.12889338,-0.07970701,0.010658111,-0.06657531,-0.022037188,-0.043459363,-0.030291978,0.05803613,0.06109021,-0.09415837,0.030727232,0.011858702,0.013526454,-0.049580045,-0.052486703,0.013218615,0.0050231596,0.07660412,-0.0046343356,-0.0692584,-0.03353194,0.022288544,0.022836983,0.01361394,-0.036331616,0.008194689,0.09226991,0.08229875,-0.047639843,-0.005033471,0.0066081914,-0.106666416,-0.06385503,0.006385333,0.06318358,0.03146985,-0.060764216,-0.043287244,-0.042385958,-0.07963737,-0.043478705,-0.031840656,0.060027804,0.027829666,0.03728562,-0.07014622,0.0649568,-0.052579004,-0.0034938955,-0.15390792,-0.04426162,0.028217599,0.05593145,0.023251446,0.029115746,-0.040321343,-1.8832651e-08,-0.037536032,0.0005056563,0.013919978,-0.038129315,0.061486673,0.005534046,-0.010768211,-0.16527015,0.061414365,0.019963786,-0.019666849,-0.059065994,0.00083181454,-0.016695915,0.01090875,0.02685501,-0.0008743374,-0.060967337,-0.049258173,0.012731965,-0.03973743,-0.0018210232,-0.03797001,0.00035104176,0.04477184,0.04185603,-0.03215796,-0.09103283,0.0467481,0.017810425,0.037437033,0.054782324,-0.0059442874,0.022557084,0.053393807,-0.03746794,-0.07080435,0.10548834,-0.04215435,0.042547524,0.05064807,0.07560324,0.04072184,-0.0011278372,0.00702018,-0.070807114,0.040035766,0.051417697,0.011316928,-0.028955372,0.04197099,-0.0026867494,-0.056857754,0.0761941,0.030621601,-0.052413527,-0.021492323,0.0957355,0.1013111,-0.082977295,-0.036080915,-0.03327806,0.0029404552,-0.03586191]'::vector)", "Filter": "(cohort = 'static'::text)" } ] } } ] }, "count_before": 10200, "idle_duration_s": 10.0, "idle_queries": { "count": 100, "ann_recall": 0.998, "ndcg": 0.9481505328688745, "latency_ms": { "min": 3.7001249147579074, "p50": 7.385333010461181, "p95": 8.529001730494201, "p99": 9.08936832682236, "max": 21.298666950315237 } }, "exact_ndcg": 0.9481505328688745, "resources_before": { "usage_usec": 5319180, "user_usec": 5010515, "system_usec": 308664, "nice_usec": 0, "nr_periods": 262, "nr_throttled": 19, "throttled_usec": 1920717, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 229781504, "pids": 23 }, "resources_after": { "usage_usec": 12603542, "user_usec": 11858877, "system_usec": 744665, "nice_usec": 0, "nr_periods": 563, "nr_throttled": 19, "throttled_usec": 1920717, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 241901568, "pids": 27 }, "count_after": 10200, "backend_after": { "version": "0.8.7", "postgres": "PostgreSQL 16.15 (Debian 16.15-1.pgdg12+2) on aarch64-unknown-linux-gnu, compiled by gcc (Debian 12.2.0-14+deb12u1) 12.2.0, 64-bit", "ef_search": 128, "ack": "autocommit, READ COMMITTED", "transport": "psycopg, connection per worker", "query_plan": [ { "Plan": { "Node Type": "Limit", "Parallel Aware": false, "Async Capable": false, "Startup Cost": 1349.45, "Total Cost": 1365.78, "Plan Rows": 10, "Plan Width": 18, "Plans": [ { "Node Type": "Index Scan", "Parent Relationship": "Outer", "Parallel Aware": false, "Async Capable": false, "Scan Direction": "Forward", "Index Name": "fb2_429e9a_047_hnsw", "Relation Name": "fb2_429e9a_047", "Alias": "fb2_429e9a_047", "Startup Cost": 1349.45, "Total Cost": 17908.19, "Plan Rows": 10139, "Plan Width": 18, "Order By": "(embedding <=> '[-0.08946895,-0.094362676,0.092023626,-0.0040844884,0.043233823,0.00062519487,0.1075197,-0.035732996,0.013746996,-0.024103122,0.0021706754,-0.04720602,0.04787615,-0.02043222,0.058315646,0.081252135,-0.05375015,-0.030799072,-0.0077205584,0.035735283,-0.04474784,0.01614922,-0.06905586,0.04194401,-0.08832908,0.007873556,0.0044992934,0.021968655,0.010953364,0.07212316,0.023185596,0.001176476,0.04725166,-0.03580423,-0.032344446,0.004752835,-0.056179125,-0.0353235,-0.037251893,-0.005143068,-0.027599342,0.024403147,0.031338032,0.0056410627,0.008425455,-0.0087741045,-0.039870497,-0.011831836,0.042621233,0.048142947,0.01603842,0.06745542,-0.038018145,0.064212486,-0.009636566,0.01607475,-0.0027868534,-0.03761097,-0.011889431,-0.01084577,0.083403684,0.028946918,0.061870236,-0.045382615,0.03812586,0.060426526,-0.017003024,0.02328925,-0.093020774,-0.024072766,0.042720392,-0.022444837,-0.011597358,0.01209231,-0.005953629,0.029691145,-0.011711495,-0.053345133,-0.081757374,-0.049303155,0.032193717,-0.038507868,-0.0060101775,-0.031470332,-0.07362443,0.03575578,0.0245484,0.01985519,-0.15116754,0.050386053,-0.0389634,0.07548389,-0.09264168,-0.012602816,-0.0035539444,-0.046416666,0.015100493,0.100691915,-0.037529,0.04600518,-0.021859422,0.014675989,-0.097722545,-0.053200893,-0.017040461,-0.01629616,-0.012339222,0.06543302,0.07252074,-0.044301156,0.03054835,-0.038570553,0.019310065,-0.005527747,-0.029386865,-0.058248505,0.065443166,0.028506057,0.0878254,0.058387812,0.081721246,-0.0005962058,0.0569388,0.00960514,0.011667701,0.014456492,-0.07218306,-3.0444767e-33,0.06687367,-0.019213954,0.058202963,0.025171444,-0.028259544,-0.04171671,0.03127722,0.04865431,-0.0721459,-0.050143186,-0.029907746,0.06248994,-0.05096076,-0.065192886,0.010210447,-0.077256374,-0.043370593,-0.0901922,-0.066522665,0.026072454,-0.022315364,0.075375885,-0.029968103,-0.0050771856,-0.0046955296,0.0065650316,-0.06791897,0.07368851,0.056069646,-0.021026025,-0.011352883,0.038811427,-0.090847075,-0.056490287,-0.039881036,0.032885592,-0.029586185,0.04753608,0.006341778,-0.0034275919,0.03681623,0.09326265,-0.0532089,0.032396164,0.012605483,0.045778517,-0.0006472902,0.029721813,0.0734947,-0.037117425,-0.09356724,-0.0071634455,-0.032400057,0.010453378,0.06286513,0.006053158,0.060524322,0.091866635,0.063640274,0.055919155,0.053222507,-0.019174587,0.089296214,-0.058922265,0.013745364,-0.022907477,0.028347451,-0.040759467,-0.06820364,-0.061398294,-0.046857864,0.0060627847,0.010442015,-0.07662317,0.031950947,-0.107813306,-0.0016125626,-0.033413693,-0.09989722,-0.03580293,-0.036118146,-0.08276185,0.1280713,0.07596603,0.049225595,-0.026312606,0.0031930343,-0.0012469466,0.000872793,-0.040353082,-0.034516506,-0.036047976,0.004999689,-0.01570079,-0.09783203,1.4186912e-33,-0.0130903,0.04742377,0.02889633,0.09809088,0.043639127,-0.0747531,0.03402747,0.029147558,0.019848268,0.111865185,0.04845458,-0.016222177,0.025501896,0.09813583,-0.002833195,0.0038929228,0.048537854,0.014111116,0.036570966,-0.017512968,-0.079755045,0.043452248,0.10843892,0.061968718,-0.015678233,0.0060376576,0.010718569,-0.03907552,0.040307663,0.04553227,0.041731495,0.083179526,-0.0844024,-0.023447752,0.014485363,-0.00788756,-0.11410207,0.039309476,0.011220049,0.12889338,-0.07970701,0.010658111,-0.06657531,-0.022037188,-0.043459363,-0.030291978,0.05803613,0.06109021,-0.09415837,0.030727232,0.011858702,0.013526454,-0.049580045,-0.052486703,0.013218615,0.0050231596,0.07660412,-0.0046343356,-0.0692584,-0.03353194,0.022288544,0.022836983,0.01361394,-0.036331616,0.008194689,0.09226991,0.08229875,-0.047639843,-0.005033471,0.0066081914,-0.106666416,-0.06385503,0.006385333,0.06318358,0.03146985,-0.060764216,-0.043287244,-0.042385958,-0.07963737,-0.043478705,-0.031840656,0.060027804,0.027829666,0.03728562,-0.07014622,0.0649568,-0.052579004,-0.0034938955,-0.15390792,-0.04426162,0.028217599,0.05593145,0.023251446,0.029115746,-0.040321343,-1.8832651e-08,-0.037536032,0.0005056563,0.013919978,-0.038129315,0.061486673,0.005534046,-0.010768211,-0.16527015,0.061414365,0.019963786,-0.019666849,-0.059065994,0.00083181454,-0.016695915,0.01090875,0.02685501,-0.0008743374,-0.060967337,-0.049258173,0.012731965,-0.03973743,-0.0018210232,-0.03797001,0.00035104176,0.04477184,0.04185603,-0.03215796,-0.09103283,0.0467481,0.017810425,0.037437033,0.054782324,-0.0059442874,0.022557084,0.053393807,-0.03746794,-0.07080435,0.10548834,-0.04215435,0.042547524,0.05064807,0.07560324,0.04072184,-0.0011278372,0.00702018,-0.070807114,0.040035766,0.051417697,0.011316928,-0.028955372,0.04197099,-0.0026867494,-0.056857754,0.0761941,0.030621601,-0.052413527,-0.021492323,0.0957355,0.1013111,-0.082977295,-0.036080915,-0.03327806,0.0029404552,-0.03586191]'::vector)", "Filter": "(cohort = 'static'::text)" } ] } } ] } }, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "eligible": true, "protocol": "mixed-mutations-v2" }, "examples": [ { "event": 0, "operation": "insert", "doc_id": "mutation-000000", "scheduled_s": 0.0, "submitted_s": 0.00023150001652538776, "schedule_delay_ms": 0.23150001652538776, "precondition": true, "status": "success", "ack_ms": 19.137625000439584, "write_started_ms": 0.9564999490976334, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 22.16008398681879, "stable_since_ms": 22.16008398681879, "confirmed_ms": 74.14995902217925, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 25.543458992615342, "stable_since_ms": 25.543458992615342, "confirmed_ms": 80.96049993764609, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 25.543458992615342, "confirmed_ms": 80.96049993764609, "search_minus_ack_ms": 6.405833992175758 }, { "event": 1, "operation": "update", "doc_id": "mutation-000001", "scheduled_s": 0.1, "submitted_s": 0.10012275003828108, "schedule_delay_ms": 0.12275003828107756, "precondition": true, "status": "success", "ack_ms": 7.7457919251173735, "write_started_ms": 0.5503339925780892, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 14.423291897401214, "stable_since_ms": 14.423291897401214, "confirmed_ms": 67.17587495222688, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 18.636791966855526, "stable_since_ms": 18.636791966855526, "confirmed_ms": 78.88216699939221, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 18.636791966855526, "confirmed_ms": 78.88216699939221, "search_minus_ack_ms": 10.891000041738153 }, { "event": 2, "operation": "delete", "doc_id": "mutation-000002", "scheduled_s": 0.2, "submitted_s": 0.20046133396681398, "schedule_delay_ms": 0.46133396681397043, "precondition": true, "status": "success", "ack_ms": 4.119707969948649, "write_started_ms": 0.5738330073654652, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 14.462791034020483, "stable_since_ms": 14.462791034020483, "confirmed_ms": 68.0525409989059, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 21.595290978439152, "stable_since_ms": 21.595290978439152, "confirmed_ms": 82.14345807209611, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 21.595290978439152, "confirmed_ms": 82.14345807209611, "search_minus_ack_ms": 17.475583008490503 } ] }, { "run": "fb2_429e9a_048", "trial": 3, "backend": "pgvector", "scenario": "nq-100k", "status": "complete", "eligible": true, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "summary": { "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "wall_s": 30.001361000002362, "operations": { "insert": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 19.540624925866723, "p50": 29.724895488470793, "p95": 35.28022266109474, "p99": 35.97347156493925, "max": 37.56287496071309 }, "confirmed_ms": { "min": 76.11208292655647, "p50": 89.99822899932042, "p95": 96.82950029382482, "p99": 99.69670225982554, "max": 100.38912494201213 }, "ack_ms": { "min": 8.699457976035774, "p50": 18.67014594608918, "p95": 23.7052247277461, "p99": 26.591031127609316, "max": 27.493333909660578 }, "search_minus_ack_ms": { "min": 7.567499997094274, "p50": 11.417937465012074, "p95": 12.927125697024167, "p99": 13.46260670106858, "max": 13.535042060539126 }, "regressions": 0, "confirmed_within_100ms": 0.99, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "update": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 16.572791035287082, "p50": 27.248208003584296, "p95": 33.677660051034756, "p99": 37.16865369584412, "max": 38.074875017628074 }, "confirmed_ms": { "min": 74.04329103883356, "p50": 86.95704204728827, "p95": 96.57452050596476, "p99": 98.54371579131112, "max": 99.13433308247477 }, "ack_ms": { "min": 5.520833074115217, "p50": 15.601395978592336, "p95": 22.025947814108804, "p99": 25.842191372066736, "max": 25.90241702273488 }, "search_minus_ack_ms": { "min": 7.331165950745344, "p50": 11.520166473928839, "p95": 12.781308655394241, "p99": 13.57037452165969, "max": 14.160166960209608 }, "regressions": 0, "confirmed_within_100ms": 1.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "delete": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 12.075791019015014, "p50": 13.900833495426923, "p95": 29.924027115339413, "p99": 36.38016295037238, "max": 46.90266598481685 }, "confirmed_ms": { "min": 64.49358304962516, "p50": 78.50327104097232, "p95": 94.00878751184791, "p99": 97.31899343896667, "max": 117.84025002270937 }, "ack_ms": { "min": 1.3207499869167805, "p50": 2.2226040018722415, "p95": 5.121852201409637, "p99": 7.169028257485635, "max": 37.412166013382375 }, "search_minus_ack_ms": { "min": 3.569082939065993, "p50": 11.61772955674678, "p95": 26.55147493351251, "p99": 28.227464770898244, "max": 32.398417009972036 }, "regressions": 0, "confirmed_within_100ms": 0.99, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 } }, "total_operations": 300, "statuses": { "success": 300, "timeout": 0, "error": 0, "invalid_precondition": 0, "client_overload": 0 }, "query": { "count": 300, "success": 300, "ann_recall_at_10": 0.9929999999999999, "ndcg_at_10": 0.7931268675492565, "latency_ms": { "min": 4.045457928441465, "p50": 10.641437431331724, "p95": 15.796393709024414, "p99": 18.333808307070285, "max": 20.817958982661366 }, "dispatch_delay_ms": { "min": 0.11999998241662979, "p50": 0.9065830381587148, "p95": 1.5627785527613018, "p99": 1.8655667104758311, "max": 2.4472499499097466 } }, "diagnostics": { "schedule_delay_ms": { "min": 0.0032080337408757487, "p50": 0.32470801379602676, "p95": 0.6051222677342594, "p99": 0.640960092655618, "max": 0.7928330684077167 }, "probe_dispatch_ms": { "min": 0.013874960131943226, "p50": 0.399416487198323, "p95": 1.3360165408812468, "p99": 1.830390761606389, "max": 2.3728329688310623 }, "probe_service_ms": { "min": 0.3074159612879157, "p50": 1.8387290183454752, "p95": 12.315841013332829, "p99": 16.090273781446736, "max": 21.284791990183294 }, "write_service_ms": { "min": 1.047917059622705, "p50": 13.815104495733976, "p95": 22.609518736135215, "p99": 25.211450253846113, "max": 37.031208048574626 } }, "actual_submission_rate": 9.999948996908849, "metadata": { "backend": "pgvector", "collection": "fb2_429e9a_048", "dataset": { "dataset": "nq", "source": "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/nq.zip", "source_sha256": "2553bf7bfbab47b1436ca00a34bce57320e18e611fd00999a2a3a1b4714be752", "source_md5": "d4d3d2e48787a744b6f6e691ff534307", "records": 124000, "queries": 100, "relevant_documents_retained": 120, "seed": 20261028, "license": "CC-BY-SA-3.0", "license_source": "https://raw.githubusercontent.com/google-research-datasets/natural-questions/master/nq_open/README.md", "attribution": "BEIR and original dataset authors; Wikipedia contributors for NQ, SciFact authors and cited article authors for SciFact.", "transformation": "200-word title+text prefix; seeded reservoir of documents with relevant documents for selected test queries retained at front. Simulated mutations; no historical replay.", "model": "sentence-transformers/all-MiniLM-L6-v2", "model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "dimensions": 384, "normalized": true, "max_tokens": 256, "device": "mps", "files": { "corpus.jsonl": "9dc487d785e6ee493eccaf764daa3bf8720a8f4fc8c406246065e1ad33a91f5d", "queries.json": "5603f52876ada5d36118b65dea490f1d43900bc159e1804e637796eb9115e999", "vectors.npy": "cb7421eb6ca5964231403f0a922843088f6fd0e01cbf67f2a1076cc7101005db", "query_vectors.npy": "5e666f8d966d8db6b0fd93d4963234d2dffa0ee3910b8fad069a8ee201e8b5d4", "UPSTREAM-LICENSE.md": "8b675521da3fa142ac6bda38c9bcfcc8cff0e2d6128be53a062ed55fb53f7b87" } }, "base_count": 100000, "profile": { "dataset": "nq", "base_count": 100000, "rate": 10, "query_rate": 10, "arrival": "steady" }, "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "preloaded_mutable": 200, "started_utc": "2026-10-29T15:43:43Z", "setup_s": 57.83600125007797, "backend_before": { "version": "0.8.7", "postgres": "PostgreSQL 16.15 (Debian 16.15-1.pgdg12+2) on aarch64-unknown-linux-gnu, compiled by gcc (Debian 12.2.0-14+deb12u1) 12.2.0, 64-bit", "ef_search": 128, "ack": "autocommit, READ COMMITTED", "transport": "psycopg, connection per worker", "query_plan": [ { "Plan": { "Node Type": "Limit", "Parallel Aware": false, "Async Capable": false, "Startup Cost": 1674.2, "Total Cost": 1691.32, "Plan Rows": 10, "Plan Width": 18, "Plans": [ { "Node Type": "Index Scan", "Parent Relationship": "Outer", "Parallel Aware": false, "Async Capable": false, "Scan Direction": "Forward", "Index Name": "fb2_429e9a_048_hnsw", "Relation Name": "fb2_429e9a_048", "Alias": "fb2_429e9a_048", "Startup Cost": 1674.2, "Total Cost": 172861.98, "Plan Rows": 99990, "Plan Width": 18, "Order By": "(embedding <=> '[-0.08946895,-0.094362676,0.092023626,-0.0040844884,0.043233823,0.00062519487,0.1075197,-0.035732996,0.013746996,-0.024103122,0.0021706754,-0.04720602,0.04787615,-0.02043222,0.058315646,0.081252135,-0.05375015,-0.030799072,-0.0077205584,0.035735283,-0.04474784,0.01614922,-0.06905586,0.04194401,-0.08832908,0.007873556,0.0044992934,0.021968655,0.010953364,0.07212316,0.023185596,0.001176476,0.04725166,-0.03580423,-0.032344446,0.004752835,-0.056179125,-0.0353235,-0.037251893,-0.005143068,-0.027599342,0.024403147,0.031338032,0.0056410627,0.008425455,-0.0087741045,-0.039870497,-0.011831836,0.042621233,0.048142947,0.01603842,0.06745542,-0.038018145,0.064212486,-0.009636566,0.01607475,-0.0027868534,-0.03761097,-0.011889431,-0.01084577,0.083403684,0.028946918,0.061870236,-0.045382615,0.03812586,0.060426526,-0.017003024,0.02328925,-0.093020774,-0.024072766,0.042720392,-0.022444837,-0.011597358,0.01209231,-0.005953629,0.029691145,-0.011711495,-0.053345133,-0.081757374,-0.049303155,0.032193717,-0.038507868,-0.0060101775,-0.031470332,-0.07362443,0.03575578,0.0245484,0.01985519,-0.15116754,0.050386053,-0.0389634,0.07548389,-0.09264168,-0.012602816,-0.0035539444,-0.046416666,0.015100493,0.100691915,-0.037529,0.04600518,-0.021859422,0.014675989,-0.097722545,-0.053200893,-0.017040461,-0.01629616,-0.012339222,0.06543302,0.07252074,-0.044301156,0.03054835,-0.038570553,0.019310065,-0.005527747,-0.029386865,-0.058248505,0.065443166,0.028506057,0.0878254,0.058387812,0.081721246,-0.0005962058,0.0569388,0.00960514,0.011667701,0.014456492,-0.07218306,-3.0444767e-33,0.06687367,-0.019213954,0.058202963,0.025171444,-0.028259544,-0.04171671,0.03127722,0.04865431,-0.0721459,-0.050143186,-0.029907746,0.06248994,-0.05096076,-0.065192886,0.010210447,-0.077256374,-0.043370593,-0.0901922,-0.066522665,0.026072454,-0.022315364,0.075375885,-0.029968103,-0.0050771856,-0.0046955296,0.0065650316,-0.06791897,0.07368851,0.056069646,-0.021026025,-0.011352883,0.038811427,-0.090847075,-0.056490287,-0.039881036,0.032885592,-0.029586185,0.04753608,0.006341778,-0.0034275919,0.03681623,0.09326265,-0.0532089,0.032396164,0.012605483,0.045778517,-0.0006472902,0.029721813,0.0734947,-0.037117425,-0.09356724,-0.0071634455,-0.032400057,0.010453378,0.06286513,0.006053158,0.060524322,0.091866635,0.063640274,0.055919155,0.053222507,-0.019174587,0.089296214,-0.058922265,0.013745364,-0.022907477,0.028347451,-0.040759467,-0.06820364,-0.061398294,-0.046857864,0.0060627847,0.010442015,-0.07662317,0.031950947,-0.107813306,-0.0016125626,-0.033413693,-0.09989722,-0.03580293,-0.036118146,-0.08276185,0.1280713,0.07596603,0.049225595,-0.026312606,0.0031930343,-0.0012469466,0.000872793,-0.040353082,-0.034516506,-0.036047976,0.004999689,-0.01570079,-0.09783203,1.4186912e-33,-0.0130903,0.04742377,0.02889633,0.09809088,0.043639127,-0.0747531,0.03402747,0.029147558,0.019848268,0.111865185,0.04845458,-0.016222177,0.025501896,0.09813583,-0.002833195,0.0038929228,0.048537854,0.014111116,0.036570966,-0.017512968,-0.079755045,0.043452248,0.10843892,0.061968718,-0.015678233,0.0060376576,0.010718569,-0.03907552,0.040307663,0.04553227,0.041731495,0.083179526,-0.0844024,-0.023447752,0.014485363,-0.00788756,-0.11410207,0.039309476,0.011220049,0.12889338,-0.07970701,0.010658111,-0.06657531,-0.022037188,-0.043459363,-0.030291978,0.05803613,0.06109021,-0.09415837,0.030727232,0.011858702,0.013526454,-0.049580045,-0.052486703,0.013218615,0.0050231596,0.07660412,-0.0046343356,-0.0692584,-0.03353194,0.022288544,0.022836983,0.01361394,-0.036331616,0.008194689,0.09226991,0.08229875,-0.047639843,-0.005033471,0.0066081914,-0.106666416,-0.06385503,0.006385333,0.06318358,0.03146985,-0.060764216,-0.043287244,-0.042385958,-0.07963737,-0.043478705,-0.031840656,0.060027804,0.027829666,0.03728562,-0.07014622,0.0649568,-0.052579004,-0.0034938955,-0.15390792,-0.04426162,0.028217599,0.05593145,0.023251446,0.029115746,-0.040321343,-1.8832651e-08,-0.037536032,0.0005056563,0.013919978,-0.038129315,0.061486673,0.005534046,-0.010768211,-0.16527015,0.061414365,0.019963786,-0.019666849,-0.059065994,0.00083181454,-0.016695915,0.01090875,0.02685501,-0.0008743374,-0.060967337,-0.049258173,0.012731965,-0.03973743,-0.0018210232,-0.03797001,0.00035104176,0.04477184,0.04185603,-0.03215796,-0.09103283,0.0467481,0.017810425,0.037437033,0.054782324,-0.0059442874,0.022557084,0.053393807,-0.03746794,-0.07080435,0.10548834,-0.04215435,0.042547524,0.05064807,0.07560324,0.04072184,-0.0011278372,0.00702018,-0.070807114,0.040035766,0.051417697,0.011316928,-0.028955372,0.04197099,-0.0026867494,-0.056857754,0.0761941,0.030621601,-0.052413527,-0.021492323,0.0957355,0.1013111,-0.082977295,-0.036080915,-0.03327806,0.0029404552,-0.03586191]'::vector)", "Filter": "(cohort = 'static'::text)" } ] } } ] }, "count_before": 100200, "idle_duration_s": 10.0, "idle_queries": { "count": 100, "ann_recall": 0.993, "ndcg": 0.7931268675492565, "latency_ms": { "min": 5.090042017400265, "p50": 16.38441701652482, "p95": 20.190355990780514, "p99": 22.506467908388014, "max": 24.569915956817567 } }, "exact_ndcg": 0.7931268675492565, "resources_before": { "usage_usec": 104193530, "user_usec": 100299271, "system_usec": 3894259, "nice_usec": 0, "nr_periods": 1271, "nr_throttled": 437, "throttled_usec": 45858291, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 1155100672, "pids": 23 }, "resources_after": { "usage_usec": 114992037, "user_usec": 107524928, "system_usec": 7467108, "nice_usec": 0, "nr_periods": 1572, "nr_throttled": 437, "throttled_usec": 45858291, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 1174286336, "pids": 30 }, "count_after": 100200, "backend_after": { "version": "0.8.7", "postgres": "PostgreSQL 16.15 (Debian 16.15-1.pgdg12+2) on aarch64-unknown-linux-gnu, compiled by gcc (Debian 12.2.0-14+deb12u1) 12.2.0, 64-bit", "ef_search": 128, "ack": "autocommit, READ COMMITTED", "transport": "psycopg, connection per worker", "query_plan": [ { "Plan": { "Node Type": "Limit", "Parallel Aware": false, "Async Capable": false, "Startup Cost": 1675.79, "Total Cost": 1692.91, "Plan Rows": 10, "Plan Width": 18, "Plans": [ { "Node Type": "Index Scan", "Parent Relationship": "Outer", "Parallel Aware": false, "Async Capable": false, "Scan Direction": "Forward", "Index Name": "fb2_429e9a_048_hnsw", "Relation Name": "fb2_429e9a_048", "Alias": "fb2_429e9a_048", "Startup Cost": 1675.79, "Total Cost": 173144.54, "Plan Rows": 100103, "Plan Width": 18, "Order By": "(embedding <=> '[-0.08946895,-0.094362676,0.092023626,-0.0040844884,0.043233823,0.00062519487,0.1075197,-0.035732996,0.013746996,-0.024103122,0.0021706754,-0.04720602,0.04787615,-0.02043222,0.058315646,0.081252135,-0.05375015,-0.030799072,-0.0077205584,0.035735283,-0.04474784,0.01614922,-0.06905586,0.04194401,-0.08832908,0.007873556,0.0044992934,0.021968655,0.010953364,0.07212316,0.023185596,0.001176476,0.04725166,-0.03580423,-0.032344446,0.004752835,-0.056179125,-0.0353235,-0.037251893,-0.005143068,-0.027599342,0.024403147,0.031338032,0.0056410627,0.008425455,-0.0087741045,-0.039870497,-0.011831836,0.042621233,0.048142947,0.01603842,0.06745542,-0.038018145,0.064212486,-0.009636566,0.01607475,-0.0027868534,-0.03761097,-0.011889431,-0.01084577,0.083403684,0.028946918,0.061870236,-0.045382615,0.03812586,0.060426526,-0.017003024,0.02328925,-0.093020774,-0.024072766,0.042720392,-0.022444837,-0.011597358,0.01209231,-0.005953629,0.029691145,-0.011711495,-0.053345133,-0.081757374,-0.049303155,0.032193717,-0.038507868,-0.0060101775,-0.031470332,-0.07362443,0.03575578,0.0245484,0.01985519,-0.15116754,0.050386053,-0.0389634,0.07548389,-0.09264168,-0.012602816,-0.0035539444,-0.046416666,0.015100493,0.100691915,-0.037529,0.04600518,-0.021859422,0.014675989,-0.097722545,-0.053200893,-0.017040461,-0.01629616,-0.012339222,0.06543302,0.07252074,-0.044301156,0.03054835,-0.038570553,0.019310065,-0.005527747,-0.029386865,-0.058248505,0.065443166,0.028506057,0.0878254,0.058387812,0.081721246,-0.0005962058,0.0569388,0.00960514,0.011667701,0.014456492,-0.07218306,-3.0444767e-33,0.06687367,-0.019213954,0.058202963,0.025171444,-0.028259544,-0.04171671,0.03127722,0.04865431,-0.0721459,-0.050143186,-0.029907746,0.06248994,-0.05096076,-0.065192886,0.010210447,-0.077256374,-0.043370593,-0.0901922,-0.066522665,0.026072454,-0.022315364,0.075375885,-0.029968103,-0.0050771856,-0.0046955296,0.0065650316,-0.06791897,0.07368851,0.056069646,-0.021026025,-0.011352883,0.038811427,-0.090847075,-0.056490287,-0.039881036,0.032885592,-0.029586185,0.04753608,0.006341778,-0.0034275919,0.03681623,0.09326265,-0.0532089,0.032396164,0.012605483,0.045778517,-0.0006472902,0.029721813,0.0734947,-0.037117425,-0.09356724,-0.0071634455,-0.032400057,0.010453378,0.06286513,0.006053158,0.060524322,0.091866635,0.063640274,0.055919155,0.053222507,-0.019174587,0.089296214,-0.058922265,0.013745364,-0.022907477,0.028347451,-0.040759467,-0.06820364,-0.061398294,-0.046857864,0.0060627847,0.010442015,-0.07662317,0.031950947,-0.107813306,-0.0016125626,-0.033413693,-0.09989722,-0.03580293,-0.036118146,-0.08276185,0.1280713,0.07596603,0.049225595,-0.026312606,0.0031930343,-0.0012469466,0.000872793,-0.040353082,-0.034516506,-0.036047976,0.004999689,-0.01570079,-0.09783203,1.4186912e-33,-0.0130903,0.04742377,0.02889633,0.09809088,0.043639127,-0.0747531,0.03402747,0.029147558,0.019848268,0.111865185,0.04845458,-0.016222177,0.025501896,0.09813583,-0.002833195,0.0038929228,0.048537854,0.014111116,0.036570966,-0.017512968,-0.079755045,0.043452248,0.10843892,0.061968718,-0.015678233,0.0060376576,0.010718569,-0.03907552,0.040307663,0.04553227,0.041731495,0.083179526,-0.0844024,-0.023447752,0.014485363,-0.00788756,-0.11410207,0.039309476,0.011220049,0.12889338,-0.07970701,0.010658111,-0.06657531,-0.022037188,-0.043459363,-0.030291978,0.05803613,0.06109021,-0.09415837,0.030727232,0.011858702,0.013526454,-0.049580045,-0.052486703,0.013218615,0.0050231596,0.07660412,-0.0046343356,-0.0692584,-0.03353194,0.022288544,0.022836983,0.01361394,-0.036331616,0.008194689,0.09226991,0.08229875,-0.047639843,-0.005033471,0.0066081914,-0.106666416,-0.06385503,0.006385333,0.06318358,0.03146985,-0.060764216,-0.043287244,-0.042385958,-0.07963737,-0.043478705,-0.031840656,0.060027804,0.027829666,0.03728562,-0.07014622,0.0649568,-0.052579004,-0.0034938955,-0.15390792,-0.04426162,0.028217599,0.05593145,0.023251446,0.029115746,-0.040321343,-1.8832651e-08,-0.037536032,0.0005056563,0.013919978,-0.038129315,0.061486673,0.005534046,-0.010768211,-0.16527015,0.061414365,0.019963786,-0.019666849,-0.059065994,0.00083181454,-0.016695915,0.01090875,0.02685501,-0.0008743374,-0.060967337,-0.049258173,0.012731965,-0.03973743,-0.0018210232,-0.03797001,0.00035104176,0.04477184,0.04185603,-0.03215796,-0.09103283,0.0467481,0.017810425,0.037437033,0.054782324,-0.0059442874,0.022557084,0.053393807,-0.03746794,-0.07080435,0.10548834,-0.04215435,0.042547524,0.05064807,0.07560324,0.04072184,-0.0011278372,0.00702018,-0.070807114,0.040035766,0.051417697,0.011316928,-0.028955372,0.04197099,-0.0026867494,-0.056857754,0.0761941,0.030621601,-0.052413527,-0.021492323,0.0957355,0.1013111,-0.082977295,-0.036080915,-0.03327806,0.0029404552,-0.03586191]'::vector)", "Filter": "(cohort = 'static'::text)" } ] } } ] } }, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "eligible": true, "protocol": "mixed-mutations-v2" }, "examples": [ { "event": 0, "operation": "insert", "doc_id": "mutation-000000", "scheduled_s": 0.0, "submitted_s": 0.00016458297614008188, "schedule_delay_ms": 0.16458297614008188, "precondition": true, "status": "success", "ack_ms": 20.788958063349128, "write_started_ms": 0.7140421075746417, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 24.158792104572058, "stable_since_ms": 24.158792104572058, "confirmed_ms": 76.36733306571841, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 30.881833052262664, "stable_since_ms": 30.881833052262664, "confirmed_ms": 87.60787500068545, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 30.881833052262664, "confirmed_ms": 87.60787500068545, "search_minus_ack_ms": 10.092874988913536 }, { "event": 1, "operation": "update", "doc_id": "mutation-000001", "scheduled_s": 0.1, "submitted_s": 0.10016562498640269, "schedule_delay_ms": 0.16562498640268486, "precondition": true, "status": "success", "ack_ms": 6.962415995076299, "write_started_ms": 0.34187501296401024, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 12.303791008889675, "stable_since_ms": 12.303791008889675, "confirmed_ms": 64.42795810289681, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 18.00954109057784, "stable_since_ms": 18.00954109057784, "confirmed_ms": 74.04329103883356, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 18.00954109057784, "confirmed_ms": 74.04329103883356, "search_minus_ack_ms": 11.047125095501542 }, { "event": 2, "operation": "delete", "doc_id": "mutation-000002", "scheduled_s": 0.2, "submitted_s": 0.2004040409810841, "schedule_delay_ms": 0.40404098108409725, "precondition": true, "status": "success", "ack_ms": 3.284167032688856, "write_started_ms": 0.33833400812000036, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 12.423584004864097, "stable_since_ms": 12.423584004864097, "confirmed_ms": 65.0205840356648, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 19.81712505221367, "stable_since_ms": 19.81712505221367, "confirmed_ms": 82.47487503103912, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 19.81712505221367, "confirmed_ms": 82.47487503103912, "search_minus_ack_ms": 16.532958019524813 } ] }, { "run": "fb2_429e9a_049", "trial": 3, "backend": "weaviate", "scenario": "nq-100k-burst", "status": "complete", "eligible": true, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "summary": { "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "burst", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "wall_s": 30.00133837503381, "operations": { "insert": { "count": 130, "success": 130, "success_rate": 1.0, "first_observed_ms": { "min": 3.575249924324453, "p50": 16.136479505803436, "p95": 31.74527310184203, "p99": 35.53969660308211, "max": 37.14641695842147 }, "confirmed_ms": { "min": 62.151749967597425, "p50": 82.29299995582551, "p95": 103.77472082618624, "p99": 109.96505391667593, "max": 112.17687500175089 }, "ack_ms": { "min": 2.85649998113513, "p50": 9.444895957130939, "p95": 13.880439178319648, "p99": 16.187825346132737, "max": 22.797125042416155 }, "search_minus_ack_ms": { "min": -0.2797089982777834, "p50": 1.7157704569399357, "p95": 18.198970920639113, "p99": 21.432591694174334, "max": 24.924000026658177 }, "regressions": 0, "confirmed_within_100ms": 0.9230769230769231, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "update": { "count": 130, "success": 130, "success_rate": 1.0, "first_observed_ms": { "min": 3.561749937944114, "p50": 17.00904202880338, "p95": 32.50151912216097, "p99": 34.512623695190996, "max": 34.63204100262374 }, "confirmed_ms": { "min": 61.17925001308322, "p50": 82.59066654136404, "p95": 103.8196001551114, "p99": 106.59792295075022, "max": 107.02512494754046 }, "ack_ms": { "min": 2.89725000038743, "p50": 9.963020973373204, "p95": 15.126502572093159, "p99": 16.218609908828512, "max": 16.609207959845662 }, "search_minus_ack_ms": { "min": -2.5158750358968973, "p50": 1.7409789725206792, "p95": 19.40096002072096, "p99": 20.514249710831788, "max": 20.81308397464454 }, "regressions": 0, "confirmed_within_100ms": 0.8538461538461538, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "delete": { "count": 130, "success": 130, "success_rate": 1.0, "first_observed_ms": { "min": 4.370542010292411, "p50": 15.302417043130845, "p95": 20.836546184727922, "p99": 26.365203092573182, "max": 31.371917109936476 }, "confirmed_ms": { "min": 64.2175420653075, "p50": 79.71550000365824, "p95": 85.46466057305224, "p99": 92.71211659419346, "max": 105.40804208721966 }, "ack_ms": { "min": 1.4703749911859632, "p50": 4.9188334960490465, "p95": 9.143500000936909, "p99": 11.617238403996444, "max": 14.65058303438127 }, "search_minus_ack_ms": { "min": 0.1867079408839345, "p50": 5.475271085742861, "p95": 9.866393404081458, "p99": 16.785419658990573, "max": 19.365583080798388 }, "regressions": 0, "confirmed_within_100ms": 0.9923076923076923, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 } }, "total_operations": 390, "statuses": { "success": 390, "timeout": 0, "error": 0, "invalid_precondition": 0, "client_overload": 0 }, "query": { "count": 300, "success": 300, "ann_recall_at_10": 0.9920000000000001, "ndcg_at_10": 0.7931268675492565, "latency_ms": { "min": 3.18404205609113, "p50": 10.751708527095616, "p95": 15.96772765624337, "p99": 20.207165026804418, "max": 22.99225004389882 }, "dispatch_delay_ms": { "min": 0.09570806287229061, "p50": 1.3802085304632783, "p95": 2.362322574481368, "p99": 2.845697177108376, "max": 3.6158330040052533 } }, "diagnostics": { "schedule_delay_ms": { "min": 0.004707998596309437, "p50": 0.31429147347861175, "p95": 0.6070768751666121, "p99": 0.6467726139831599, "max": 1.0886669857427256 }, "probe_dispatch_ms": { "min": 0.02604094333946705, "p50": 0.4185829311609268, "p95": 1.8494125106371933, "p99": 2.7468283590860665, "max": 3.9520409191027284 }, "probe_service_ms": { "min": 0.6514580454677343, "p50": 4.1987079894170165, "p95": 11.646133416797966, "p99": 13.912416873499751, "max": 21.98970806784928 }, "write_service_ms": { "min": 1.2799999676644802, "p50": 5.953978979960084, "p95": 13.535927474731569, "p99": 15.337494919076562, "max": 22.169583942741156 } }, "actual_submission_rate": 13.009979308927694, "metadata": { "backend": "weaviate", "collection": "fb2_429e9a_049", "dataset": { "dataset": "nq", "source": "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/nq.zip", "source_sha256": "2553bf7bfbab47b1436ca00a34bce57320e18e611fd00999a2a3a1b4714be752", "source_md5": "d4d3d2e48787a744b6f6e691ff534307", "records": 124000, "queries": 100, "relevant_documents_retained": 120, "seed": 20261028, "license": "CC-BY-SA-3.0", "license_source": "https://raw.githubusercontent.com/google-research-datasets/natural-questions/master/nq_open/README.md", "attribution": "BEIR and original dataset authors; Wikipedia contributors for NQ, SciFact authors and cited article authors for SciFact.", "transformation": "200-word title+text prefix; seeded reservoir of documents with relevant documents for selected test queries retained at front. Simulated mutations; no historical replay.", "model": "sentence-transformers/all-MiniLM-L6-v2", "model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "dimensions": 384, "normalized": true, "max_tokens": 256, "device": "mps", "files": { "corpus.jsonl": "9dc487d785e6ee493eccaf764daa3bf8720a8f4fc8c406246065e1ad33a91f5d", "queries.json": "5603f52876ada5d36118b65dea490f1d43900bc159e1804e637796eb9115e999", "vectors.npy": "cb7421eb6ca5964231403f0a922843088f6fd0e01cbf67f2a1076cc7101005db", "query_vectors.npy": "5e666f8d966d8db6b0fd93d4963234d2dffa0ee3910b8fad069a8ee201e8b5d4", "UPSTREAM-LICENSE.md": "8b675521da3fa142ac6bda38c9bcfcc8cff0e2d6128be53a062ed55fb53f7b87" } }, "base_count": 100000, "profile": { "dataset": "nq", "base_count": 100000, "rate": 10, "query_rate": 10, "arrival": "burst" }, "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "burst", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "preloaded_mutable": 260, "started_utc": "2026-10-29T15:45:25Z", "setup_s": 36.8292039160151, "backend_before": { "version": "1.40.3", "schema": { "class": "Ffb2429e9a049", "invertedIndexConfig": { "bm25": { "b": 0.75, "k1": 1.2 }, "cleanupIntervalSeconds": 60, "stopwords": { "additions": null, "preset": "en", "removals": null }, "usingBlockMaxWAND": true }, "multiTenancyConfig": { "autoTenantActivation": false, "autoTenantCreation": false, "enabled": false }, "properties": [ { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "doc_id", "tokenization": "field" }, { "dataType": [ "int" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": false, "name": "version" }, { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "cohort", "tokenization": "field" }, { "dataType": [ "text" ], "indexFilterable": false, "indexRangeFilters": false, "indexSearchable": false, "name": "text", "tokenization": "word" } ], "shardingConfig": { "actualCount": 1, "actualVirtualCount": 128, "desiredCount": 1, "desiredVirtualCount": 128, "function": "murmur3", "key": "_id", "strategy": "hash", "virtualPerPhysical": 128 }, "vectorIndexConfig": { "bq": { "enabled": false }, "cleanupIntervalSeconds": 300, "distance": "cosine", "dynamicEfFactor": 8, "dynamicEfMax": 500, "dynamicEfMin": 100, "ef": 128, "efConstruction": 128, "filterStrategy": "acorn", "flatSearchCutoff": 0, "maxConnections": 16, "multivector": { "aggregation": "maxSim", "enabled": false, "muvera": { "dprojections": 16, "enabled": false, "ksim": 4, "repetitions": 10 } }, "pq": { "bitCompression": false, "centroids": 256, "enabled": false, "encoder": { "distribution": "log-normal", "type": "kmeans" }, "segments": 0, "trainingLimit": 100000 }, "rq": { "bits": 8, "centering": false, "enabled": false, "rescoreLimit": 20, "trainingLimit": 10000 }, "skip": false, "skipDefaultQuantization": false, "sq": { "enabled": false, "rescoreLimit": 20, "trainingLimit": 100000 }, "trackDefaultQuantization": false, "vectorCacheMaxObjects": 1000000000000 }, "vectorIndexType": "hnsw", "vectorizer": "none", "replicationConfig": { "deletionStrategy": "TimeBasedResolution", "factor": 1, "asyncEnabled": false } }, "shards": [ { "name": "1szdsA3C1JHM", "per_node_status": { "node1": "READY" }, "status": "READY", "vectorQueueSize": 0 } ], "ack": "synchronous indexing, object PUT/POST/DELETE", "transport": "HTTP REST writes + GraphQL search, pooled httpx" }, "count_before": 100260, "idle_duration_s": 10.0, "idle_queries": { "count": 100, "ann_recall": 0.992, "ndcg": 0.7931268675492565, "latency_ms": { "min": 4.204040975309908, "p50": 11.1640834948048, "p95": 13.159548112889752, "p99": 13.600842393934728, "max": 14.06370906624943 } }, "exact_ndcg": 0.7931268675492565, "resources_before": { "usage_usec": 101106129, "user_usec": 98935129, "system_usec": 2170999, "nice_usec": 0, "nr_periods": 2152, "nr_throttled": 148, "throttled_usec": 378705, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 1203044352, "pids": 11 }, "resources_after": { "usage_usec": 111076629, "user_usec": 108444169, "system_usec": 2632460, "nice_usec": 0, "nr_periods": 2453, "nr_throttled": 148, "throttled_usec": 378705, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 1155043328, "pids": 11 }, "count_after": 100260, "backend_after": { "version": "1.40.3", "schema": { "class": "Ffb2429e9a049", "invertedIndexConfig": { "bm25": { "b": 0.75, "k1": 1.2 }, "cleanupIntervalSeconds": 60, "stopwords": { "additions": null, "preset": "en", "removals": null }, "usingBlockMaxWAND": true }, "multiTenancyConfig": { "autoTenantActivation": false, "autoTenantCreation": false, "enabled": false }, "properties": [ { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "doc_id", "tokenization": "field" }, { "dataType": [ "int" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": false, "name": "version" }, { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "cohort", "tokenization": "field" }, { "dataType": [ "text" ], "indexFilterable": false, "indexRangeFilters": false, "indexSearchable": false, "name": "text", "tokenization": "word" } ], "shardingConfig": { "actualCount": 1, "actualVirtualCount": 128, "desiredCount": 1, "desiredVirtualCount": 128, "function": "murmur3", "key": "_id", "strategy": "hash", "virtualPerPhysical": 128 }, "vectorIndexConfig": { "bq": { "enabled": false }, "cleanupIntervalSeconds": 300, "distance": "cosine", "dynamicEfFactor": 8, "dynamicEfMax": 500, "dynamicEfMin": 100, "ef": 128, "efConstruction": 128, "filterStrategy": "acorn", "flatSearchCutoff": 0, "maxConnections": 16, "multivector": { "aggregation": "maxSim", "enabled": false, "muvera": { "dprojections": 16, "enabled": false, "ksim": 4, "repetitions": 10 } }, "pq": { "bitCompression": false, "centroids": 256, "enabled": false, "encoder": { "distribution": "log-normal", "type": "kmeans" }, "segments": 0, "trainingLimit": 100000 }, "rq": { "bits": 8, "centering": false, "enabled": false, "rescoreLimit": 20, "trainingLimit": 10000 }, "skip": false, "skipDefaultQuantization": false, "sq": { "enabled": false, "rescoreLimit": 20, "trainingLimit": 100000 }, "trackDefaultQuantization": false, "vectorCacheMaxObjects": 1000000000000 }, "vectorIndexType": "hnsw", "vectorizer": "none", "replicationConfig": { "deletionStrategy": "TimeBasedResolution", "factor": 1, "asyncEnabled": false } }, "shards": [ { "name": "1szdsA3C1JHM", "per_node_status": { "node1": "READY" }, "status": "READY", "vectorQueueSize": 0 } ], "ack": "synchronous indexing, object PUT/POST/DELETE", "transport": "HTTP REST writes + GraphQL search, pooled httpx" } }, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "eligible": true, "protocol": "mixed-mutations-v2" }, "examples": [ { "event": 0, "operation": "insert", "doc_id": "mutation-000000", "scheduled_s": 0.0, "submitted_s": 0.00020820798818022013, "schedule_delay_ms": 0.20820798818022013, "precondition": true, "status": "success", "ack_ms": 5.654292064718902, "write_started_ms": 1.3191669713705778, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 15.491749974898994, "stable_since_ms": 15.491749974898994, "confirmed_ms": 67.8407090017572, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 5.952625069767237, "stable_since_ms": 5.952625069767237, "confirmed_ms": 64.2614170210436, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 15.491749974898994, "confirmed_ms": 67.8407090017572, "search_minus_ack_ms": 0.2983330050483346 }, { "event": 1, "operation": "update", "doc_id": "mutation-000001", "scheduled_s": 0.025, "submitted_s": 0.025444458005949855, "schedule_delay_ms": 0.44445800594985346, "precondition": true, "status": "success", "ack_ms": 3.2432919833809137, "write_started_ms": 0.2508750185370445, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 12.707917019724846, "stable_since_ms": 12.707917019724846, "confirmed_ms": 65.32166700344533, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 4.222749965265393, "stable_since_ms": 4.222749965265393, "confirmed_ms": 60.05412503145635, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 12.707917019724846, "confirmed_ms": 65.32166700344533, "search_minus_ack_ms": 0.9794579818844795 }, { "event": 2, "operation": "delete", "doc_id": "mutation-000002", "scheduled_s": 0.05, "submitted_s": 0.050591500010341406, "schedule_delay_ms": 0.5915000103414031, "precondition": true, "status": "success", "ack_ms": 2.0790830021724105, "write_started_ms": 0.2150830114260316, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 13.975457986816764, "stable_since_ms": 13.975457986816764, "confirmed_ms": 66.47716695442796, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 4.525374970398843, "stable_since_ms": 4.525374970398843, "confirmed_ms": 61.4406670210883, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 13.975457986816764, "confirmed_ms": 66.47716695442796, "search_minus_ack_ms": 2.446291968226433 } ] }, { "run": "fb2_429e9a_050", "trial": 5, "backend": "weaviate", "scenario": "nq-100k", "status": "complete", "eligible": true, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "summary": { "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "wall_s": 30.007160458015278, "operations": { "insert": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 7.582332938909531, "p50": 18.527687585446984, "p95": 32.879156223498285, "p99": 34.36869620578364, "max": 36.104083992540836 }, "confirmed_ms": { "min": 67.11266690399498, "p50": 87.05427101813257, "p95": 105.1834607613273, "p99": 109.26434070337568, "max": 116.56624998431653 }, "ack_ms": { "min": 5.353041924536228, "p50": 11.103229073341936, "p95": 16.183926799567416, "p99": 18.345103259198392, "max": 23.317708051763475 }, "search_minus_ack_ms": { "min": -0.8265409851446748, "p50": 4.4691250077448785, "p95": 20.14877150650136, "p99": 21.168380037415776, "max": 21.552499965764582 }, "regressions": 0, "confirmed_within_100ms": 0.85, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "update": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 6.081916042603552, "p50": 17.910562513861805, "p95": 32.30716179823503, "p99": 34.47358368779534, "max": 36.86608304269612 }, "confirmed_ms": { "min": 66.02825003210455, "p50": 86.8002709466964, "p95": 104.0129562432412, "p99": 106.95687001920305, "max": 107.21625003498048 }, "ack_ms": { "min": 3.3905000891536474, "p50": 10.819916555192322, "p95": 15.786966384621337, "p99": 19.011120839277286, "max": 20.000708056613803 }, "search_minus_ack_ms": { "min": 0.09008299093693495, "p50": 4.634333017747849, "p95": 19.455889891833067, "p99": 20.83982626441866, "max": 21.156749920919538 }, "regressions": 0, "confirmed_within_100ms": 0.84, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "delete": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 6.306000053882599, "p50": 16.551187552977353, "p95": 20.256677718134597, "p99": 28.669927850132833, "max": 33.65812508855015 }, "confirmed_ms": { "min": 65.59329200536013, "p50": 80.577083514072, "p95": 86.7377937247511, "p99": 102.31957627693193, "max": 105.02487502526492 }, "ack_ms": { "min": 2.110125031322241, "p50": 5.019500502385199, "p95": 11.973410466453059, "p99": 13.465448033530267, "max": 13.810957898385823 }, "search_minus_ack_ms": { "min": 0.12545904610306025, "p50": 5.851374997291714, "p95": 7.698697870364411, "p99": 18.546744767809294, "max": 20.19616705365479 }, "regressions": 0, "confirmed_within_100ms": 0.98, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 } }, "total_operations": 300, "statuses": { "success": 300, "timeout": 0, "error": 0, "invalid_precondition": 0, "client_overload": 0 }, "query": { "count": 300, "success": 300, "ann_recall_at_10": 0.99, "ndcg_at_10": 0.7931268675492565, "latency_ms": { "min": 4.455249989405274, "p50": 11.219853477086872, "p95": 16.115874663228173, "p99": 19.393333219923072, "max": 22.603790974244475 }, "dispatch_delay_ms": { "min": 0.05795795004814863, "p50": 1.4069374301470816, "p95": 2.38628534716554, "p99": 2.644472040701656, "max": 3.257500007748604 } }, "diagnostics": { "schedule_delay_ms": { "min": 0.007082964293303462, "p50": 0.3181454492739988, "p95": 0.6072686950209462, "p99": 0.6388941626057587, "max": 0.6503749638788747 }, "probe_dispatch_ms": { "min": 0.03208394628018141, "p50": 0.43883296893909574, "p95": 2.0042437012307337, "p99": 2.7254147233907147, "max": 3.4886669600382447 }, "probe_service_ms": { "min": 0.879166997037828, "p50": 5.279103992506862, "p95": 12.023010622942818, "p99": 14.958054299932018, "max": 25.522832991555333 }, "write_service_ms": { "min": 1.935041043907404, "p50": 8.944812521804124, "p95": 14.618821133626625, "p99": 18.120267367921766, "max": 22.838916047476232 } }, "actual_submission_rate": 9.999912737580521, "metadata": { "backend": "weaviate", "collection": "fb2_429e9a_050", "dataset": { "dataset": "nq", "source": "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/nq.zip", "source_sha256": "2553bf7bfbab47b1436ca00a34bce57320e18e611fd00999a2a3a1b4714be752", "source_md5": "d4d3d2e48787a744b6f6e691ff534307", "records": 124000, "queries": 100, "relevant_documents_retained": 120, "seed": 20261028, "license": "CC-BY-SA-3.0", "license_source": "https://raw.githubusercontent.com/google-research-datasets/natural-questions/master/nq_open/README.md", "attribution": "BEIR and original dataset authors; Wikipedia contributors for NQ, SciFact authors and cited article authors for SciFact.", "transformation": "200-word title+text prefix; seeded reservoir of documents with relevant documents for selected test queries retained at front. Simulated mutations; no historical replay.", "model": "sentence-transformers/all-MiniLM-L6-v2", "model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "dimensions": 384, "normalized": true, "max_tokens": 256, "device": "mps", "files": { "corpus.jsonl": "9dc487d785e6ee493eccaf764daa3bf8720a8f4fc8c406246065e1ad33a91f5d", "queries.json": "5603f52876ada5d36118b65dea490f1d43900bc159e1804e637796eb9115e999", "vectors.npy": "cb7421eb6ca5964231403f0a922843088f6fd0e01cbf67f2a1076cc7101005db", "query_vectors.npy": "5e666f8d966d8db6b0fd93d4963234d2dffa0ee3910b8fad069a8ee201e8b5d4", "UPSTREAM-LICENSE.md": "8b675521da3fa142ac6bda38c9bcfcc8cff0e2d6128be53a062ed55fb53f7b87" } }, "base_count": 100000, "profile": { "dataset": "nq", "base_count": 100000, "rate": 10, "query_rate": 10, "arrival": "steady" }, "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "preloaded_mutable": 200, "started_utc": "2026-10-29T15:46:47Z", "setup_s": 37.194705249974504, "backend_before": { "version": "1.40.3", "schema": { "class": "Ffb2429e9a050", "invertedIndexConfig": { "bm25": { "b": 0.75, "k1": 1.2 }, "cleanupIntervalSeconds": 60, "stopwords": { "additions": null, "preset": "en", "removals": null }, "usingBlockMaxWAND": true }, "multiTenancyConfig": { "autoTenantActivation": false, "autoTenantCreation": false, "enabled": false }, "properties": [ { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "doc_id", "tokenization": "field" }, { "dataType": [ "int" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": false, "name": "version" }, { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "cohort", "tokenization": "field" }, { "dataType": [ "text" ], "indexFilterable": false, "indexRangeFilters": false, "indexSearchable": false, "name": "text", "tokenization": "word" } ], "shardingConfig": { "actualCount": 1, "actualVirtualCount": 128, "desiredCount": 1, "desiredVirtualCount": 128, "function": "murmur3", "key": "_id", "strategy": "hash", "virtualPerPhysical": 128 }, "vectorIndexConfig": { "bq": { "enabled": false }, "cleanupIntervalSeconds": 300, "distance": "cosine", "dynamicEfFactor": 8, "dynamicEfMax": 500, "dynamicEfMin": 100, "ef": 128, "efConstruction": 128, "filterStrategy": "acorn", "flatSearchCutoff": 0, "maxConnections": 16, "multivector": { "aggregation": "maxSim", "enabled": false, "muvera": { "dprojections": 16, "enabled": false, "ksim": 4, "repetitions": 10 } }, "pq": { "bitCompression": false, "centroids": 256, "enabled": false, "encoder": { "distribution": "log-normal", "type": "kmeans" }, "segments": 0, "trainingLimit": 100000 }, "rq": { "bits": 8, "centering": false, "enabled": false, "rescoreLimit": 20, "trainingLimit": 10000 }, "skip": false, "skipDefaultQuantization": false, "sq": { "enabled": false, "rescoreLimit": 20, "trainingLimit": 100000 }, "trackDefaultQuantization": false, "vectorCacheMaxObjects": 1000000000000 }, "vectorIndexType": "hnsw", "vectorizer": "none", "replicationConfig": { "deletionStrategy": "TimeBasedResolution", "factor": 1, "asyncEnabled": false } }, "shards": [ { "name": "i5PhE2IOds82", "per_node_status": { "node1": "READY" }, "status": "READY", "vectorQueueSize": 0 } ], "ack": "synchronous indexing, object PUT/POST/DELETE", "transport": "HTTP REST writes + GraphQL search, pooled httpx" }, "count_before": 100200, "idle_duration_s": 10.0, "idle_queries": { "count": 100, "ann_recall": 0.99, "ndcg": 0.7931268675492565, "latency_ms": { "min": 4.1582080302760005, "p50": 11.6215834277682, "p95": 13.544200599426404, "p99": 13.718244971241802, "max": 13.767249998636544 } }, "exact_ndcg": 0.7931268675492565, "resources_before": { "usage_usec": 186546302, "user_usec": 182878010, "system_usec": 3668292, "nice_usec": 0, "nr_periods": 2956, "nr_throttled": 269, "throttled_usec": 620446, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 1378226176, "pids": 11 }, "resources_after": { "usage_usec": 196354505, "user_usec": 192174037, "system_usec": 4180467, "nice_usec": 0, "nr_periods": 3258, "nr_throttled": 269, "throttled_usec": 620446, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 1233932288, "pids": 11 }, "count_after": 100200, "backend_after": { "version": "1.40.3", "schema": { "class": "Ffb2429e9a050", "invertedIndexConfig": { "bm25": { "b": 0.75, "k1": 1.2 }, "cleanupIntervalSeconds": 60, "stopwords": { "additions": null, "preset": "en", "removals": null }, "usingBlockMaxWAND": true }, "multiTenancyConfig": { "autoTenantActivation": false, "autoTenantCreation": false, "enabled": false }, "properties": [ { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "doc_id", "tokenization": "field" }, { "dataType": [ "int" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": false, "name": "version" }, { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "cohort", "tokenization": "field" }, { "dataType": [ "text" ], "indexFilterable": false, "indexRangeFilters": false, "indexSearchable": false, "name": "text", "tokenization": "word" } ], "shardingConfig": { "actualCount": 1, "actualVirtualCount": 128, "desiredCount": 1, "desiredVirtualCount": 128, "function": "murmur3", "key": "_id", "strategy": "hash", "virtualPerPhysical": 128 }, "vectorIndexConfig": { "bq": { "enabled": false }, "cleanupIntervalSeconds": 300, "distance": "cosine", "dynamicEfFactor": 8, "dynamicEfMax": 500, "dynamicEfMin": 100, "ef": 128, "efConstruction": 128, "filterStrategy": "acorn", "flatSearchCutoff": 0, "maxConnections": 16, "multivector": { "aggregation": "maxSim", "enabled": false, "muvera": { "dprojections": 16, "enabled": false, "ksim": 4, "repetitions": 10 } }, "pq": { "bitCompression": false, "centroids": 256, "enabled": false, "encoder": { "distribution": "log-normal", "type": "kmeans" }, "segments": 0, "trainingLimit": 100000 }, "rq": { "bits": 8, "centering": false, "enabled": false, "rescoreLimit": 20, "trainingLimit": 10000 }, "skip": false, "skipDefaultQuantization": false, "sq": { "enabled": false, "rescoreLimit": 20, "trainingLimit": 100000 }, "trackDefaultQuantization": false, "vectorCacheMaxObjects": 1000000000000 }, "vectorIndexType": "hnsw", "vectorizer": "none", "replicationConfig": { "deletionStrategy": "TimeBasedResolution", "factor": 1, "asyncEnabled": false } }, "shards": [ { "name": "i5PhE2IOds82", "per_node_status": { "node1": "READY" }, "status": "READY", "vectorQueueSize": 0 } ], "ack": "synchronous indexing, object PUT/POST/DELETE", "transport": "HTTP REST writes + GraphQL search, pooled httpx" } }, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "eligible": true, "protocol": "mixed-mutations-v2" }, "examples": [ { "event": 0, "operation": "insert", "doc_id": "mutation-000000", "scheduled_s": 0.0, "submitted_s": 0.0002151250373572111, "schedule_delay_ms": 0.2151250373572111, "precondition": true, "status": "success", "ack_ms": 5.353041924536228, "write_started_ms": 1.4268329832702875, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 14.32324992492795, "stable_since_ms": 14.32324992492795, "confirmed_ms": 67.11266690399498, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 5.955750006251037, "stable_since_ms": 5.955750006251037, "confirmed_ms": 66.45920791197568, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 14.32324992492795, "confirmed_ms": 67.11266690399498, "search_minus_ack_ms": 0.6027080817148089 }, { "event": 1, "operation": "update", "doc_id": "mutation-000001", "scheduled_s": 0.1, "submitted_s": 0.10011450003366917, "schedule_delay_ms": 0.11450003366916817, "precondition": true, "status": "success", "ack_ms": 8.448291919194162, "write_started_ms": 0.3725419519469142, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 8.531041909009218, "stable_since_ms": 8.531041909009218, "confirmed_ms": 62.32654198538512, "consecutive": 3, "regressions": 0, "requests": 3 }, "search": { "first_ms": 25.149041903205216, "stable_since_ms": 25.149041903205216, "confirmed_ms": 92.06362499389797, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 25.149041903205216, "confirmed_ms": 92.06362499389797, "search_minus_ack_ms": 16.700749984011054 }, { "event": 2, "operation": "delete", "doc_id": "mutation-000002", "scheduled_s": 0.2, "submitted_s": 0.20011741702910513, "schedule_delay_ms": 0.11741702910511576, "precondition": true, "status": "success", "ack_ms": 6.904790992848575, "write_started_ms": 0.45204092748463154, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 7.027040934190154, "stable_since_ms": 7.027040934190154, "confirmed_ms": 64.12829097826034, "consecutive": 3, "regressions": 0, "requests": 3 }, "search": { "first_ms": 9.15116595569998, "stable_since_ms": 9.15116595569998, "confirmed_ms": 76.80070796050131, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 9.15116595569998, "confirmed_ms": 76.80070796050131, "search_minus_ack_ms": 2.246374962851405 } ] }, { "run": "fb2_429e9a_051", "trial": 2, "backend": "qdrant", "scenario": "nq-10k", "status": "complete", "eligible": true, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "summary": { "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "wall_s": 30.058481041924097, "operations": { "insert": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 17.694167094305158, "p50": 66.37152045732364, "p95": 70.81292299553752, "p99": 102.79061543988065, "max": 106.68779106345028 }, "confirmed_ms": { "min": 113.03775000851601, "p50": 211.35424997191876, "p95": 217.55178402527235, "p99": 219.17072334093973, "max": 220.70258401799947 }, "ack_ms": { "min": 4.978374927304685, "p50": 9.336458460893482, "p95": 12.055401754332706, "p99": 12.808603001758492, "max": 19.703458063304424 }, "search_minus_ack_ms": { "min": -1.202374929562211, "p50": 56.778437516186386, "p95": 59.37375595094636, "p99": 96.22014973079786, "max": 96.32574999704957 }, "regressions": 0, "confirmed_within_100ms": 0.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "update": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 17.30204198975116, "p50": 66.4779789512977, "p95": 69.54829166061245, "p99": 108.10732666403057, "max": 110.6683339457959 }, "confirmed_ms": { "min": 106.37470905203372, "p50": 211.58604143420234, "p95": 216.25494557083584, "p99": 217.76662949705516, "max": 258.5137919522822 }, "ack_ms": { "min": 4.029083997011185, "p50": 9.834812546614558, "p95": 12.063429161207749, "p99": 14.173590047284964, "max": 14.788875007070601 }, "search_minus_ack_ms": { "min": 1.6413340345025063, "p50": 56.60187447210774, "p95": 58.818181563401595, "p99": 95.9327114291955, "max": 101.20470798574388 }, "regressions": 0, "confirmed_within_100ms": 0.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "delete": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 15.061124926432967, "p50": 18.62916606478393, "p95": 31.5520374802872, "p99": 65.52842378034258, "max": 67.909250035882 }, "confirmed_ms": { "min": 106.23287502676249, "p50": 152.71610399940982, "p95": 163.42962707858533, "p99": 209.98773924889977, "max": 213.1794160231948 }, "ack_ms": { "min": 4.086375003680587, "p50": 8.547791512683034, "p95": 11.460387444822116, "p99": 18.997040499234583, "max": 21.034708013758063 }, "search_minus_ack_ms": { "min": -2.004374982789159, "p50": 0.20883348770439625, "p95": 5.764914548490095, "p99": 54.336127196438625, "max": 56.80729204323143 }, "regressions": 0, "confirmed_within_100ms": 0.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 } }, "total_operations": 300, "statuses": { "success": 300, "timeout": 0, "error": 0, "invalid_precondition": 0, "client_overload": 0 }, "query": { "count": 300, "success": 300, "ann_recall_at_10": 1.0, "ndcg_at_10": 0.9481505328688746, "latency_ms": { "min": 3.061083029024303, "p50": 8.623270492535084, "p95": 15.280519053339978, "p99": 48.510054955258965, "max": 49.49666699394584 }, "dispatch_delay_ms": { "min": 0.07966696284711361, "p50": 1.4055839856155217, "p95": 2.1979630866553634, "p99": 2.481296615442261, "max": 3.0195419676601887 } }, "diagnostics": { "schedule_delay_ms": { "min": 0.00904200132945121, "p50": 0.35299996379700715, "p95": 0.6219104048809321, "p99": 0.6836827651135418, "max": 0.8563339011757876 }, "probe_dispatch_ms": { "min": 0.036792014725506306, "p50": 0.40747947059571743, "p95": 1.7057580000255255, "p99": 2.4894592491909866, "max": 22.122083930298686 }, "probe_service_ms": { "min": 1.0989999864250422, "p50": 6.297396030277014, "p95": 47.98525793594308, "p99": 49.24197147600354, "max": 60.20395900122821 }, "write_service_ms": { "min": 3.785917069762945, "p50": 8.627958013676107, "p95": 11.472193495137617, "p99": 17.481113822432228, "max": 20.598458009772003 } }, "actual_submission_rate": 9.999945025358524, "metadata": { "backend": "qdrant", "collection": "fb2_429e9a_051", "dataset": { "dataset": "nq", "source": "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/nq.zip", "source_sha256": "2553bf7bfbab47b1436ca00a34bce57320e18e611fd00999a2a3a1b4714be752", "source_md5": "d4d3d2e48787a744b6f6e691ff534307", "records": 124000, "queries": 100, "relevant_documents_retained": 120, "seed": 20261028, "license": "CC-BY-SA-3.0", "license_source": "https://raw.githubusercontent.com/google-research-datasets/natural-questions/master/nq_open/README.md", "attribution": "BEIR and original dataset authors; Wikipedia contributors for NQ, SciFact authors and cited article authors for SciFact.", "transformation": "200-word title+text prefix; seeded reservoir of documents with relevant documents for selected test queries retained at front. Simulated mutations; no historical replay.", "model": "sentence-transformers/all-MiniLM-L6-v2", "model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "dimensions": 384, "normalized": true, "max_tokens": 256, "device": "mps", "files": { "corpus.jsonl": "9dc487d785e6ee493eccaf764daa3bf8720a8f4fc8c406246065e1ad33a91f5d", "queries.json": "5603f52876ada5d36118b65dea490f1d43900bc159e1804e637796eb9115e999", "vectors.npy": "cb7421eb6ca5964231403f0a922843088f6fd0e01cbf67f2a1076cc7101005db", "query_vectors.npy": "5e666f8d966d8db6b0fd93d4963234d2dffa0ee3910b8fad069a8ee201e8b5d4", "UPSTREAM-LICENSE.md": "8b675521da3fa142ac6bda38c9bcfcc8cff0e2d6128be53a062ed55fb53f7b87" } }, "base_count": 10000, "profile": { "dataset": "nq", "base_count": 10000, "rate": 10, "query_rate": 10, "arrival": "steady" }, "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "preloaded_mutable": 200, "started_utc": "2026-10-29T15:48:09Z", "setup_s": 3.4613817079225555, "backend_before": { "version": "1.20.0", "collection": { "status": "green", "optimizer_status": "ok", "indexed_vectors_count": 10200, "points_count": 10200, "segments_count": 2, "config": { "params": { "vectors": { "size": 384, "distance": "Cosine" }, "shard_number": 1, "replication_factor": 1, "write_consistency_factor": 1, "on_disk_payload": true }, "hnsw_config": { "m": 16, "ef_construct": 128, "full_scan_threshold": 10, "max_indexing_threads": 0, "on_disk": false }, "optimizer_config": { "deleted_threshold": 0.2, "vacuum_min_vector_number": 1000, "default_segment_number": 0, "max_segment_size": null, "memmap_threshold": null, "indexing_threshold": 1024, "flush_interval_sec": 5, "max_optimization_threads": null, "prevent_unoptimized": null }, "wal_config": { "wal_capacity_mb": 32, "wal_segments_ahead": 0, "wal_retain_closed": 1 }, "quantization_config": null }, "payload_schema": { "cohort": { "data_type": "keyword", "points": 10200 } }, "update_queue": { "length": 0 }, "created_at": "2026-10-29T15:48:09.010Z" }, "ack": "wait=true", "transport": "HTTP REST, pooled httpx" }, "count_before": 10200, "idle_duration_s": 10.0, "idle_queries": { "count": 100, "ann_recall": 1.0, "ndcg": 0.9481505328688745, "latency_ms": { "min": 3.9809999288991094, "p50": 8.419145946390927, "p95": 9.847558243200183, "p99": 10.57732957997379, "max": 48.6424999544397 } }, "exact_ndcg": 0.9481505328688745, "resources_before": { "usage_usec": 10116491, "user_usec": 8683077, "system_usec": 1433413, "nice_usec": 2193000, "nr_periods": 831, "nr_throttled": 4, "throttled_usec": 1305, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 351547392, "pids": 31 }, "resources_after": { "usage_usec": 16160769, "user_usec": 14030263, "system_usec": 2130505, "nice_usec": 2193000, "nr_periods": 1133, "nr_throttled": 4, "throttled_usec": 1305, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 348299264, "pids": 33 }, "count_after": 10200, "backend_after": { "version": "1.20.0", "collection": { "status": "green", "optimizer_status": "ok", "indexed_vectors_count": 10200, "points_count": 10200, "segments_count": 2, "config": { "params": { "vectors": { "size": 384, "distance": "Cosine" }, "shard_number": 1, "replication_factor": 1, "write_consistency_factor": 1, "on_disk_payload": true }, "hnsw_config": { "m": 16, "ef_construct": 128, "full_scan_threshold": 10, "max_indexing_threads": 0, "on_disk": false }, "optimizer_config": { "deleted_threshold": 0.2, "vacuum_min_vector_number": 1000, "default_segment_number": 0, "max_segment_size": null, "memmap_threshold": null, "indexing_threshold": 1024, "flush_interval_sec": 5, "max_optimization_threads": null, "prevent_unoptimized": null }, "wal_config": { "wal_capacity_mb": 32, "wal_segments_ahead": 0, "wal_retain_closed": 1 }, "quantization_config": null }, "payload_schema": { "cohort": { "data_type": "keyword", "points": 10200 } }, "update_queue": { "length": 0 }, "created_at": "2026-10-29T15:48:09.010Z" }, "ack": "wait=true", "transport": "HTTP REST, pooled httpx" } }, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "eligible": true, "protocol": "mixed-mutations-v2" }, "examples": [ { "event": 0, "operation": "insert", "doc_id": "mutation-000000", "scheduled_s": 0.0, "submitted_s": 0.00020720891188830137, "schedule_delay_ms": 0.20720891188830137, "precondition": true, "status": "success", "ack_ms": 6.72479101922363, "write_started_ms": 1.2095000129193068, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 14.947624993510544, "stable_since_ms": 14.947624993510544, "confirmed_ms": 69.83066606335342, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 60.33325009047985, "stable_since_ms": 60.33325009047985, "confirmed_ms": 203.63545801956207, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 60.33325009047985, "confirmed_ms": 203.63545801956207, "search_minus_ack_ms": 53.60845907125622 }, { "event": 1, "operation": "update", "doc_id": "mutation-000001", "scheduled_s": 0.1, "submitted_s": 0.10028166696429253, "schedule_delay_ms": 0.2816669642925207, "precondition": true, "status": "success", "ack_ms": 8.025457966141403, "write_started_ms": 1.3104999670758843, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 17.461500014178455, "stable_since_ms": 17.461500014178455, "confirmed_ms": 74.86291695386171, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 64.71362500451505, "stable_since_ms": 64.71362500451505, "confirmed_ms": 210.12879197951406, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 64.71362500451505, "confirmed_ms": 210.12879197951406, "search_minus_ack_ms": 56.68816703837365 }, { "event": 2, "operation": "delete", "doc_id": "mutation-000002", "scheduled_s": 0.2, "submitted_s": 0.20050520892255008, "schedule_delay_ms": 0.5052089225500711, "precondition": true, "status": "success", "ack_ms": 8.43195803463459, "write_started_ms": 1.2355410726740956, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 17.92741601821035, "stable_since_ms": 17.92741601821035, "confirmed_ms": 73.25962500181049, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 8.633791003376245, "stable_since_ms": 8.633791003376245, "confirmed_ms": 152.6183329988271, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 17.92741601821035, "confirmed_ms": 152.6183329988271, "search_minus_ack_ms": 0.20183296874165535 } ] }, { "run": "fb2_429e9a_052", "trial": 4, "backend": "weaviate", "scenario": "nq-100k-burst", "status": "complete", "eligible": true, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "summary": { "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "burst", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "wall_s": 30.00101041700691, "operations": { "insert": { "count": 130, "success": 130, "success_rate": 1.0, "first_observed_ms": { "min": 6.130042020231485, "p50": 16.780999954789877, "p95": 31.982566096121445, "p99": 33.76565104932524, "max": 35.14504199847579 }, "confirmed_ms": { "min": 63.5668330360204, "p50": 80.8947705081664, "p95": 104.47401251876727, "p99": 107.37415547599083, "max": 108.8892500847578 }, "ack_ms": { "min": 2.913916017860174, "p50": 9.330500557553023, "p95": 14.621570752933621, "p99": 17.280326302861788, "max": 18.52599997073412 }, "search_minus_ack_ms": { "min": -2.27312499191612, "p50": 1.589895982760936, "p95": 18.526360264513638, "p99": 19.819222793448716, "max": 20.858709001913667 }, "regressions": 0, "confirmed_within_100ms": 0.9230769230769231, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "update": { "count": 130, "success": 130, "success_rate": 1.0, "first_observed_ms": { "min": 3.3290840219706297, "p50": 17.39347947295755, "p95": 33.66597680142149, "p99": 37.00913584674707, "max": 37.844332982786 }, "confirmed_ms": { "min": 60.228834045119584, "p50": 83.06422899477184, "p95": 105.57187323574908, "p99": 110.08093735901637, "max": 112.4870419735089 }, "ack_ms": { "min": 3.0083329183980823, "p50": 9.252937510609627, "p95": 15.462123253382742, "p99": 16.341510656056926, "max": 16.83787500951439 }, "search_minus_ack_ms": { "min": -0.2699169563129544, "p50": 2.3958749952726066, "p95": 19.84945656149648, "p99": 21.857332233339537, "max": 22.29424996767193 }, "regressions": 0, "confirmed_within_100ms": 0.8461538461538461, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "delete": { "count": 130, "success": 130, "success_rate": 1.0, "first_observed_ms": { "min": 4.799583926796913, "p50": 14.908229000866413, "p95": 19.055741681950167, "p99": 22.039233592804514, "max": 24.48512497358024 }, "confirmed_ms": { "min": 63.69762506801635, "p50": 78.3061669790186, "p95": 86.53099359362386, "p99": 91.33518381160684, "max": 92.33970800414681 }, "ack_ms": { "min": 1.1787500698119402, "p50": 4.450895532499999, "p95": 9.118520701304078, "p99": 13.47195373149588, "max": 14.30116593837738 }, "search_minus_ack_ms": { "min": 0.314209028147161, "p50": 4.532541497610509, "p95": 7.417771185282618, "p99": 15.183510894421469, "max": 17.553000012412667 }, "regressions": 0, "confirmed_within_100ms": 1.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 } }, "total_operations": 390, "statuses": { "success": 390, "timeout": 0, "error": 0, "invalid_precondition": 0, "client_overload": 0 }, "query": { "count": 300, "success": 300, "ann_recall_at_10": 0.991, "ndcg_at_10": 0.7931268675492565, "latency_ms": { "min": 3.5183330764994025, "p50": 10.828209051396698, "p95": 15.520567301427945, "p99": 16.934402071638022, "max": 20.671250065788627 }, "dispatch_delay_ms": { "min": 0.13816694263368845, "p50": 1.3033960130997002, "p95": 2.4178207793738693, "p99": 2.621710845269261, "max": 3.5941670648753643 } }, "diagnostics": { "schedule_delay_ms": { "min": 0.00445805489945883, "p50": 0.31333351507767393, "p95": 0.6161080196032563, "p99": 0.6569485003125488, "max": 0.9742499794809234 }, "probe_dispatch_ms": { "min": 0.025666900910437107, "p50": 0.40149991400539875, "p95": 1.9658829201944128, "p99": 2.748340964317322, "max": 3.8109170272946358 }, "probe_service_ms": { "min": 0.5565830506384373, "p50": 3.9318749913945794, "p95": 11.77556248148903, "p99": 14.277648634742949, "max": 22.441708017140627 }, "write_service_ms": { "min": 1.0578329674899578, "p50": 5.798916972707957, "p95": 13.751360011519857, "p99": 15.83842351450585, "max": 18.0050830822438 } }, "actual_submission_rate": 13.009920043155539, "metadata": { "backend": "weaviate", "collection": "fb2_429e9a_052", "dataset": { "dataset": "nq", "source": "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/nq.zip", "source_sha256": "2553bf7bfbab47b1436ca00a34bce57320e18e611fd00999a2a3a1b4714be752", "source_md5": "d4d3d2e48787a744b6f6e691ff534307", "records": 124000, "queries": 100, "relevant_documents_retained": 120, "seed": 20261028, "license": "CC-BY-SA-3.0", "license_source": "https://raw.githubusercontent.com/google-research-datasets/natural-questions/master/nq_open/README.md", "attribution": "BEIR and original dataset authors; Wikipedia contributors for NQ, SciFact authors and cited article authors for SciFact.", "transformation": "200-word title+text prefix; seeded reservoir of documents with relevant documents for selected test queries retained at front. Simulated mutations; no historical replay.", "model": "sentence-transformers/all-MiniLM-L6-v2", "model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "dimensions": 384, "normalized": true, "max_tokens": 256, "device": "mps", "files": { "corpus.jsonl": "9dc487d785e6ee493eccaf764daa3bf8720a8f4fc8c406246065e1ad33a91f5d", "queries.json": "5603f52876ada5d36118b65dea490f1d43900bc159e1804e637796eb9115e999", "vectors.npy": "cb7421eb6ca5964231403f0a922843088f6fd0e01cbf67f2a1076cc7101005db", "query_vectors.npy": "5e666f8d966d8db6b0fd93d4963234d2dffa0ee3910b8fad069a8ee201e8b5d4", "UPSTREAM-LICENSE.md": "8b675521da3fa142ac6bda38c9bcfcc8cff0e2d6128be53a062ed55fb53f7b87" } }, "base_count": 100000, "profile": { "dataset": "nq", "base_count": 100000, "rate": 10, "query_rate": 10, "arrival": "burst" }, "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "burst", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "preloaded_mutable": 260, "started_utc": "2026-10-29T15:48:58Z", "setup_s": 37.00480237498414, "backend_before": { "version": "1.40.3", "schema": { "class": "Ffb2429e9a052", "invertedIndexConfig": { "bm25": { "b": 0.75, "k1": 1.2 }, "cleanupIntervalSeconds": 60, "stopwords": { "additions": null, "preset": "en", "removals": null }, "usingBlockMaxWAND": true }, "multiTenancyConfig": { "autoTenantActivation": false, "autoTenantCreation": false, "enabled": false }, "properties": [ { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "doc_id", "tokenization": "field" }, { "dataType": [ "int" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": false, "name": "version" }, { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "cohort", "tokenization": "field" }, { "dataType": [ "text" ], "indexFilterable": false, "indexRangeFilters": false, "indexSearchable": false, "name": "text", "tokenization": "word" } ], "shardingConfig": { "actualCount": 1, "actualVirtualCount": 128, "desiredCount": 1, "desiredVirtualCount": 128, "function": "murmur3", "key": "_id", "strategy": "hash", "virtualPerPhysical": 128 }, "vectorIndexConfig": { "bq": { "enabled": false }, "cleanupIntervalSeconds": 300, "distance": "cosine", "dynamicEfFactor": 8, "dynamicEfMax": 500, "dynamicEfMin": 100, "ef": 128, "efConstruction": 128, "filterStrategy": "acorn", "flatSearchCutoff": 0, "maxConnections": 16, "multivector": { "aggregation": "maxSim", "enabled": false, "muvera": { "dprojections": 16, "enabled": false, "ksim": 4, "repetitions": 10 } }, "pq": { "bitCompression": false, "centroids": 256, "enabled": false, "encoder": { "distribution": "log-normal", "type": "kmeans" }, "segments": 0, "trainingLimit": 100000 }, "rq": { "bits": 8, "centering": false, "enabled": false, "rescoreLimit": 20, "trainingLimit": 10000 }, "skip": false, "skipDefaultQuantization": false, "sq": { "enabled": false, "rescoreLimit": 20, "trainingLimit": 100000 }, "trackDefaultQuantization": false, "vectorCacheMaxObjects": 1000000000000 }, "vectorIndexType": "hnsw", "vectorizer": "none", "replicationConfig": { "deletionStrategy": "TimeBasedResolution", "factor": 1, "asyncEnabled": false } }, "shards": [ { "name": "D2QgkhtEFKrl", "per_node_status": { "node1": "READY" }, "status": "READY", "vectorQueueSize": 0 } ], "ack": "synchronous indexing, object PUT/POST/DELETE", "transport": "HTTP REST writes + GraphQL search, pooled httpx" }, "count_before": 100260, "idle_duration_s": 10.0, "idle_queries": { "count": 100, "ann_recall": 0.991, "ndcg": 0.7931268675492565, "latency_ms": { "min": 5.261957994662225, "p50": 11.98510400718078, "p95": 13.482326426310465, "p99": 20.477475418010833, "max": 41.15366702899337 } }, "exact_ndcg": 0.7931268675492565, "resources_before": { "usage_usec": 271044692, "user_usec": 265772057, "system_usec": 5272634, "nice_usec": 0, "nr_periods": 3899, "nr_throttled": 395, "throttled_usec": 688324, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 1437761536, "pids": 11 }, "resources_after": { "usage_usec": 281826843, "user_usec": 276075234, "system_usec": 5751609, "nice_usec": 0, "nr_periods": 4200, "nr_throttled": 395, "throttled_usec": 688324, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 1276841984, "pids": 11 }, "count_after": 100260, "backend_after": { "version": "1.40.3", "schema": { "class": "Ffb2429e9a052", "invertedIndexConfig": { "bm25": { "b": 0.75, "k1": 1.2 }, "cleanupIntervalSeconds": 60, "stopwords": { "additions": null, "preset": "en", "removals": null }, "usingBlockMaxWAND": true }, "multiTenancyConfig": { "autoTenantActivation": false, "autoTenantCreation": false, "enabled": false }, "properties": [ { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "doc_id", "tokenization": "field" }, { "dataType": [ "int" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": false, "name": "version" }, { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "cohort", "tokenization": "field" }, { "dataType": [ "text" ], "indexFilterable": false, "indexRangeFilters": false, "indexSearchable": false, "name": "text", "tokenization": "word" } ], "shardingConfig": { "actualCount": 1, "actualVirtualCount": 128, "desiredCount": 1, "desiredVirtualCount": 128, "function": "murmur3", "key": "_id", "strategy": "hash", "virtualPerPhysical": 128 }, "vectorIndexConfig": { "bq": { "enabled": false }, "cleanupIntervalSeconds": 300, "distance": "cosine", "dynamicEfFactor": 8, "dynamicEfMax": 500, "dynamicEfMin": 100, "ef": 128, "efConstruction": 128, "filterStrategy": "acorn", "flatSearchCutoff": 0, "maxConnections": 16, "multivector": { "aggregation": "maxSim", "enabled": false, "muvera": { "dprojections": 16, "enabled": false, "ksim": 4, "repetitions": 10 } }, "pq": { "bitCompression": false, "centroids": 256, "enabled": false, "encoder": { "distribution": "log-normal", "type": "kmeans" }, "segments": 0, "trainingLimit": 100000 }, "rq": { "bits": 8, "centering": false, "enabled": false, "rescoreLimit": 20, "trainingLimit": 10000 }, "skip": false, "skipDefaultQuantization": false, "sq": { "enabled": false, "rescoreLimit": 20, "trainingLimit": 100000 }, "trackDefaultQuantization": false, "vectorCacheMaxObjects": 1000000000000 }, "vectorIndexType": "hnsw", "vectorizer": "none", "replicationConfig": { "deletionStrategy": "TimeBasedResolution", "factor": 1, "asyncEnabled": false } }, "shards": [ { "name": "D2QgkhtEFKrl", "per_node_status": { "node1": "READY" }, "status": "READY", "vectorQueueSize": 0 } ], "ack": "synchronous indexing, object PUT/POST/DELETE", "transport": "HTTP REST writes + GraphQL search, pooled httpx" } }, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "eligible": true, "protocol": "mixed-mutations-v2" }, "examples": [ { "event": 0, "operation": "insert", "doc_id": "mutation-000000", "scheduled_s": 0.0, "submitted_s": 0.00027783308178186417, "schedule_delay_ms": 0.27783308178186417, "precondition": true, "status": "success", "ack_ms": 7.307374966330826, "write_started_ms": 1.5610839473083615, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 14.964249916374683, "stable_since_ms": 14.964249916374683, "confirmed_ms": 67.4793339567259, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 7.397791952826083, "stable_since_ms": 7.397791952826083, "confirmed_ms": 66.06508395634592, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 14.964249916374683, "confirmed_ms": 67.4793339567259, "search_minus_ack_ms": 0.09041698649525642 }, { "event": 1, "operation": "update", "doc_id": "mutation-000001", "scheduled_s": 0.025, "submitted_s": 0.02556966699194163, "schedule_delay_ms": 0.5696669919416295, "precondition": true, "status": "success", "ack_ms": 3.6170000676065683, "write_started_ms": 0.04416599404066801, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 13.609500019811094, "stable_since_ms": 13.609500019811094, "confirmed_ms": 66.39933306723833, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 5.601875018328428, "stable_since_ms": 5.601875018328428, "confirmed_ms": 64.80079097673297, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 13.609500019811094, "confirmed_ms": 66.39933306723833, "search_minus_ack_ms": 1.98487495072186 }, { "event": 2, "operation": "delete", "doc_id": "mutation-000002", "scheduled_s": 0.05, "submitted_s": 0.05052866705227643, "schedule_delay_ms": 0.5286670522764297, "precondition": true, "status": "success", "ack_ms": 2.129458007402718, "write_started_ms": 0.2448749728500843, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 14.373040990903974, "stable_since_ms": 14.373040990903974, "confirmed_ms": 67.55737494677305, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 5.034082918427885, "stable_since_ms": 5.034082918427885, "confirmed_ms": 63.97749995812774, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 14.373040990903974, "confirmed_ms": 67.55737494677305, "search_minus_ack_ms": 2.9046249110251665 } ] }, { "run": "fb2_429e9a_053", "trial": 5, "backend": "weaviate", "scenario": "nq-100k-burst", "status": "complete", "eligible": true, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "summary": { "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "burst", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "wall_s": 30.001452749944292, "operations": { "insert": { "count": 130, "success": 130, "success_rate": 1.0, "first_observed_ms": { "min": 6.268917000852525, "p50": 17.014187469612807, "p95": 32.53313969471492, "p99": 35.717368909390636, "max": 36.91079095005989 }, "confirmed_ms": { "min": 66.58020801842213, "p50": 82.43364602094516, "p95": 105.7334188837558, "p99": 109.12353405030444, "max": 109.35254103969783 }, "ack_ms": { "min": 3.534874995239079, "p50": 10.121333005372435, "p95": 15.46801055665128, "p99": 16.949741094140332, "max": 17.26270793005824 }, "search_minus_ack_ms": { "min": -0.31958299223333597, "p50": 2.2051254636608064, "p95": 19.23203316400759, "p99": 20.344805628992617, "max": 21.28316694870591 }, "regressions": 0, "confirmed_within_100ms": 0.8615384615384616, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "update": { "count": 130, "success": 130, "success_rate": 1.0, "first_observed_ms": { "min": 8.417083066888154, "p50": 16.885562508832663, "p95": 33.43243093113415, "p99": 34.45018504047766, "max": 35.308417049236596 }, "confirmed_ms": { "min": 66.55766605399549, "p50": 81.95652096765116, "p95": 105.29979015118442, "p99": 106.94705282105133, "max": 108.6851250147447 }, "ack_ms": { "min": 3.642709110863507, "p50": 9.346082981210202, "p95": 15.587585419416428, "p99": 17.592175297904753, "max": 18.35391588974744 }, "search_minus_ack_ms": { "min": -0.30891597270965576, "p50": 2.2325204336084425, "p95": 19.22253558295779, "p99": 20.92176859499886, "max": 21.376334014348686 }, "regressions": 0, "confirmed_within_100ms": 0.8461538461538461, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "delete": { "count": 130, "success": 130, "success_rate": 1.0, "first_observed_ms": { "min": 4.455583053641021, "p50": 15.161103976424783, "p95": 19.025168888038024, "p99": 30.931402334244872, "max": 36.51516593527049 }, "confirmed_ms": { "min": 63.9292080886662, "p50": 79.14400054141879, "p95": 87.09083738503978, "p99": 101.60394886042911, "max": 111.66529101319611 }, "ack_ms": { "min": 1.60604203119874, "p50": 4.602374974638224, "p95": 9.53818491543643, "p99": 13.962851215619592, "max": 14.202915946952999 }, "search_minus_ack_ms": { "min": -0.7807080401107669, "p50": 4.6019580331631005, "p95": 7.5045226025395095, "p99": 15.997365491930324, "max": 22.31224998831749 }, "regressions": 0, "confirmed_within_100ms": 0.9846153846153847, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 } }, "total_operations": 390, "statuses": { "success": 390, "timeout": 0, "error": 0, "invalid_precondition": 0, "client_overload": 0 }, "query": { "count": 300, "success": 300, "ann_recall_at_10": 0.991, "ndcg_at_10": 0.7931268675492565, "latency_ms": { "min": 4.803084069862962, "p50": 10.919437510892749, "p95": 16.295252030249685, "p99": 17.993805042933673, "max": 18.58250005170703 }, "dispatch_delay_ms": { "min": 0.14945899602025747, "p50": 1.369646517559886, "p95": 2.1432208421174437, "p99": 2.5106365699321027, "max": 3.346041892655194 } }, "diagnostics": { "schedule_delay_ms": { "min": 0.0038339523591446323, "p50": 0.3522295155565791, "p95": 0.7422168052297451, "p99": 0.7709643901325614, "max": 0.9648340055719018 }, "probe_dispatch_ms": { "min": 0.01679197885096073, "p50": 0.4733339883387089, "p95": 1.7155752284452312, "p99": 2.6650833571329717, "max": 3.5757499281316996 }, "probe_service_ms": { "min": 0.6845419993624091, "p50": 4.1487079579383135, "p95": 11.751675186678767, "p99": 14.451022371649742, "max": 20.83283301908523 }, "write_service_ms": { "min": 1.4933340717107058, "p50": 6.3132920186035335, "p95": 14.435862470418215, "p99": 16.05057080974803, "max": 17.841915949247777 } }, "actual_submission_rate": 13.00992745828074, "metadata": { "backend": "weaviate", "collection": "fb2_429e9a_053", "dataset": { "dataset": "nq", "source": "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/nq.zip", "source_sha256": "2553bf7bfbab47b1436ca00a34bce57320e18e611fd00999a2a3a1b4714be752", "source_md5": "d4d3d2e48787a744b6f6e691ff534307", "records": 124000, "queries": 100, "relevant_documents_retained": 120, "seed": 20261028, "license": "CC-BY-SA-3.0", "license_source": "https://raw.githubusercontent.com/google-research-datasets/natural-questions/master/nq_open/README.md", "attribution": "BEIR and original dataset authors; Wikipedia contributors for NQ, SciFact authors and cited article authors for SciFact.", "transformation": "200-word title+text prefix; seeded reservoir of documents with relevant documents for selected test queries retained at front. Simulated mutations; no historical replay.", "model": "sentence-transformers/all-MiniLM-L6-v2", "model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "dimensions": 384, "normalized": true, "max_tokens": 256, "device": "mps", "files": { "corpus.jsonl": "9dc487d785e6ee493eccaf764daa3bf8720a8f4fc8c406246065e1ad33a91f5d", "queries.json": "5603f52876ada5d36118b65dea490f1d43900bc159e1804e637796eb9115e999", "vectors.npy": "cb7421eb6ca5964231403f0a922843088f6fd0e01cbf67f2a1076cc7101005db", "query_vectors.npy": "5e666f8d966d8db6b0fd93d4963234d2dffa0ee3910b8fad069a8ee201e8b5d4", "UPSTREAM-LICENSE.md": "8b675521da3fa142ac6bda38c9bcfcc8cff0e2d6128be53a062ed55fb53f7b87" } }, "base_count": 100000, "profile": { "dataset": "nq", "base_count": 100000, "rate": 10, "query_rate": 10, "arrival": "burst" }, "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "burst", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "preloaded_mutable": 260, "started_utc": "2026-10-29T15:50:20Z", "setup_s": 37.684799541020766, "backend_before": { "version": "1.40.3", "schema": { "class": "Ffb2429e9a053", "invertedIndexConfig": { "bm25": { "b": 0.75, "k1": 1.2 }, "cleanupIntervalSeconds": 60, "stopwords": { "additions": null, "preset": "en", "removals": null }, "usingBlockMaxWAND": true }, "multiTenancyConfig": { "autoTenantActivation": false, "autoTenantCreation": false, "enabled": false }, "properties": [ { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "doc_id", "tokenization": "field" }, { "dataType": [ "int" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": false, "name": "version" }, { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "cohort", "tokenization": "field" }, { "dataType": [ "text" ], "indexFilterable": false, "indexRangeFilters": false, "indexSearchable": false, "name": "text", "tokenization": "word" } ], "shardingConfig": { "actualCount": 1, "actualVirtualCount": 128, "desiredCount": 1, "desiredVirtualCount": 128, "function": "murmur3", "key": "_id", "strategy": "hash", "virtualPerPhysical": 128 }, "vectorIndexConfig": { "bq": { "enabled": false }, "cleanupIntervalSeconds": 300, "distance": "cosine", "dynamicEfFactor": 8, "dynamicEfMax": 500, "dynamicEfMin": 100, "ef": 128, "efConstruction": 128, "filterStrategy": "acorn", "flatSearchCutoff": 0, "maxConnections": 16, "multivector": { "aggregation": "maxSim", "enabled": false, "muvera": { "dprojections": 16, "enabled": false, "ksim": 4, "repetitions": 10 } }, "pq": { "bitCompression": false, "centroids": 256, "enabled": false, "encoder": { "distribution": "log-normal", "type": "kmeans" }, "segments": 0, "trainingLimit": 100000 }, "rq": { "bits": 8, "centering": false, "enabled": false, "rescoreLimit": 20, "trainingLimit": 10000 }, "skip": false, "skipDefaultQuantization": false, "sq": { "enabled": false, "rescoreLimit": 20, "trainingLimit": 100000 }, "trackDefaultQuantization": false, "vectorCacheMaxObjects": 1000000000000 }, "vectorIndexType": "hnsw", "vectorizer": "none", "replicationConfig": { "deletionStrategy": "TimeBasedResolution", "factor": 1, "asyncEnabled": false } }, "shards": [ { "name": "fATXCC7gH0cL", "per_node_status": { "node1": "READY" }, "status": "READY", "vectorQueueSize": 0 } ], "ack": "synchronous indexing, object PUT/POST/DELETE", "transport": "HTTP REST writes + GraphQL search, pooled httpx" }, "count_before": 100260, "idle_duration_s": 10.0, "idle_queries": { "count": 100, "ann_recall": 0.991, "ndcg": 0.7931268675492565, "latency_ms": { "min": 5.261542042717338, "p50": 11.540416511707008, "p95": 13.225008343579246, "p99": 14.600624209269979, "max": 33.62091595772654 } }, "exact_ndcg": 0.7931268675492565, "resources_before": { "usage_usec": 357643470, "user_usec": 350839992, "system_usec": 6803477, "nice_usec": 0, "nr_periods": 4707, "nr_throttled": 519, "throttled_usec": 959166, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 1444151296, "pids": 11 }, "resources_after": { "usage_usec": 368757354, "user_usec": 361458525, "system_usec": 7298829, "nice_usec": 0, "nr_periods": 5009, "nr_throttled": 519, "throttled_usec": 959166, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 1311891456, "pids": 11 }, "count_after": 100260, "backend_after": { "version": "1.40.3", "schema": { "class": "Ffb2429e9a053", "invertedIndexConfig": { "bm25": { "b": 0.75, "k1": 1.2 }, "cleanupIntervalSeconds": 60, "stopwords": { "additions": null, "preset": "en", "removals": null }, "usingBlockMaxWAND": true }, "multiTenancyConfig": { "autoTenantActivation": false, "autoTenantCreation": false, "enabled": false }, "properties": [ { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "doc_id", "tokenization": "field" }, { "dataType": [ "int" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": false, "name": "version" }, { "dataType": [ "text" ], "indexFilterable": true, "indexRangeFilters": false, "indexSearchable": true, "name": "cohort", "tokenization": "field" }, { "dataType": [ "text" ], "indexFilterable": false, "indexRangeFilters": false, "indexSearchable": false, "name": "text", "tokenization": "word" } ], "shardingConfig": { "actualCount": 1, "actualVirtualCount": 128, "desiredCount": 1, "desiredVirtualCount": 128, "function": "murmur3", "key": "_id", "strategy": "hash", "virtualPerPhysical": 128 }, "vectorIndexConfig": { "bq": { "enabled": false }, "cleanupIntervalSeconds": 300, "distance": "cosine", "dynamicEfFactor": 8, "dynamicEfMax": 500, "dynamicEfMin": 100, "ef": 128, "efConstruction": 128, "filterStrategy": "acorn", "flatSearchCutoff": 0, "maxConnections": 16, "multivector": { "aggregation": "maxSim", "enabled": false, "muvera": { "dprojections": 16, "enabled": false, "ksim": 4, "repetitions": 10 } }, "pq": { "bitCompression": false, "centroids": 256, "enabled": false, "encoder": { "distribution": "log-normal", "type": "kmeans" }, "segments": 0, "trainingLimit": 100000 }, "rq": { "bits": 8, "centering": false, "enabled": false, "rescoreLimit": 20, "trainingLimit": 10000 }, "skip": false, "skipDefaultQuantization": false, "sq": { "enabled": false, "rescoreLimit": 20, "trainingLimit": 100000 }, "trackDefaultQuantization": false, "vectorCacheMaxObjects": 1000000000000 }, "vectorIndexType": "hnsw", "vectorizer": "none", "replicationConfig": { "deletionStrategy": "TimeBasedResolution", "factor": 1, "asyncEnabled": false } }, "shards": [ { "name": "fATXCC7gH0cL", "per_node_status": { "node1": "READY" }, "status": "READY", "vectorQueueSize": 0 } ], "ack": "synchronous indexing, object PUT/POST/DELETE", "transport": "HTTP REST writes + GraphQL search, pooled httpx" } }, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "eligible": true, "protocol": "mixed-mutations-v2" }, "examples": [ { "event": 0, "operation": "insert", "doc_id": "mutation-000000", "scheduled_s": 0.0, "submitted_s": 0.000203792005777359, "schedule_delay_ms": 0.203792005777359, "precondition": true, "status": "success", "ack_ms": 6.097916979342699, "write_started_ms": 1.5591669362038374, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 14.870541985146701, "stable_since_ms": 14.870541985146701, "confirmed_ms": 68.18591698538512, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 8.3360830321908, "stable_since_ms": 8.3360830321908, "confirmed_ms": 67.26974993944168, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 14.870541985146701, "confirmed_ms": 68.18591698538512, "search_minus_ack_ms": 2.2381660528481007 }, { "event": 1, "operation": "update", "doc_id": "mutation-000001", "scheduled_s": 0.025, "submitted_s": 0.025443334016017616, "schedule_delay_ms": 0.4433340160176144, "precondition": true, "status": "success", "ack_ms": 3.7284159334376454, "write_started_ms": 0.49120793119072914, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 13.32083297893405, "stable_since_ms": 13.32083297893405, "confirmed_ms": 68.96337494254112, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 4.819958005100489, "stable_since_ms": 4.819958005100489, "confirmed_ms": 62.88020801730454, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 13.32083297893405, "confirmed_ms": 68.96337494254112, "search_minus_ack_ms": 1.0915420716628432 }, { "event": 2, "operation": "delete", "doc_id": "mutation-000002", "scheduled_s": 0.05, "submitted_s": 0.05026995902881026, "schedule_delay_ms": 0.2699590288102599, "precondition": true, "status": "success", "ack_ms": 2.4064580211415887, "write_started_ms": 0.42512500658631325, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 14.513582922518253, "stable_since_ms": 14.513582922518253, "confirmed_ms": 67.35966599080712, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 5.1801250083372, "stable_since_ms": 5.1801250083372, "confirmed_ms": 63.90287494286895, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 14.513582922518253, "confirmed_ms": 67.35966599080712, "search_minus_ack_ms": 2.773666987195611 } ] }, { "run": "fb2_429e9a_054", "trial": 1, "backend": "qdrant", "scenario": "nq-100k", "status": "complete", "eligible": true, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "summary": { "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "wall_s": 30.06147262500599, "operations": { "insert": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 19.38491698820144, "p50": 68.15589551115409, "p95": 72.57950241910294, "p99": 108.46536690136419, "max": 111.10866605304182 }, "confirmed_ms": { "min": 158.28199998941272, "p50": 216.43693750957027, "p95": 222.84132711356506, "p99": 251.65722704958173, "max": 255.30558300670236 }, "ack_ms": { "min": 5.402249982580543, "p50": 8.770604035817087, "p95": 12.046300008660182, "p99": 15.671576303430031, "max": 40.77562503516674 }, "search_minus_ack_ms": { "min": 3.3304999815300107, "p50": 59.344895533286035, "p95": 64.43119975156151, "p99": 96.69352736789736, "max": 102.92908304836601 }, "regressions": 0, "confirmed_within_100ms": 0.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "update": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 15.544291003607213, "p50": 67.64235399896279, "p95": 71.09883775701746, "p99": 71.48268555523828, "max": 71.71975006349385 }, "confirmed_ms": { "min": 107.84454189706594, "p50": 214.76218744646758, "p95": 221.1335648142267, "p99": 222.69368753186436, "max": 241.08091602101922 }, "ack_ms": { "min": 3.6336659686639905, "p50": 10.12410398107022, "p95": 11.976749764289707, "p99": 13.124649844830905, "max": 22.6104169851169 }, "search_minus_ack_ms": { "min": 1.0200419928878546, "p50": 57.0962920319289, "p95": 59.79007303249091, "p99": 60.15493574202993, "max": 61.14679202437401 }, "regressions": 0, "confirmed_within_100ms": 0.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "delete": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 7.613915950059891, "p50": 17.78031192952767, "p95": 20.973041333490983, "p99": 51.09532616101214, "max": 67.8462500218302 }, "confirmed_ms": { "min": 107.76799998711795, "p50": 155.33466695342213, "p95": 161.32449613069184, "p99": 196.4521665824578, "max": 215.46429209411144 }, "ack_ms": { "min": 3.77541605848819, "p50": 8.741270517930388, "p95": 10.899380641058087, "p99": 12.063457001931967, "max": 12.451042071916163 }, "search_minus_ack_ms": { "min": -2.45741696562618, "p50": 0.19387499196454883, "p95": 3.339915198739616, "p99": 40.670346376719046, "max": 58.97845793515444 }, "regressions": 0, "confirmed_within_100ms": 0.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 } }, "total_operations": 300, "statuses": { "success": 300, "timeout": 0, "error": 0, "invalid_precondition": 0, "client_overload": 0 }, "query": { "count": 300, "success": 300, "ann_recall_at_10": 0.9879999999999999, "ndcg_at_10": 0.7931268675492565, "latency_ms": { "min": 3.8372919661924243, "p50": 8.782458025962114, "p95": 11.14022918045521, "p99": 46.076008877716944, "max": 51.823875051923096 }, "dispatch_delay_ms": { "min": 0.28912490233778954, "p50": 1.4726464869454503, "p95": 2.3573958955239505, "p99": 2.583438561996444, "max": 3.138334024697542 } }, "diagnostics": { "schedule_delay_ms": { "min": 0.008624908513610308, "p50": 0.3995834267691567, "p95": 0.7187729142603239, "p99": 0.7750868729780479, "max": 0.9855419397357679 }, "probe_dispatch_ms": { "min": 0.02887495793402195, "p50": 0.46335457591339946, "p95": 1.86677686870098, "p99": 2.6429631945211476, "max": 29.755458002910018 }, "probe_service_ms": { "min": 1.1517920065671206, "p50": 6.121354061178863, "p95": 49.32566901552491, "p99": 50.47462148126216, "max": 74.70912497956306 }, "write_service_ms": { "min": 3.397165914066136, "p50": 8.65775003330782, "p95": 11.254841898335146, "p99": 13.094014273956402, "max": 40.024083107709885 } }, "actual_submission_rate": 9.999891890768053, "metadata": { "backend": "qdrant", "collection": "fb2_429e9a_054", "dataset": { "dataset": "nq", "source": "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/nq.zip", "source_sha256": "2553bf7bfbab47b1436ca00a34bce57320e18e611fd00999a2a3a1b4714be752", "source_md5": "d4d3d2e48787a744b6f6e691ff534307", "records": 124000, "queries": 100, "relevant_documents_retained": 120, "seed": 20261028, "license": "CC-BY-SA-3.0", "license_source": "https://raw.githubusercontent.com/google-research-datasets/natural-questions/master/nq_open/README.md", "attribution": "BEIR and original dataset authors; Wikipedia contributors for NQ, SciFact authors and cited article authors for SciFact.", "transformation": "200-word title+text prefix; seeded reservoir of documents with relevant documents for selected test queries retained at front. Simulated mutations; no historical replay.", "model": "sentence-transformers/all-MiniLM-L6-v2", "model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "dimensions": 384, "normalized": true, "max_tokens": 256, "device": "mps", "files": { "corpus.jsonl": "9dc487d785e6ee493eccaf764daa3bf8720a8f4fc8c406246065e1ad33a91f5d", "queries.json": "5603f52876ada5d36118b65dea490f1d43900bc159e1804e637796eb9115e999", "vectors.npy": "cb7421eb6ca5964231403f0a922843088f6fd0e01cbf67f2a1076cc7101005db", "query_vectors.npy": "5e666f8d966d8db6b0fd93d4963234d2dffa0ee3910b8fad069a8ee201e8b5d4", "UPSTREAM-LICENSE.md": "8b675521da3fa142ac6bda38c9bcfcc8cff0e2d6128be53a062ed55fb53f7b87" } }, "base_count": 100000, "profile": { "dataset": "nq", "base_count": 100000, "rate": 10, "query_rate": 10, "arrival": "steady" }, "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "preloaded_mutable": 200, "started_utc": "2026-10-29T15:51:42Z", "setup_s": 42.23197729093954, "backend_before": { "version": "1.20.0", "collection": { "status": "green", "optimizer_status": "ok", "indexed_vectors_count": 100200, "points_count": 100200, "segments_count": 2, "config": { "params": { "vectors": { "size": 384, "distance": "Cosine" }, "shard_number": 1, "replication_factor": 1, "write_consistency_factor": 1, "on_disk_payload": true }, "hnsw_config": { "m": 16, "ef_construct": 128, "full_scan_threshold": 10, "max_indexing_threads": 0, "on_disk": false }, "optimizer_config": { "deleted_threshold": 0.2, "vacuum_min_vector_number": 1000, "default_segment_number": 0, "max_segment_size": null, "memmap_threshold": null, "indexing_threshold": 1024, "flush_interval_sec": 5, "max_optimization_threads": null, "prevent_unoptimized": null }, "wal_config": { "wal_capacity_mb": 32, "wal_segments_ahead": 0, "wal_retain_closed": 1 }, "quantization_config": null }, "payload_schema": { "cohort": { "data_type": "keyword", "points": 100200 } }, "update_queue": { "length": 0 }, "created_at": "2026-10-29T15:51:42.329Z" }, "ack": "wait=true", "transport": "HTTP REST, pooled httpx" }, "count_before": 100200, "idle_duration_s": 10.0, "idle_queries": { "count": 100, "ann_recall": 0.988, "ndcg": 0.7931268675492565, "latency_ms": { "min": 4.698084085248411, "p50": 9.512479533441365, "p95": 11.200495902448893, "p99": 11.986310601933114, "max": 48.15620800945908 } }, "exact_ndcg": 0.7931268675492565, "resources_before": { "usage_usec": 78268149, "user_usec": 73931116, "system_usec": 4337032, "nice_usec": 57678000, "nr_periods": 1749, "nr_throttled": 153, "throttled_usec": 871409, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 592470016, "pids": 29 }, "resources_after": { "usage_usec": 85060173, "user_usec": 80002283, "system_usec": 5057889, "nice_usec": 57678000, "nr_periods": 2051, "nr_throttled": 153, "throttled_usec": 871409, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 573558784, "pids": 32 }, "count_after": 100200, "backend_after": { "version": "1.20.0", "collection": { "status": "green", "optimizer_status": "ok", "indexed_vectors_count": 100200, "points_count": 100200, "segments_count": 2, "config": { "params": { "vectors": { "size": 384, "distance": "Cosine" }, "shard_number": 1, "replication_factor": 1, "write_consistency_factor": 1, "on_disk_payload": true }, "hnsw_config": { "m": 16, "ef_construct": 128, "full_scan_threshold": 10, "max_indexing_threads": 0, "on_disk": false }, "optimizer_config": { "deleted_threshold": 0.2, "vacuum_min_vector_number": 1000, "default_segment_number": 0, "max_segment_size": null, "memmap_threshold": null, "indexing_threshold": 1024, "flush_interval_sec": 5, "max_optimization_threads": null, "prevent_unoptimized": null }, "wal_config": { "wal_capacity_mb": 32, "wal_segments_ahead": 0, "wal_retain_closed": 1 }, "quantization_config": null }, "payload_schema": { "cohort": { "data_type": "keyword", "points": 100200 } }, "update_queue": { "length": 0 }, "created_at": "2026-10-29T15:51:42.329Z" }, "ack": "wait=true", "transport": "HTTP REST, pooled httpx" } }, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "eligible": true, "protocol": "mixed-mutations-v2" }, "examples": [ { "event": 0, "operation": "insert", "doc_id": "mutation-000000", "scheduled_s": 0.0, "submitted_s": 0.00020729191601276398, "schedule_delay_ms": 0.20729191601276398, "precondition": true, "status": "success", "ack_ms": 6.969375070184469, "write_started_ms": 1.4446250861510634, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 14.662958099506795, "stable_since_ms": 14.662958099506795, "confirmed_ms": 69.6968330303207, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 67.75545806158334, "stable_since_ms": 67.75545806158334, "confirmed_ms": 213.072125101462, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 67.75545806158334, "confirmed_ms": 213.072125101462, "search_minus_ack_ms": 60.78608299139887 }, { "event": 1, "operation": "update", "doc_id": "mutation-000001", "scheduled_s": 0.1, "submitted_s": 0.10025612497702241, "schedule_delay_ms": 0.2561249770224039, "precondition": true, "status": "success", "ack_ms": 11.563792009837925, "write_started_ms": 1.263541984371841, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 20.18345904070884, "stable_since_ms": 20.18345904070884, "confirmed_ms": 75.17075003124774, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 67.51704192720354, "stable_since_ms": 67.51704192720354, "confirmed_ms": 212.89066702593118, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 67.51704192720354, "confirmed_ms": 212.89066702593118, "search_minus_ack_ms": 55.95324991736561 }, { "event": 2, "operation": "delete", "doc_id": "mutation-000002", "scheduled_s": 0.2, "submitted_s": 0.20002483401913196, "schedule_delay_ms": 0.024834019131947382, "precondition": true, "status": "success", "ack_ms": 7.417040993459523, "write_started_ms": 0.7240409031510353, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 14.989665942266583, "stable_since_ms": 14.989665942266583, "confirmed_ms": 70.38791594095528, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 7.537915953435004, "stable_since_ms": 7.537915953435004, "confirmed_ms": 153.28112489078194, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 14.989665942266583, "confirmed_ms": 153.28112489078194, "search_minus_ack_ms": 0.12087495997548103 } ] }, { "run": "fb2_429e9a_055", "trial": 4, "backend": "qdrant", "scenario": "nq-10k", "status": "complete", "eligible": true, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "summary": { "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "wall_s": 30.053632750059478, "operations": { "insert": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 17.50679104588926, "p50": 66.65060401428491, "p95": 74.55190595355808, "p99": 107.8415582771413, "max": 108.49904199130833 }, "confirmed_ms": { "min": 107.3551670415327, "p50": 212.9812915227376, "p95": 219.56211106735282, "p99": 252.9014225001447, "max": 253.31449997611344 }, "ack_ms": { "min": 4.970500012859702, "p50": 9.16262500686571, "p95": 11.4746208419092, "p99": 12.571769526693974, "max": 13.353333109989762 }, "search_minus_ack_ms": { "min": -0.9331670589745045, "p50": 57.150999491568655, "p95": 65.8482792263384, "p99": 98.0611235613469, "max": 98.80754095502198 }, "regressions": 0, "confirmed_within_100ms": 0.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "update": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 15.796083025634289, "p50": 66.24833296518773, "p95": 69.96879338403232, "p99": 72.04009295091966, "max": 74.75120900198817 }, "confirmed_ms": { "min": 108.40274998918176, "p50": 211.58589603146538, "p95": 216.87819581711665, "p99": 220.0513918348588, "max": 221.52162494603544 }, "ack_ms": { "min": 3.663750016130507, "p50": 9.384603938087821, "p95": 11.91852503689006, "p99": 15.870865470496948, "max": 18.076834036037326 }, "search_minus_ack_ms": { "min": 0.8963750442489982, "p50": 56.576937495265156, "p95": 59.314075333531946, "p99": 60.146984545281164, "max": 61.12308299634606 }, "regressions": 0, "confirmed_within_100ms": 0.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "delete": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 14.263707911595702, "p50": 17.875208985060453, "p95": 21.223464282229536, "p99": 67.15186998597348, "max": 68.88387503568083 }, "confirmed_ms": { "min": 105.7984170038253, "p50": 153.01572909811512, "p95": 159.06769998837262, "p99": 214.49022670043632, "max": 214.8962909122929 }, "ack_ms": { "min": 3.1828749924898148, "p50": 8.238353999331594, "p95": 10.141706862486897, "p99": 11.078583927592264, "max": 11.39220898039639 }, "search_minus_ack_ms": { "min": -2.106082974933088, "p50": 0.20235450938344002, "p95": 2.9865417163819044, "p99": 57.50108855543659, "max": 58.433916070498526 }, "regressions": 0, "confirmed_within_100ms": 0.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 } }, "total_operations": 300, "statuses": { "success": 300, "timeout": 0, "error": 0, "invalid_precondition": 0, "client_overload": 0 }, "query": { "count": 300, "success": 300, "ann_recall_at_10": 0.9979999999999999, "ndcg_at_10": 0.948294881680092, "latency_ms": { "min": 3.64154193084687, "p50": 8.389895490836352, "p95": 10.658175317803398, "p99": 47.964971729088575, "max": 53.900124970823526 }, "dispatch_delay_ms": { "min": 0.09566708467900753, "p50": 1.4513750211335719, "p95": 2.3903874855022877, "p99": 2.5887059513479467, "max": 3.0310419388115406 } }, "diagnostics": { "schedule_delay_ms": { "min": 0.010917079634964466, "p50": 0.3972085658467961, "p95": 0.7149039220537112, "p99": 0.7679307961385007, "max": 1.2450000504031777 }, "probe_dispatch_ms": { "min": 0.03574998117983341, "p50": 0.44579198583960533, "p95": 1.7642751801758998, "p99": 2.6169899804517622, "max": 3.4453340340405703 }, "probe_service_ms": { "min": 1.288583967834711, "p50": 6.0092920903116465, "p95": 48.58852457255125, "p99": 49.83628998976201, "max": 53.77970798872411 }, "write_service_ms": { "min": 2.9182499274611473, "p50": 8.356062462553382, "p95": 10.796816338552162, "p99": 11.95314100477844, "max": 17.681124969385564 } }, "actual_submission_rate": 9.999818091629274, "metadata": { "backend": "qdrant", "collection": "fb2_429e9a_055", "dataset": { "dataset": "nq", "source": "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/nq.zip", "source_sha256": "2553bf7bfbab47b1436ca00a34bce57320e18e611fd00999a2a3a1b4714be752", "source_md5": "d4d3d2e48787a744b6f6e691ff534307", "records": 124000, "queries": 100, "relevant_documents_retained": 120, "seed": 20261028, "license": "CC-BY-SA-3.0", "license_source": "https://raw.githubusercontent.com/google-research-datasets/natural-questions/master/nq_open/README.md", "attribution": "BEIR and original dataset authors; Wikipedia contributors for NQ, SciFact authors and cited article authors for SciFact.", "transformation": "200-word title+text prefix; seeded reservoir of documents with relevant documents for selected test queries retained at front. Simulated mutations; no historical replay.", "model": "sentence-transformers/all-MiniLM-L6-v2", "model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "dimensions": 384, "normalized": true, "max_tokens": 256, "device": "mps", "files": { "corpus.jsonl": "9dc487d785e6ee493eccaf764daa3bf8720a8f4fc8c406246065e1ad33a91f5d", "queries.json": "5603f52876ada5d36118b65dea490f1d43900bc159e1804e637796eb9115e999", "vectors.npy": "cb7421eb6ca5964231403f0a922843088f6fd0e01cbf67f2a1076cc7101005db", "query_vectors.npy": "5e666f8d966d8db6b0fd93d4963234d2dffa0ee3910b8fad069a8ee201e8b5d4", "UPSTREAM-LICENSE.md": "8b675521da3fa142ac6bda38c9bcfcc8cff0e2d6128be53a062ed55fb53f7b87" } }, "base_count": 10000, "profile": { "dataset": "nq", "base_count": 10000, "rate": 10, "query_rate": 10, "arrival": "steady" }, "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "preloaded_mutable": 200, "started_utc": "2026-10-29T15:53:10Z", "setup_s": 3.466475374996662, "backend_before": { "version": "1.20.0", "collection": { "status": "green", "optimizer_status": "ok", "indexed_vectors_count": 10200, "points_count": 10200, "segments_count": 2, "config": { "params": { "vectors": { "size": 384, "distance": "Cosine" }, "shard_number": 1, "replication_factor": 1, "write_consistency_factor": 1, "on_disk_payload": true }, "hnsw_config": { "m": 16, "ef_construct": 128, "full_scan_threshold": 10, "max_indexing_threads": 0, "on_disk": false }, "optimizer_config": { "deleted_threshold": 0.2, "vacuum_min_vector_number": 1000, "default_segment_number": 0, "max_segment_size": null, "memmap_threshold": null, "indexing_threshold": 1024, "flush_interval_sec": 5, "max_optimization_threads": null, "prevent_unoptimized": null }, "wal_config": { "wal_capacity_mb": 32, "wal_segments_ahead": 0, "wal_retain_closed": 1 }, "quantization_config": null }, "payload_schema": { "cohort": { "data_type": "keyword", "points": 10200 } }, "update_queue": { "length": 0 }, "created_at": "2026-10-29T15:53:10.625Z" }, "ack": "wait=true", "transport": "HTTP REST, pooled httpx" }, "count_before": 10200, "idle_duration_s": 10.0, "idle_queries": { "count": 100, "ann_recall": 0.998, "ndcg": 0.948294881680092, "latency_ms": { "min": 3.027541097253561, "p50": 8.547375036869198, "p95": 9.536757942987606, "p99": 11.209904474672113, "max": 46.50216596201062 } }, "exact_ndcg": 0.9481505328688745, "resources_before": { "usage_usec": 88579380, "user_usec": 83156980, "system_usec": 5422399, "nice_usec": 59445000, "nr_periods": 2218, "nr_throttled": 158, "throttled_usec": 897339, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 395411456, "pids": 31 }, "resources_after": { "usage_usec": 94542133, "user_usec": 88428195, "system_usec": 6113937, "nice_usec": 59445000, "nr_periods": 2520, "nr_throttled": 158, "throttled_usec": 897339, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 384729088, "pids": 33 }, "count_after": 10200, "backend_after": { "version": "1.20.0", "collection": { "status": "green", "optimizer_status": "ok", "indexed_vectors_count": 10200, "points_count": 10200, "segments_count": 2, "config": { "params": { "vectors": { "size": 384, "distance": "Cosine" }, "shard_number": 1, "replication_factor": 1, "write_consistency_factor": 1, "on_disk_payload": true }, "hnsw_config": { "m": 16, "ef_construct": 128, "full_scan_threshold": 10, "max_indexing_threads": 0, "on_disk": false }, "optimizer_config": { "deleted_threshold": 0.2, "vacuum_min_vector_number": 1000, "default_segment_number": 0, "max_segment_size": null, "memmap_threshold": null, "indexing_threshold": 1024, "flush_interval_sec": 5, "max_optimization_threads": null, "prevent_unoptimized": null }, "wal_config": { "wal_capacity_mb": 32, "wal_segments_ahead": 0, "wal_retain_closed": 1 }, "quantization_config": null }, "payload_schema": { "cohort": { "data_type": "keyword", "points": 10200 } }, "update_queue": { "length": 0 }, "created_at": "2026-10-29T15:53:10.625Z" }, "ack": "wait=true", "transport": "HTTP REST, pooled httpx" } }, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "eligible": true, "protocol": "mixed-mutations-v2" }, "examples": [ { "event": 0, "operation": "insert", "doc_id": "mutation-000000", "scheduled_s": 0.0, "submitted_s": 0.00021229207050055265, "schedule_delay_ms": 0.21229207050055265, "precondition": true, "status": "success", "ack_ms": 5.2335410146042705, "write_started_ms": 1.5020410064607859, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 15.492582926526666, "stable_since_ms": 15.492582926526666, "confirmed_ms": 69.02449997141957, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 59.148125001229346, "stable_since_ms": 59.148125001229346, "confirmed_ms": 202.98516598995775, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 59.148125001229346, "confirmed_ms": 202.98516598995775, "search_minus_ack_ms": 53.914583986625075 }, { "event": 1, "operation": "update", "doc_id": "mutation-000001", "scheduled_s": 0.1, "submitted_s": 0.10055829200427979, "schedule_delay_ms": 0.5582920042797868, "precondition": true, "status": "success", "ack_ms": 10.060708038508892, "write_started_ms": 1.0979999788105488, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 20.883082994259894, "stable_since_ms": 20.883082994259894, "confirmed_ms": 78.89612508006394, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 64.78450004942715, "stable_since_ms": 64.78450004942715, "confirmed_ms": 210.65483300480992, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 64.78450004942715, "confirmed_ms": 210.65483300480992, "search_minus_ack_ms": 54.72379201091826 }, { "event": 2, "operation": "delete", "doc_id": "mutation-000002", "scheduled_s": 0.2, "submitted_s": 0.20009554200805724, "schedule_delay_ms": 0.09554200805722557, "precondition": true, "status": "success", "ack_ms": 8.521458017639816, "write_started_ms": 1.818541088141501, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 18.53500003926456, "stable_since_ms": 18.53500003926456, "confirmed_ms": 74.84595803543925, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 8.898416068404913, "stable_since_ms": 8.898416068404913, "confirmed_ms": 154.53825006261468, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 18.53500003926456, "confirmed_ms": 154.53825006261468, "search_minus_ack_ms": 0.37695805076509714 } ] }, { "run": "fb2_429e9a_056", "trial": 4, "backend": "pgvector", "scenario": "nq-100k", "status": "complete", "eligible": true, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "summary": { "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "wall_s": 30.001029207953252, "operations": { "insert": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 18.890999956056476, "p50": 30.448874982539564, "p95": 35.69996252772398, "p99": 37.540039452724166, "max": 38.740166928619146 }, "confirmed_ms": { "min": 74.68737498857081, "p50": 91.38960397103801, "p95": 99.6068541368004, "p99": 102.1211441862397, "max": 104.57745799794793 }, "ack_ms": { "min": 7.070624968037009, "p50": 18.92147900070995, "p95": 24.47736905887723, "p99": 26.410563300596554, "max": 27.122208033688366 }, "search_minus_ack_ms": { "min": 7.6043750159442425, "p50": 11.648082989268005, "p95": 12.962176842847839, "p99": 13.199344944441691, "max": 13.32012505736202 }, "regressions": 0, "confirmed_within_100ms": 0.97, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "update": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 17.730374936945736, "p50": 27.822270523756742, "p95": 34.444959729444236, "p99": 36.739808402489885, "max": 39.02675001882017 }, "confirmed_ms": { "min": 75.785499997437, "p50": 87.75858301669359, "p95": 96.34577494580299, "p99": 98.99715531850237, "max": 103.20350003894418 }, "ack_ms": { "min": 6.329791969619691, "p50": 16.048833436798304, "p95": 23.421254445565864, "p99": 24.768287467304628, "max": 26.69833309482783 }, "search_minus_ack_ms": { "min": 8.16054199822247, "p50": 11.703396041411906, "p95": 13.258466980187222, "p99": 13.73404317884706, "max": 13.895125011913478 }, "regressions": 0, "confirmed_within_100ms": 0.99, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "delete": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 11.990207945927978, "p50": 13.337979442439973, "p95": 31.789086025673896, "p99": 34.29359183064665, "max": 39.5290000597015 }, "confirmed_ms": { "min": 63.543583964928985, "p50": 77.48447899939492, "p95": 94.53502705437131, "p99": 99.21963494736704, "max": 103.62195805646479 }, "ack_ms": { "min": 1.2500829761847854, "p50": 2.1211040439084172, "p95": 4.682049324037505, "p99": 6.376208346337082, "max": 6.710374960675836 }, "search_minus_ack_ms": { "min": 3.627624944783747, "p50": 11.365895974449813, "p95": 28.916239936370403, "p99": 31.62757241516377, "max": 36.83225007262081 }, "regressions": 0, "confirmed_within_100ms": 0.99, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 } }, "total_operations": 300, "statuses": { "success": 300, "timeout": 0, "error": 0, "invalid_precondition": 0, "client_overload": 0 }, "query": { "count": 300, "success": 300, "ann_recall_at_10": 0.995, "ndcg_at_10": 0.7931268675492565, "latency_ms": { "min": 4.07466699834913, "p50": 10.933687502983958, "p95": 16.50958546670154, "p99": 18.904216850642104, "max": 21.992291091009974 }, "dispatch_delay_ms": { "min": 0.08737493772059679, "p50": 0.9356039809063077, "p95": 1.6535559378098696, "p99": 1.9533966132439673, "max": 2.207582932896912 } }, "diagnostics": { "schedule_delay_ms": { "min": 0.009457976557492032, "p50": 0.40402048034593463, "p95": 0.7150541373995178, "p99": 0.7592999951451951, "max": 0.7720419904213571 }, "probe_dispatch_ms": { "min": 0.023791915737092495, "p50": 0.4866250092163682, "p95": 1.300391973927617, "p99": 1.8183134449645884, "max": 2.6086249854415655 }, "probe_service_ms": { "min": 0.33599999733269215, "p50": 1.7752919811755419, "p95": 12.452308018691836, "p99": 17.064446592703465, "max": 22.345292032696307 }, "write_service_ms": { "min": 1.0063749505206943, "p50": 14.67100001173094, "p95": 22.90707049542107, "p99": 24.299110830761478, "max": 26.440167101100087 } }, "actual_submission_rate": 9.999953358430535, "metadata": { "backend": "pgvector", "collection": "fb2_429e9a_056", "dataset": { "dataset": "nq", "source": "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/nq.zip", "source_sha256": "2553bf7bfbab47b1436ca00a34bce57320e18e611fd00999a2a3a1b4714be752", "source_md5": "d4d3d2e48787a744b6f6e691ff534307", "records": 124000, "queries": 100, "relevant_documents_retained": 120, "seed": 20261028, "license": "CC-BY-SA-3.0", "license_source": "https://raw.githubusercontent.com/google-research-datasets/natural-questions/master/nq_open/README.md", "attribution": "BEIR and original dataset authors; Wikipedia contributors for NQ, SciFact authors and cited article authors for SciFact.", "transformation": "200-word title+text prefix; seeded reservoir of documents with relevant documents for selected test queries retained at front. Simulated mutations; no historical replay.", "model": "sentence-transformers/all-MiniLM-L6-v2", "model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "dimensions": 384, "normalized": true, "max_tokens": 256, "device": "mps", "files": { "corpus.jsonl": "9dc487d785e6ee493eccaf764daa3bf8720a8f4fc8c406246065e1ad33a91f5d", "queries.json": "5603f52876ada5d36118b65dea490f1d43900bc159e1804e637796eb9115e999", "vectors.npy": "cb7421eb6ca5964231403f0a922843088f6fd0e01cbf67f2a1076cc7101005db", "query_vectors.npy": "5e666f8d966d8db6b0fd93d4963234d2dffa0ee3910b8fad069a8ee201e8b5d4", "UPSTREAM-LICENSE.md": "8b675521da3fa142ac6bda38c9bcfcc8cff0e2d6128be53a062ed55fb53f7b87" } }, "base_count": 100000, "profile": { "dataset": "nq", "base_count": 100000, "rate": 10, "query_rate": 10, "arrival": "steady" }, "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "preloaded_mutable": 200, "started_utc": "2026-10-29T15:54:00Z", "setup_s": 56.86111133301165, "backend_before": { "version": "0.8.7", "postgres": "PostgreSQL 16.15 (Debian 16.15-1.pgdg12+2) on aarch64-unknown-linux-gnu, compiled by gcc (Debian 12.2.0-14+deb12u1) 12.2.0, 64-bit", "ef_search": 128, "ack": "autocommit, READ COMMITTED", "transport": "psycopg, connection per worker", "query_plan": [ { "Plan": { "Node Type": "Limit", "Parallel Aware": false, "Async Capable": false, "Startup Cost": 1674.2, "Total Cost": 1691.32, "Plan Rows": 10, "Plan Width": 18, "Plans": [ { "Node Type": "Index Scan", "Parent Relationship": "Outer", "Parallel Aware": false, "Async Capable": false, "Scan Direction": "Forward", "Index Name": "fb2_429e9a_056_hnsw", "Relation Name": "fb2_429e9a_056", "Alias": "fb2_429e9a_056", "Startup Cost": 1674.2, "Total Cost": 172870.0, "Plan Rows": 100000, "Plan Width": 18, "Order By": "(embedding <=> '[-0.08946895,-0.094362676,0.092023626,-0.0040844884,0.043233823,0.00062519487,0.1075197,-0.035732996,0.013746996,-0.024103122,0.0021706754,-0.04720602,0.04787615,-0.02043222,0.058315646,0.081252135,-0.05375015,-0.030799072,-0.0077205584,0.035735283,-0.04474784,0.01614922,-0.06905586,0.04194401,-0.08832908,0.007873556,0.0044992934,0.021968655,0.010953364,0.07212316,0.023185596,0.001176476,0.04725166,-0.03580423,-0.032344446,0.004752835,-0.056179125,-0.0353235,-0.037251893,-0.005143068,-0.027599342,0.024403147,0.031338032,0.0056410627,0.008425455,-0.0087741045,-0.039870497,-0.011831836,0.042621233,0.048142947,0.01603842,0.06745542,-0.038018145,0.064212486,-0.009636566,0.01607475,-0.0027868534,-0.03761097,-0.011889431,-0.01084577,0.083403684,0.028946918,0.061870236,-0.045382615,0.03812586,0.060426526,-0.017003024,0.02328925,-0.093020774,-0.024072766,0.042720392,-0.022444837,-0.011597358,0.01209231,-0.005953629,0.029691145,-0.011711495,-0.053345133,-0.081757374,-0.049303155,0.032193717,-0.038507868,-0.0060101775,-0.031470332,-0.07362443,0.03575578,0.0245484,0.01985519,-0.15116754,0.050386053,-0.0389634,0.07548389,-0.09264168,-0.012602816,-0.0035539444,-0.046416666,0.015100493,0.100691915,-0.037529,0.04600518,-0.021859422,0.014675989,-0.097722545,-0.053200893,-0.017040461,-0.01629616,-0.012339222,0.06543302,0.07252074,-0.044301156,0.03054835,-0.038570553,0.019310065,-0.005527747,-0.029386865,-0.058248505,0.065443166,0.028506057,0.0878254,0.058387812,0.081721246,-0.0005962058,0.0569388,0.00960514,0.011667701,0.014456492,-0.07218306,-3.0444767e-33,0.06687367,-0.019213954,0.058202963,0.025171444,-0.028259544,-0.04171671,0.03127722,0.04865431,-0.0721459,-0.050143186,-0.029907746,0.06248994,-0.05096076,-0.065192886,0.010210447,-0.077256374,-0.043370593,-0.0901922,-0.066522665,0.026072454,-0.022315364,0.075375885,-0.029968103,-0.0050771856,-0.0046955296,0.0065650316,-0.06791897,0.07368851,0.056069646,-0.021026025,-0.011352883,0.038811427,-0.090847075,-0.056490287,-0.039881036,0.032885592,-0.029586185,0.04753608,0.006341778,-0.0034275919,0.03681623,0.09326265,-0.0532089,0.032396164,0.012605483,0.045778517,-0.0006472902,0.029721813,0.0734947,-0.037117425,-0.09356724,-0.0071634455,-0.032400057,0.010453378,0.06286513,0.006053158,0.060524322,0.091866635,0.063640274,0.055919155,0.053222507,-0.019174587,0.089296214,-0.058922265,0.013745364,-0.022907477,0.028347451,-0.040759467,-0.06820364,-0.061398294,-0.046857864,0.0060627847,0.010442015,-0.07662317,0.031950947,-0.107813306,-0.0016125626,-0.033413693,-0.09989722,-0.03580293,-0.036118146,-0.08276185,0.1280713,0.07596603,0.049225595,-0.026312606,0.0031930343,-0.0012469466,0.000872793,-0.040353082,-0.034516506,-0.036047976,0.004999689,-0.01570079,-0.09783203,1.4186912e-33,-0.0130903,0.04742377,0.02889633,0.09809088,0.043639127,-0.0747531,0.03402747,0.029147558,0.019848268,0.111865185,0.04845458,-0.016222177,0.025501896,0.09813583,-0.002833195,0.0038929228,0.048537854,0.014111116,0.036570966,-0.017512968,-0.079755045,0.043452248,0.10843892,0.061968718,-0.015678233,0.0060376576,0.010718569,-0.03907552,0.040307663,0.04553227,0.041731495,0.083179526,-0.0844024,-0.023447752,0.014485363,-0.00788756,-0.11410207,0.039309476,0.011220049,0.12889338,-0.07970701,0.010658111,-0.06657531,-0.022037188,-0.043459363,-0.030291978,0.05803613,0.06109021,-0.09415837,0.030727232,0.011858702,0.013526454,-0.049580045,-0.052486703,0.013218615,0.0050231596,0.07660412,-0.0046343356,-0.0692584,-0.03353194,0.022288544,0.022836983,0.01361394,-0.036331616,0.008194689,0.09226991,0.08229875,-0.047639843,-0.005033471,0.0066081914,-0.106666416,-0.06385503,0.006385333,0.06318358,0.03146985,-0.060764216,-0.043287244,-0.042385958,-0.07963737,-0.043478705,-0.031840656,0.060027804,0.027829666,0.03728562,-0.07014622,0.0649568,-0.052579004,-0.0034938955,-0.15390792,-0.04426162,0.028217599,0.05593145,0.023251446,0.029115746,-0.040321343,-1.8832651e-08,-0.037536032,0.0005056563,0.013919978,-0.038129315,0.061486673,0.005534046,-0.010768211,-0.16527015,0.061414365,0.019963786,-0.019666849,-0.059065994,0.00083181454,-0.016695915,0.01090875,0.02685501,-0.0008743374,-0.060967337,-0.049258173,0.012731965,-0.03973743,-0.0018210232,-0.03797001,0.00035104176,0.04477184,0.04185603,-0.03215796,-0.09103283,0.0467481,0.017810425,0.037437033,0.054782324,-0.0059442874,0.022557084,0.053393807,-0.03746794,-0.07080435,0.10548834,-0.04215435,0.042547524,0.05064807,0.07560324,0.04072184,-0.0011278372,0.00702018,-0.070807114,0.040035766,0.051417697,0.011316928,-0.028955372,0.04197099,-0.0026867494,-0.056857754,0.0761941,0.030621601,-0.052413527,-0.021492323,0.0957355,0.1013111,-0.082977295,-0.036080915,-0.03327806,0.0029404552,-0.03586191]'::vector)", "Filter": "(cohort = 'static'::text)" } ] } } ] }, "count_before": 100200, "idle_duration_s": 10.0, "idle_queries": { "count": 100, "ann_recall": 0.995, "ndcg": 0.7931268675492565, "latency_ms": { "min": 4.034625017084181, "p50": 18.89722904888913, "p95": 22.15062077739276, "p99": 23.05805725278335, "max": 27.20112493261695 } }, "exact_ndcg": 0.7931268675492565, "resources_before": { "usage_usec": 204071712, "user_usec": 193465730, "system_usec": 10605982, "nice_usec": 0, "nr_periods": 2440, "nr_throttled": 844, "throttled_usec": 88601694, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 1328136192, "pids": 23 }, "resources_after": { "usage_usec": 215352105, "user_usec": 200871493, "system_usec": 14480611, "nice_usec": 0, "nr_periods": 2741, "nr_throttled": 844, "throttled_usec": 88601694, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 1340841984, "pids": 28 }, "count_after": 100200, "backend_after": { "version": "0.8.7", "postgres": "PostgreSQL 16.15 (Debian 16.15-1.pgdg12+2) on aarch64-unknown-linux-gnu, compiled by gcc (Debian 12.2.0-14+deb12u1) 12.2.0, 64-bit", "ef_search": 128, "ack": "autocommit, READ COMMITTED", "transport": "psycopg, connection per worker", "query_plan": [ { "Plan": { "Node Type": "Limit", "Parallel Aware": false, "Async Capable": false, "Startup Cost": 1675.79, "Total Cost": 1692.91, "Plan Rows": 10, "Plan Width": 18, "Plans": [ { "Node Type": "Index Scan", "Parent Relationship": "Outer", "Parallel Aware": false, "Async Capable": false, "Scan Direction": "Forward", "Index Name": "fb2_429e9a_056_hnsw", "Relation Name": "fb2_429e9a_056", "Alias": "fb2_429e9a_056", "Startup Cost": 1675.79, "Total Cost": 173152.56, "Plan Rows": 100113, "Plan Width": 18, "Order By": "(embedding <=> '[-0.08946895,-0.094362676,0.092023626,-0.0040844884,0.043233823,0.00062519487,0.1075197,-0.035732996,0.013746996,-0.024103122,0.0021706754,-0.04720602,0.04787615,-0.02043222,0.058315646,0.081252135,-0.05375015,-0.030799072,-0.0077205584,0.035735283,-0.04474784,0.01614922,-0.06905586,0.04194401,-0.08832908,0.007873556,0.0044992934,0.021968655,0.010953364,0.07212316,0.023185596,0.001176476,0.04725166,-0.03580423,-0.032344446,0.004752835,-0.056179125,-0.0353235,-0.037251893,-0.005143068,-0.027599342,0.024403147,0.031338032,0.0056410627,0.008425455,-0.0087741045,-0.039870497,-0.011831836,0.042621233,0.048142947,0.01603842,0.06745542,-0.038018145,0.064212486,-0.009636566,0.01607475,-0.0027868534,-0.03761097,-0.011889431,-0.01084577,0.083403684,0.028946918,0.061870236,-0.045382615,0.03812586,0.060426526,-0.017003024,0.02328925,-0.093020774,-0.024072766,0.042720392,-0.022444837,-0.011597358,0.01209231,-0.005953629,0.029691145,-0.011711495,-0.053345133,-0.081757374,-0.049303155,0.032193717,-0.038507868,-0.0060101775,-0.031470332,-0.07362443,0.03575578,0.0245484,0.01985519,-0.15116754,0.050386053,-0.0389634,0.07548389,-0.09264168,-0.012602816,-0.0035539444,-0.046416666,0.015100493,0.100691915,-0.037529,0.04600518,-0.021859422,0.014675989,-0.097722545,-0.053200893,-0.017040461,-0.01629616,-0.012339222,0.06543302,0.07252074,-0.044301156,0.03054835,-0.038570553,0.019310065,-0.005527747,-0.029386865,-0.058248505,0.065443166,0.028506057,0.0878254,0.058387812,0.081721246,-0.0005962058,0.0569388,0.00960514,0.011667701,0.014456492,-0.07218306,-3.0444767e-33,0.06687367,-0.019213954,0.058202963,0.025171444,-0.028259544,-0.04171671,0.03127722,0.04865431,-0.0721459,-0.050143186,-0.029907746,0.06248994,-0.05096076,-0.065192886,0.010210447,-0.077256374,-0.043370593,-0.0901922,-0.066522665,0.026072454,-0.022315364,0.075375885,-0.029968103,-0.0050771856,-0.0046955296,0.0065650316,-0.06791897,0.07368851,0.056069646,-0.021026025,-0.011352883,0.038811427,-0.090847075,-0.056490287,-0.039881036,0.032885592,-0.029586185,0.04753608,0.006341778,-0.0034275919,0.03681623,0.09326265,-0.0532089,0.032396164,0.012605483,0.045778517,-0.0006472902,0.029721813,0.0734947,-0.037117425,-0.09356724,-0.0071634455,-0.032400057,0.010453378,0.06286513,0.006053158,0.060524322,0.091866635,0.063640274,0.055919155,0.053222507,-0.019174587,0.089296214,-0.058922265,0.013745364,-0.022907477,0.028347451,-0.040759467,-0.06820364,-0.061398294,-0.046857864,0.0060627847,0.010442015,-0.07662317,0.031950947,-0.107813306,-0.0016125626,-0.033413693,-0.09989722,-0.03580293,-0.036118146,-0.08276185,0.1280713,0.07596603,0.049225595,-0.026312606,0.0031930343,-0.0012469466,0.000872793,-0.040353082,-0.034516506,-0.036047976,0.004999689,-0.01570079,-0.09783203,1.4186912e-33,-0.0130903,0.04742377,0.02889633,0.09809088,0.043639127,-0.0747531,0.03402747,0.029147558,0.019848268,0.111865185,0.04845458,-0.016222177,0.025501896,0.09813583,-0.002833195,0.0038929228,0.048537854,0.014111116,0.036570966,-0.017512968,-0.079755045,0.043452248,0.10843892,0.061968718,-0.015678233,0.0060376576,0.010718569,-0.03907552,0.040307663,0.04553227,0.041731495,0.083179526,-0.0844024,-0.023447752,0.014485363,-0.00788756,-0.11410207,0.039309476,0.011220049,0.12889338,-0.07970701,0.010658111,-0.06657531,-0.022037188,-0.043459363,-0.030291978,0.05803613,0.06109021,-0.09415837,0.030727232,0.011858702,0.013526454,-0.049580045,-0.052486703,0.013218615,0.0050231596,0.07660412,-0.0046343356,-0.0692584,-0.03353194,0.022288544,0.022836983,0.01361394,-0.036331616,0.008194689,0.09226991,0.08229875,-0.047639843,-0.005033471,0.0066081914,-0.106666416,-0.06385503,0.006385333,0.06318358,0.03146985,-0.060764216,-0.043287244,-0.042385958,-0.07963737,-0.043478705,-0.031840656,0.060027804,0.027829666,0.03728562,-0.07014622,0.0649568,-0.052579004,-0.0034938955,-0.15390792,-0.04426162,0.028217599,0.05593145,0.023251446,0.029115746,-0.040321343,-1.8832651e-08,-0.037536032,0.0005056563,0.013919978,-0.038129315,0.061486673,0.005534046,-0.010768211,-0.16527015,0.061414365,0.019963786,-0.019666849,-0.059065994,0.00083181454,-0.016695915,0.01090875,0.02685501,-0.0008743374,-0.060967337,-0.049258173,0.012731965,-0.03973743,-0.0018210232,-0.03797001,0.00035104176,0.04477184,0.04185603,-0.03215796,-0.09103283,0.0467481,0.017810425,0.037437033,0.054782324,-0.0059442874,0.022557084,0.053393807,-0.03746794,-0.07080435,0.10548834,-0.04215435,0.042547524,0.05064807,0.07560324,0.04072184,-0.0011278372,0.00702018,-0.070807114,0.040035766,0.051417697,0.011316928,-0.028955372,0.04197099,-0.0026867494,-0.056857754,0.0761941,0.030621601,-0.052413527,-0.021492323,0.0957355,0.1013111,-0.082977295,-0.036080915,-0.03327806,0.0029404552,-0.03586191]'::vector)", "Filter": "(cohort = 'static'::text)" } ] } } ] } }, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "eligible": true, "protocol": "mixed-mutations-v2" }, "examples": [ { "event": 0, "operation": "insert", "doc_id": "mutation-000000", "scheduled_s": 0.0, "submitted_s": 0.00023520796094089746, "schedule_delay_ms": 0.23520796094089746, "precondition": true, "status": "success", "ack_ms": 25.385083979927003, "write_started_ms": 1.2910839868709445, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 27.283958974294364, "stable_since_ms": 27.283958974294364, "confirmed_ms": 79.54458403401077, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 35.687500028871, "stable_since_ms": 35.687500028871, "confirmed_ms": 91.86679194681346, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 35.687500028871, "confirmed_ms": 91.86679194681346, "search_minus_ack_ms": 10.302416048943996 }, { "event": 1, "operation": "update", "doc_id": "mutation-000001", "scheduled_s": 0.1, "submitted_s": 0.10020024992991239, "schedule_delay_ms": 0.20024992991238277, "precondition": true, "status": "success", "ack_ms": 10.564083000645041, "write_started_ms": 0.14433299656957388, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 12.741749989800155, "stable_since_ms": 12.741749989800155, "confirmed_ms": 65.52512501366436, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 22.84829202108085, "stable_since_ms": 22.84829202108085, "confirmed_ms": 84.33216705452651, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 22.84829202108085, "confirmed_ms": 84.33216705452651, "search_minus_ack_ms": 12.28420902043581 }, { "event": 2, "operation": "delete", "doc_id": "mutation-000002", "scheduled_s": 0.2, "submitted_s": 0.20024354197084904, "schedule_delay_ms": 0.24354197084902607, "precondition": true, "status": "success", "ack_ms": 3.0582499457523227, "write_started_ms": 0.6067500216886401, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 13.325040927156806, "stable_since_ms": 13.325040927156806, "confirmed_ms": 65.60604099649936, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 24.55791598185897, "stable_since_ms": 24.55791598185897, "confirmed_ms": 88.34829099941999, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 24.55791598185897, "confirmed_ms": 88.34829099941999, "search_minus_ack_ms": 21.499666036106646 } ] }, { "run": "fb2_429e9a_057", "trial": 4, "backend": "pgvector", "scenario": "scifact-small", "status": "complete", "eligible": true, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "summary": { "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "wall_s": 30.000833500060253, "operations": { "insert": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 15.465916018001735, "p50": 20.761041494552046, "p95": 23.143760720267892, "p99": 23.586054141633227, "max": 26.561417034827173 }, "confirmed_ms": { "min": 72.25791702512652, "p50": 81.79750002454966, "p95": 85.95974551863037, "p99": 86.44807503093034, "max": 86.65350009687245 }, "ack_ms": { "min": 4.102499922737479, "p50": 8.112104027532041, "p95": 9.722404408967122, "p99": 10.438014119863531, "max": 14.473667019046843 }, "search_minus_ack_ms": { "min": 10.135459015145898, "p50": 12.158708472270519, "p95": 14.543831598712131, "p99": 15.035759494639937, "max": 15.762667055241764 }, "regressions": 0, "confirmed_within_100ms": 1.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "update": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 16.32995903491974, "p50": 20.613999979104847, "p95": 22.381274751387537, "p99": 23.193735342938457, "max": 23.53879204019904 }, "confirmed_ms": { "min": 72.83670897595584, "p50": 81.82302099885419, "p95": 84.39206248731352, "p99": 84.93205903097987, "max": 86.27025003079325 }, "ack_ms": { "min": 4.5947079779580235, "p50": 8.235791465267539, "p95": 9.929489286150782, "p99": 11.348059787414968, "max": 11.444667004980147 }, "search_minus_ack_ms": { "min": 10.13304095249623, "p50": 12.204917031340301, "p95": 14.53082078951411, "p99": 15.163418431766333, "max": 15.518374973908067 }, "regressions": 0, "confirmed_within_100ms": 1.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "delete": { "count": 100, "success": 100, "success_rate": 1.0, "first_observed_ms": { "min": 12.115292018279433, "p50": 14.145228953566402, "p95": 22.03007848584093, "p99": 22.93911700719036, "max": 23.020791937597096 }, "confirmed_ms": { "min": 65.28524996247143, "p50": 68.69424943579361, "p95": 83.92241245601326, "p99": 86.26365000498481, "max": 86.80241706315428 }, "ack_ms": { "min": 1.410165918059647, "p50": 2.967458451166749, "p95": 4.654841619776562, "p99": 5.24945292272605, "max": 5.277749965898693 }, "search_minus_ack_ms": { "min": 1.088708988390863, "p50": 4.0018545114435256, "p95": 17.739749583415687, "p99": 18.231605670880526, "max": 18.448291928507388 }, "regressions": 0, "confirmed_within_100ms": 1.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 } }, "total_operations": 300, "statuses": { "success": 300, "timeout": 0, "error": 0, "invalid_precondition": 0, "client_overload": 0 }, "query": { "count": 300, "success": 300, "ann_recall_at_10": 0.9993333333333334, "ndcg_at_10": 0.6567157762068453, "latency_ms": { "min": 1.8597079906612635, "p50": 4.735083028208464, "p95": 6.18157065473497, "p99": 6.894889100221902, "max": 15.041499980725348 }, "dispatch_delay_ms": { "min": 0.08837506175041199, "p50": 1.1724794749170542, "p95": 1.997610682155937, "p99": 2.1342036477290094, "max": 2.2030420368537307 } }, "diagnostics": { "schedule_delay_ms": { "min": 0.004584086127579212, "p50": 0.3421880421239365, "p95": 0.741554516134113, "p99": 0.7648982261314606, "max": 0.7923749741167718 }, "probe_dispatch_ms": { "min": 0.04054198507219553, "p50": 0.43604144593700767, "p95": 1.736750011332333, "p99": 2.45151017443277, "max": 2.6203750167042017 }, "probe_service_ms": { "min": 0.4002500791102648, "p50": 2.898228995036334, "p95": 5.853937705978751, "p99": 6.735329062212263, "max": 13.154207961633801 }, "write_service_ms": { "min": 1.241832971572876, "p50": 7.2138329851441085, "p95": 9.111641283379868, "p99": 10.216094154166058, "max": 13.741291943006217 } }, "actual_submission_rate": 9.999974484349542, "metadata": { "backend": "pgvector", "collection": "fb2_429e9a_057", "dataset": { "dataset": "scifact", "source": "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/scifact.zip", "source_sha256": "536e14446a0ba56ed1398ab1055f39fe852686ecad24a6306c80c490fa8e0165", "source_md5": "5f7d1de60b170fc8027bb7898e2efca1", "records": 5183, "queries": 100, "relevant_documents_retained": 107, "seed": 20261028, "license": "Corpus: ODC-By-1.0; claims/evidence: CC-BY-4.0", "license_source": "https://raw.githubusercontent.com/allenai/scifact/master/LICENSE.md", "attribution": "BEIR and original dataset authors; Wikipedia contributors for NQ, SciFact authors and cited article authors for SciFact.", "transformation": "200-word title+text prefix; seeded reservoir of documents with relevant documents for selected test queries retained at front. Simulated mutations; no historical replay.", "model": "sentence-transformers/all-MiniLM-L6-v2", "model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "dimensions": 384, "normalized": true, "max_tokens": 256, "device": "mps", "files": { "corpus.jsonl": "189d58f273468352c9f73c6333c5120dcfb7ba1695984f073f0ebb6c100ed0cd", "queries.json": "c8b333c73353a8c4aeb7d6b63067c9a241aa5d39a0314f0c315ec63019ee9558", "vectors.npy": "b6b21404977151ce565e7d6d57d9931a4679e1ecf4aa44fcd9b29eb54955d09e", "query_vectors.npy": "02324eb7c3b3d2e2f0ec3208453e2ec0efd5d61d9fc845b1f8b7950e913f2730", "UPSTREAM-LICENSE.md": "56ee69bc639cbab8f0b2e4b67091ec5e9390aa99a1d65f0a35e4274686c9a806" } }, "base_count": 4000, "profile": { "dataset": "scifact", "base_count": 4000, "rate": 10, "query_rate": 10, "arrival": "steady" }, "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "preloaded_mutable": 200, "started_utc": "2026-10-29T15:55:41Z", "setup_s": 1.6537487080786377, "backend_before": { "version": "0.8.7", "postgres": "PostgreSQL 16.15 (Debian 16.15-1.pgdg12+2) on aarch64-unknown-linux-gnu, compiled by gcc (Debian 12.2.0-14+deb12u1) 12.2.0, 64-bit", "ef_search": 128, "ack": "autocommit, READ COMMITTED", "transport": "psycopg, connection per worker", "query_plan": [ { "Plan": { "Node Type": "Limit", "Parallel Aware": false, "Async Capable": false, "Startup Cost": 1216.56, "Total Cost": 1229.9, "Plan Rows": 10, "Plan Width": 16, "Plans": [ { "Node Type": "Index Scan", "Parent Relationship": "Outer", "Parallel Aware": false, "Async Capable": false, "Scan Direction": "Forward", "Index Name": "fb2_429e9a_057_hnsw", "Relation Name": "fb2_429e9a_057", "Alias": "fb2_429e9a_057", "Startup Cost": 1216.56, "Total Cost": 6554.0, "Plan Rows": 4000, "Plan Width": 16, "Order By": "(embedding <=> '[-0.025241137,-0.0037172192,-0.046351995,0.046476826,0.01966622,0.015340689,0.01234306,0.07297605,-0.018329926,-0.013840419,-0.07423897,0.0226357,-0.060784698,-0.028650705,0.05173048,-0.03129233,0.07885052,-0.024390804,0.034739245,0.002474597,0.06695868,-0.03807076,0.016879125,-0.007840027,0.026065528,0.014293832,-0.07275669,0.053708147,-0.020156316,0.07862689,-0.122224644,0.023270043,-0.03987285,-0.029884664,0.008407885,0.06157951,-0.045617852,-0.0013952592,-0.1272149,0.012878671,0.03199118,-0.054489113,-0.07601716,0.016042225,-0.02170239,0.0031456149,-0.03146653,0.036349196,0.030850349,0.101148084,-0.04483407,-0.02210456,0.015648471,0.008023243,0.02329231,-0.047311362,-0.00033133395,-0.060376305,-0.031292018,-0.052068412,-0.055494312,0.03995384,-0.05426452,-0.027163653,0.060765423,-0.029587762,0.062952094,-0.032716926,0.03284741,0.03383774,0.025922673,0.09215419,-0.045252554,0.033018086,-0.0063568754,0.003192685,0.036198895,0.071328856,0.061275735,-0.03834579,0.04380842,0.10884386,-0.06443366,0.027753524,0.016000912,0.031735342,-0.048004173,0.04857545,-0.058097336,0.043568186,-0.014455384,0.008579895,0.023373613,0.0023358837,0.070720226,-0.008269884,-0.011370413,-0.016605737,-0.002399004,-0.03909848,0.017561054,0.042421922,0.011341604,0.025210828,0.09824794,0.0038353435,0.015525245,0.082866095,0.0066154706,0.027564062,0.051245574,-0.009089501,0.13282458,0.04902157,-0.013380719,0.042171925,0.033162206,0.048019677,0.029081784,0.116638325,0.00094674755,0.07029037,0.004596228,-0.02894104,-0.0037263758,0.0041222423,-0.04681563,2.8784053e-33,0.09736768,0.02274426,-0.05078067,0.03864741,0.052899208,0.0496459,-0.060387712,-0.08358956,0.07020227,0.042317823,-0.037229892,-0.1324623,0.08240096,0.039485864,-0.09648351,-0.04383094,-0.06106865,0.025340972,0.031598456,0.04274742,-0.0010978275,0.0007333723,0.041061625,0.011878232,-0.12491255,0.06554766,-0.06920273,0.05781889,-0.05118846,0.043497268,-0.029946037,-0.08991292,0.095012516,-0.12172844,-0.048556324,-0.022020115,-0.014565633,-0.045546,-0.05527385,0.06144455,-0.040042106,0.011806558,-0.13014492,-0.024195487,0.008652814,-0.05887302,0.074748434,-0.03390347,-0.026171094,-0.046370335,-0.02871423,0.01613992,0.050530907,-0.072944105,0.0142246755,-0.0029443551,-0.028645331,0.024415584,-0.021900453,-0.035877362,-0.012359783,0.0469807,0.062022846,0.09110963,-0.011605537,-0.02338493,0.0334725,-0.03523439,-0.039391812,0.020923749,0.045133665,-0.0050133714,0.05868334,0.05490907,0.002556749,0.04097332,0.008711031,-0.06200574,0.03028164,-0.11109333,0.0780934,0.012912109,0.07097299,-0.04068633,0.008197527,0.030907633,0.05238045,0.0011536852,-0.01918761,-0.15324447,0.016286323,-0.05679703,-0.04369561,-0.06417541,0.02090238,-3.0734922e-33,-0.005520113,0.007917555,0.051423088,-0.08971186,0.009558501,0.04202714,-0.032392696,0.007947915,-0.033026196,0.057292204,0.049505785,-0.017359108,-0.087639086,-0.047633022,0.012320442,0.036888227,-0.032912884,-0.05555789,-0.030620035,0.040593192,0.056201898,-0.0435655,0.031078862,0.0809475,0.0595156,-0.008109408,-0.10603491,0.0726334,-0.06864153,-0.034877256,0.023989158,-0.04435082,-0.019972347,-0.12077245,0.044683173,0.014618874,-0.020796062,0.114251524,0.01766482,0.0160371,0.012701173,-0.0053563626,0.074527174,-0.021048209,-0.022659509,0.08406541,-0.06423724,-0.071950026,0.05369397,0.01140933,-0.088458955,-0.01212312,0.0024443534,0.12023613,-0.027182862,-0.022696352,0.020270811,0.08858118,-0.10382351,0.086645365,-0.06383489,-0.04042209,0.04494627,0.016389402,0.13325493,0.08580276,0.060353823,0.03661967,0.041859947,0.021768665,0.0073611615,-0.009300088,-0.033082195,-0.040072612,0.06993921,0.02415639,0.012099922,0.034952287,-0.10813522,-0.024580343,0.05526578,-0.043421194,-0.00020226264,0.013380121,-0.013031641,-0.0023380888,0.020561703,-0.026072076,-0.058541298,-0.027999775,-0.024194289,-0.07036205,-0.05425848,-0.07206627,-0.026656583,-2.3484924e-08,-0.037731905,0.015433777,-0.059194244,-0.062783524,-0.047979638,-0.024383545,-0.1138203,0.040749338,-0.038158197,0.008040415,0.024223538,-0.018507883,0.108268395,0.025551548,0.068713784,-0.043465756,0.017785417,0.07973369,-0.0075122416,-0.04605511,-0.047828607,-0.017772881,0.05118608,0.0022625374,0.03168905,0.0143064195,0.017061999,-0.05608364,0.0064040497,0.0072717075,-0.013301113,0.018809363,-0.017954899,0.08202268,-0.06270375,0.029983893,0.0048537604,-0.035181895,-0.06037397,-0.0063187,0.0034529127,0.011674293,0.013009845,0.012574133,0.031151973,-0.087101504,0.030415354,0.036086276,-0.06438181,-0.08931628,0.0122331055,-0.0099123195,0.008941321,-0.024656717,-0.025116868,-0.0044285995,-0.065338664,0.0398255,0.039999187,0.011154338,0.010968719,-0.048901215,0.096052974,-0.08210191]'::vector)", "Filter": "(cohort = 'static'::text)" } ] } } ] }, "count_before": 4200, "idle_duration_s": 10.0, "idle_queries": { "count": 100, "ann_recall": 1.0, "ndcg": 0.6567157762068453, "latency_ms": { "min": 2.4202080676332116, "p50": 5.703333008568734, "p95": 7.499441324034705, "p99": 8.341373036382775, "max": 19.024958019144833 } }, "exact_ndcg": 0.6567157762068453, "resources_before": { "usage_usec": 217217726, "user_usec": 202553247, "system_usec": 14664479, "nice_usec": 0, "nr_periods": 2880, "nr_throttled": 844, "throttled_usec": 88601694, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 913203200, "pids": 23 }, "resources_after": { "usage_usec": 222521348, "user_usec": 207412664, "system_usec": 15108683, "nice_usec": 0, "nr_periods": 3181, "nr_throttled": 844, "throttled_usec": 88601694, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 923103232, "pids": 27 }, "count_after": 4200, "backend_after": { "version": "0.8.7", "postgres": "PostgreSQL 16.15 (Debian 16.15-1.pgdg12+2) on aarch64-unknown-linux-gnu, compiled by gcc (Debian 12.2.0-14+deb12u1) 12.2.0, 64-bit", "ef_search": 128, "ack": "autocommit, READ COMMITTED", "transport": "psycopg, connection per worker", "query_plan": [ { "Plan": { "Node Type": "Limit", "Parallel Aware": false, "Async Capable": false, "Startup Cost": 1252.56, "Total Cost": 1266.17, "Plan Rows": 10, "Plan Width": 16, "Plans": [ { "Node Type": "Index Scan", "Parent Relationship": "Outer", "Parallel Aware": false, "Async Capable": false, "Scan Direction": "Forward", "Index Name": "fb2_429e9a_057_hnsw", "Relation Name": "fb2_429e9a_057", "Alias": "fb2_429e9a_057", "Startup Cost": 1252.56, "Total Cost": 6799.83, "Plan Rows": 4078, "Plan Width": 16, "Order By": "(embedding <=> '[-0.025241137,-0.0037172192,-0.046351995,0.046476826,0.01966622,0.015340689,0.01234306,0.07297605,-0.018329926,-0.013840419,-0.07423897,0.0226357,-0.060784698,-0.028650705,0.05173048,-0.03129233,0.07885052,-0.024390804,0.034739245,0.002474597,0.06695868,-0.03807076,0.016879125,-0.007840027,0.026065528,0.014293832,-0.07275669,0.053708147,-0.020156316,0.07862689,-0.122224644,0.023270043,-0.03987285,-0.029884664,0.008407885,0.06157951,-0.045617852,-0.0013952592,-0.1272149,0.012878671,0.03199118,-0.054489113,-0.07601716,0.016042225,-0.02170239,0.0031456149,-0.03146653,0.036349196,0.030850349,0.101148084,-0.04483407,-0.02210456,0.015648471,0.008023243,0.02329231,-0.047311362,-0.00033133395,-0.060376305,-0.031292018,-0.052068412,-0.055494312,0.03995384,-0.05426452,-0.027163653,0.060765423,-0.029587762,0.062952094,-0.032716926,0.03284741,0.03383774,0.025922673,0.09215419,-0.045252554,0.033018086,-0.0063568754,0.003192685,0.036198895,0.071328856,0.061275735,-0.03834579,0.04380842,0.10884386,-0.06443366,0.027753524,0.016000912,0.031735342,-0.048004173,0.04857545,-0.058097336,0.043568186,-0.014455384,0.008579895,0.023373613,0.0023358837,0.070720226,-0.008269884,-0.011370413,-0.016605737,-0.002399004,-0.03909848,0.017561054,0.042421922,0.011341604,0.025210828,0.09824794,0.0038353435,0.015525245,0.082866095,0.0066154706,0.027564062,0.051245574,-0.009089501,0.13282458,0.04902157,-0.013380719,0.042171925,0.033162206,0.048019677,0.029081784,0.116638325,0.00094674755,0.07029037,0.004596228,-0.02894104,-0.0037263758,0.0041222423,-0.04681563,2.8784053e-33,0.09736768,0.02274426,-0.05078067,0.03864741,0.052899208,0.0496459,-0.060387712,-0.08358956,0.07020227,0.042317823,-0.037229892,-0.1324623,0.08240096,0.039485864,-0.09648351,-0.04383094,-0.06106865,0.025340972,0.031598456,0.04274742,-0.0010978275,0.0007333723,0.041061625,0.011878232,-0.12491255,0.06554766,-0.06920273,0.05781889,-0.05118846,0.043497268,-0.029946037,-0.08991292,0.095012516,-0.12172844,-0.048556324,-0.022020115,-0.014565633,-0.045546,-0.05527385,0.06144455,-0.040042106,0.011806558,-0.13014492,-0.024195487,0.008652814,-0.05887302,0.074748434,-0.03390347,-0.026171094,-0.046370335,-0.02871423,0.01613992,0.050530907,-0.072944105,0.0142246755,-0.0029443551,-0.028645331,0.024415584,-0.021900453,-0.035877362,-0.012359783,0.0469807,0.062022846,0.09110963,-0.011605537,-0.02338493,0.0334725,-0.03523439,-0.039391812,0.020923749,0.045133665,-0.0050133714,0.05868334,0.05490907,0.002556749,0.04097332,0.008711031,-0.06200574,0.03028164,-0.11109333,0.0780934,0.012912109,0.07097299,-0.04068633,0.008197527,0.030907633,0.05238045,0.0011536852,-0.01918761,-0.15324447,0.016286323,-0.05679703,-0.04369561,-0.06417541,0.02090238,-3.0734922e-33,-0.005520113,0.007917555,0.051423088,-0.08971186,0.009558501,0.04202714,-0.032392696,0.007947915,-0.033026196,0.057292204,0.049505785,-0.017359108,-0.087639086,-0.047633022,0.012320442,0.036888227,-0.032912884,-0.05555789,-0.030620035,0.040593192,0.056201898,-0.0435655,0.031078862,0.0809475,0.0595156,-0.008109408,-0.10603491,0.0726334,-0.06864153,-0.034877256,0.023989158,-0.04435082,-0.019972347,-0.12077245,0.044683173,0.014618874,-0.020796062,0.114251524,0.01766482,0.0160371,0.012701173,-0.0053563626,0.074527174,-0.021048209,-0.022659509,0.08406541,-0.06423724,-0.071950026,0.05369397,0.01140933,-0.088458955,-0.01212312,0.0024443534,0.12023613,-0.027182862,-0.022696352,0.020270811,0.08858118,-0.10382351,0.086645365,-0.06383489,-0.04042209,0.04494627,0.016389402,0.13325493,0.08580276,0.060353823,0.03661967,0.041859947,0.021768665,0.0073611615,-0.009300088,-0.033082195,-0.040072612,0.06993921,0.02415639,0.012099922,0.034952287,-0.10813522,-0.024580343,0.05526578,-0.043421194,-0.00020226264,0.013380121,-0.013031641,-0.0023380888,0.020561703,-0.026072076,-0.058541298,-0.027999775,-0.024194289,-0.07036205,-0.05425848,-0.07206627,-0.026656583,-2.3484924e-08,-0.037731905,0.015433777,-0.059194244,-0.062783524,-0.047979638,-0.024383545,-0.1138203,0.040749338,-0.038158197,0.008040415,0.024223538,-0.018507883,0.108268395,0.025551548,0.068713784,-0.043465756,0.017785417,0.07973369,-0.0075122416,-0.04605511,-0.047828607,-0.017772881,0.05118608,0.0022625374,0.03168905,0.0143064195,0.017061999,-0.05608364,0.0064040497,0.0072717075,-0.013301113,0.018809363,-0.017954899,0.08202268,-0.06270375,0.029983893,0.0048537604,-0.035181895,-0.06037397,-0.0063187,0.0034529127,0.011674293,0.013009845,0.012574133,0.031151973,-0.087101504,0.030415354,0.036086276,-0.06438181,-0.08931628,0.0122331055,-0.0099123195,0.008941321,-0.024656717,-0.025116868,-0.0044285995,-0.065338664,0.0398255,0.039999187,0.011154338,0.010968719,-0.048901215,0.096052974,-0.08210191]'::vector)", "Filter": "(cohort = 'static'::text)" } ] } } ] } }, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "eligible": true, "protocol": "mixed-mutations-v2" }, "examples": [ { "event": 0, "operation": "insert", "doc_id": "mutation-000000", "scheduled_s": 0.0, "submitted_s": 0.00018579198513180017, "schedule_delay_ms": 0.18579198513180017, "precondition": true, "status": "success", "ack_ms": 14.473667019046843, "write_started_ms": 0.7323750760406256, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 22.443916997872293, "stable_since_ms": 22.443916997872293, "confirmed_ms": 75.14629198703915, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 26.561417034827173, "stable_since_ms": 26.561417034827173, "confirmed_ms": 81.57004206441343, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 26.561417034827173, "confirmed_ms": 81.57004206441343, "search_minus_ack_ms": 12.08775001578033 }, { "event": 1, "operation": "update", "doc_id": "mutation-000001", "scheduled_s": 0.1, "submitted_s": 0.10065687505993992, "schedule_delay_ms": 0.6568750599399154, "precondition": true, "status": "success", "ack_ms": 4.9102919874712825, "write_started_ms": 0.15633401926606894, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 13.094166992232203, "stable_since_ms": 13.094166992232203, "confirmed_ms": 67.25129194092005, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 16.4054170018062, "stable_since_ms": 16.4054170018062, "confirmed_ms": 75.53841697517782, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 16.4054170018062, "confirmed_ms": 75.53841697517782, "search_minus_ack_ms": 11.495125014334917 }, { "event": 2, "operation": "delete", "doc_id": "mutation-000002", "scheduled_s": 0.2, "submitted_s": 0.20074800006113946, "schedule_delay_ms": 0.7480000611394533, "precondition": true, "status": "success", "ack_ms": 4.648124915547669, "write_started_ms": 0.5877499934285879, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 17.670833971351385, "stable_since_ms": 17.670833971351385, "confirmed_ms": 71.97620894294232, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 22.756458958610892, "stable_since_ms": 22.756458958610892, "confirmed_ms": 83.94449995830655, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 22.756458958610892, "confirmed_ms": 83.94449995830655, "search_minus_ack_ms": 18.108334043063223 } ] }, { "run": "fb2_429e9a_058", "trial": 3, "backend": "pgvector", "scenario": "nq-100k-burst", "status": "complete", "eligible": true, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "summary": { "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "burst", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "wall_s": 30.00175458297599, "operations": { "insert": { "count": 130, "success": 130, "success_rate": 1.0, "first_observed_ms": { "min": 15.681125107221305, "p50": 26.69927093666047, "p95": 33.20730993291363, "p99": 36.26384835341013, "max": 37.87387500051409 }, "confirmed_ms": { "min": 71.45183300599456, "p50": 87.00997900450602, "p95": 95.71547285304405, "p99": 97.73145091137849, "max": 100.2126659732312 }, "ack_ms": { "min": 4.143500002101064, "p50": 15.426792029757053, "p95": 22.325543803162873, "p99": 24.809086734894663, "max": 25.64199990592897 }, "search_minus_ack_ms": { "min": 8.308666991069913, "p50": 11.593291477765888, "p95": 13.202414120314643, "p99": 13.680446210782975, "max": 14.311583014205098 }, "regressions": 0, "confirmed_within_100ms": 0.9923076923076923, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "update": { "count": 130, "success": 130, "success_rate": 1.0, "first_observed_ms": { "min": 15.821667038835585, "p50": 25.46308346791193, "p95": 30.92455990263261, "p99": 32.377239459892735, "max": 32.81750006135553 }, "confirmed_ms": { "min": 70.33229200169444, "p50": 85.47933353111148, "p95": 91.99532480561174, "p99": 94.83070002985187, "max": 95.80000001005828 }, "ack_ms": { "min": 4.159833071753383, "p50": 13.59193748794496, "p95": 18.984024960082024, "p99": 20.14593503670767, "max": 21.16154204122722 }, "search_minus_ack_ms": { "min": 8.348125033080578, "p50": 11.836645484436303, "p95": 13.256527739576995, "p99": 14.13083669845946, "max": 14.775208896026015 }, "regressions": 0, "confirmed_within_100ms": 1.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "delete": { "count": 130, "success": 130, "success_rate": 1.0, "first_observed_ms": { "min": 5.256374948658049, "p50": 13.419603987131268, "p95": 29.47893098462373, "p99": 30.192186108324673, "max": 31.227458966895938 }, "confirmed_ms": { "min": 61.55166693497449, "p50": 73.77933303359896, "p95": 94.38110638875514, "p99": 95.06586944451556, "max": 96.30216704681516 }, "ack_ms": { "min": 0.598916900344193, "p50": 1.8665835377760231, "p95": 4.50947253848426, "p99": 6.012895905878391, "max": 7.586500025354326 }, "search_minus_ack_ms": { "min": 3.307332983240485, "p50": 9.725479001644999, "p95": 26.33820415358059, "p99": 27.050676933722567, "max": 28.483042027801275 }, "regressions": 0, "confirmed_within_100ms": 1.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 } }, "total_operations": 390, "statuses": { "success": 390, "timeout": 0, "error": 0, "invalid_precondition": 0, "client_overload": 0 }, "query": { "count": 300, "success": 300, "ann_recall_at_10": 0.9920000000000001, "ndcg_at_10": 0.7931268675492565, "latency_ms": { "min": 3.1064170179888606, "p50": 9.650895488448441, "p95": 14.25962975481525, "p99": 17.04989290330559, "max": 20.597249967977405 }, "dispatch_delay_ms": { "min": 0.13062497600913048, "p50": 0.8531875209882855, "p95": 1.6506853047758343, "p99": 1.8771670875139532, "max": 1.9997500348836184 } }, "diagnostics": { "schedule_delay_ms": { "min": 0.006542052140900978, "p50": 0.3284790203892385, "p95": 0.6250458175779405, "p99": 0.647647752891949, "max": 0.9170420235022902 }, "probe_dispatch_ms": { "min": 0.02020795363932848, "p50": 0.3851250512525439, "p95": 1.3159829308278856, "p99": 2.121978465002029, "max": 2.5699579855427146 }, "probe_service_ms": { "min": 0.21779094822704792, "p50": 1.935709035024047, "p95": 10.352567280642686, "p99": 14.762893849983811, "max": 22.1047920640558 }, "write_service_ms": { "min": 0.4770419327542186, "p50": 8.011187543161213, "p95": 19.91601277841255, "p99": 22.407681182958186, "max": 24.836999946273863 } }, "actual_submission_rate": 13.009948978041187, "metadata": { "backend": "pgvector", "collection": "fb2_429e9a_058", "dataset": { "dataset": "nq", "source": "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/nq.zip", "source_sha256": "2553bf7bfbab47b1436ca00a34bce57320e18e611fd00999a2a3a1b4714be752", "source_md5": "d4d3d2e48787a744b6f6e691ff534307", "records": 124000, "queries": 100, "relevant_documents_retained": 120, "seed": 20261028, "license": "CC-BY-SA-3.0", "license_source": "https://raw.githubusercontent.com/google-research-datasets/natural-questions/master/nq_open/README.md", "attribution": "BEIR and original dataset authors; Wikipedia contributors for NQ, SciFact authors and cited article authors for SciFact.", "transformation": "200-word title+text prefix; seeded reservoir of documents with relevant documents for selected test queries retained at front. Simulated mutations; no historical replay.", "model": "sentence-transformers/all-MiniLM-L6-v2", "model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "dimensions": 384, "normalized": true, "max_tokens": 256, "device": "mps", "files": { "corpus.jsonl": "9dc487d785e6ee493eccaf764daa3bf8720a8f4fc8c406246065e1ad33a91f5d", "queries.json": "5603f52876ada5d36118b65dea490f1d43900bc159e1804e637796eb9115e999", "vectors.npy": "cb7421eb6ca5964231403f0a922843088f6fd0e01cbf67f2a1076cc7101005db", "query_vectors.npy": "5e666f8d966d8db6b0fd93d4963234d2dffa0ee3910b8fad069a8ee201e8b5d4", "UPSTREAM-LICENSE.md": "8b675521da3fa142ac6bda38c9bcfcc8cff0e2d6128be53a062ed55fb53f7b87" } }, "base_count": 100000, "profile": { "dataset": "nq", "base_count": 100000, "rate": 10, "query_rate": 10, "arrival": "burst" }, "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "burst", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "preloaded_mutable": 260, "started_utc": "2026-10-29T15:56:27Z", "setup_s": 57.92668850009795, "backend_before": { "version": "0.8.7", "postgres": "PostgreSQL 16.15 (Debian 16.15-1.pgdg12+2) on aarch64-unknown-linux-gnu, compiled by gcc (Debian 12.2.0-14+deb12u1) 12.2.0, 64-bit", "ef_search": 128, "ack": "autocommit, READ COMMITTED", "transport": "psycopg, connection per worker", "query_plan": [ { "Plan": { "Node Type": "Limit", "Parallel Aware": false, "Async Capable": false, "Startup Cost": 1674.28, "Total Cost": 1691.41, "Plan Rows": 10, "Plan Width": 18, "Plans": [ { "Node Type": "Index Scan", "Parent Relationship": "Outer", "Parallel Aware": false, "Async Capable": false, "Scan Direction": "Forward", "Index Name": "fb2_429e9a_058_hnsw", "Relation Name": "fb2_429e9a_058", "Alias": "fb2_429e9a_058", "Startup Cost": 1674.28, "Total Cost": 172959.21, "Plan Rows": 100006, "Plan Width": 18, "Order By": "(embedding <=> '[-0.08946895,-0.094362676,0.092023626,-0.0040844884,0.043233823,0.00062519487,0.1075197,-0.035732996,0.013746996,-0.024103122,0.0021706754,-0.04720602,0.04787615,-0.02043222,0.058315646,0.081252135,-0.05375015,-0.030799072,-0.0077205584,0.035735283,-0.04474784,0.01614922,-0.06905586,0.04194401,-0.08832908,0.007873556,0.0044992934,0.021968655,0.010953364,0.07212316,0.023185596,0.001176476,0.04725166,-0.03580423,-0.032344446,0.004752835,-0.056179125,-0.0353235,-0.037251893,-0.005143068,-0.027599342,0.024403147,0.031338032,0.0056410627,0.008425455,-0.0087741045,-0.039870497,-0.011831836,0.042621233,0.048142947,0.01603842,0.06745542,-0.038018145,0.064212486,-0.009636566,0.01607475,-0.0027868534,-0.03761097,-0.011889431,-0.01084577,0.083403684,0.028946918,0.061870236,-0.045382615,0.03812586,0.060426526,-0.017003024,0.02328925,-0.093020774,-0.024072766,0.042720392,-0.022444837,-0.011597358,0.01209231,-0.005953629,0.029691145,-0.011711495,-0.053345133,-0.081757374,-0.049303155,0.032193717,-0.038507868,-0.0060101775,-0.031470332,-0.07362443,0.03575578,0.0245484,0.01985519,-0.15116754,0.050386053,-0.0389634,0.07548389,-0.09264168,-0.012602816,-0.0035539444,-0.046416666,0.015100493,0.100691915,-0.037529,0.04600518,-0.021859422,0.014675989,-0.097722545,-0.053200893,-0.017040461,-0.01629616,-0.012339222,0.06543302,0.07252074,-0.044301156,0.03054835,-0.038570553,0.019310065,-0.005527747,-0.029386865,-0.058248505,0.065443166,0.028506057,0.0878254,0.058387812,0.081721246,-0.0005962058,0.0569388,0.00960514,0.011667701,0.014456492,-0.07218306,-3.0444767e-33,0.06687367,-0.019213954,0.058202963,0.025171444,-0.028259544,-0.04171671,0.03127722,0.04865431,-0.0721459,-0.050143186,-0.029907746,0.06248994,-0.05096076,-0.065192886,0.010210447,-0.077256374,-0.043370593,-0.0901922,-0.066522665,0.026072454,-0.022315364,0.075375885,-0.029968103,-0.0050771856,-0.0046955296,0.0065650316,-0.06791897,0.07368851,0.056069646,-0.021026025,-0.011352883,0.038811427,-0.090847075,-0.056490287,-0.039881036,0.032885592,-0.029586185,0.04753608,0.006341778,-0.0034275919,0.03681623,0.09326265,-0.0532089,0.032396164,0.012605483,0.045778517,-0.0006472902,0.029721813,0.0734947,-0.037117425,-0.09356724,-0.0071634455,-0.032400057,0.010453378,0.06286513,0.006053158,0.060524322,0.091866635,0.063640274,0.055919155,0.053222507,-0.019174587,0.089296214,-0.058922265,0.013745364,-0.022907477,0.028347451,-0.040759467,-0.06820364,-0.061398294,-0.046857864,0.0060627847,0.010442015,-0.07662317,0.031950947,-0.107813306,-0.0016125626,-0.033413693,-0.09989722,-0.03580293,-0.036118146,-0.08276185,0.1280713,0.07596603,0.049225595,-0.026312606,0.0031930343,-0.0012469466,0.000872793,-0.040353082,-0.034516506,-0.036047976,0.004999689,-0.01570079,-0.09783203,1.4186912e-33,-0.0130903,0.04742377,0.02889633,0.09809088,0.043639127,-0.0747531,0.03402747,0.029147558,0.019848268,0.111865185,0.04845458,-0.016222177,0.025501896,0.09813583,-0.002833195,0.0038929228,0.048537854,0.014111116,0.036570966,-0.017512968,-0.079755045,0.043452248,0.10843892,0.061968718,-0.015678233,0.0060376576,0.010718569,-0.03907552,0.040307663,0.04553227,0.041731495,0.083179526,-0.0844024,-0.023447752,0.014485363,-0.00788756,-0.11410207,0.039309476,0.011220049,0.12889338,-0.07970701,0.010658111,-0.06657531,-0.022037188,-0.043459363,-0.030291978,0.05803613,0.06109021,-0.09415837,0.030727232,0.011858702,0.013526454,-0.049580045,-0.052486703,0.013218615,0.0050231596,0.07660412,-0.0046343356,-0.0692584,-0.03353194,0.022288544,0.022836983,0.01361394,-0.036331616,0.008194689,0.09226991,0.08229875,-0.047639843,-0.005033471,0.0066081914,-0.106666416,-0.06385503,0.006385333,0.06318358,0.03146985,-0.060764216,-0.043287244,-0.042385958,-0.07963737,-0.043478705,-0.031840656,0.060027804,0.027829666,0.03728562,-0.07014622,0.0649568,-0.052579004,-0.0034938955,-0.15390792,-0.04426162,0.028217599,0.05593145,0.023251446,0.029115746,-0.040321343,-1.8832651e-08,-0.037536032,0.0005056563,0.013919978,-0.038129315,0.061486673,0.005534046,-0.010768211,-0.16527015,0.061414365,0.019963786,-0.019666849,-0.059065994,0.00083181454,-0.016695915,0.01090875,0.02685501,-0.0008743374,-0.060967337,-0.049258173,0.012731965,-0.03973743,-0.0018210232,-0.03797001,0.00035104176,0.04477184,0.04185603,-0.03215796,-0.09103283,0.0467481,0.017810425,0.037437033,0.054782324,-0.0059442874,0.022557084,0.053393807,-0.03746794,-0.07080435,0.10548834,-0.04215435,0.042547524,0.05064807,0.07560324,0.04072184,-0.0011278372,0.00702018,-0.070807114,0.040035766,0.051417697,0.011316928,-0.028955372,0.04197099,-0.0026867494,-0.056857754,0.0761941,0.030621601,-0.052413527,-0.021492323,0.0957355,0.1013111,-0.082977295,-0.036080915,-0.03327806,0.0029404552,-0.03586191]'::vector)", "Filter": "(cohort = 'static'::text)" } ] } } ] }, "count_before": 100260, "idle_duration_s": 10.0, "idle_queries": { "count": 100, "ann_recall": 0.992, "ndcg": 0.7931268675492565, "latency_ms": { "min": 4.350124974735081, "p50": 19.32810398284346, "p95": 22.567189624533057, "p99": 23.119254000484954, "max": 25.15737502835691 } }, "exact_ndcg": 0.7931268675492565, "resources_before": { "usage_usec": 311928695, "user_usec": 293586819, "system_usec": 18341876, "nice_usec": 0, "nr_periods": 3892, "nr_throttled": 1252, "throttled_usec": 131550353, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 1535201280, "pids": 23 }, "resources_after": { "usage_usec": 323074261, "user_usec": 301311210, "system_usec": 21763051, "nice_usec": 0, "nr_periods": 4193, "nr_throttled": 1252, "throttled_usec": 131550353, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 1565085696, "pids": 35 }, "count_after": 100260, "backend_after": { "version": "0.8.7", "postgres": "PostgreSQL 16.15 (Debian 16.15-1.pgdg12+2) on aarch64-unknown-linux-gnu, compiled by gcc (Debian 12.2.0-14+deb12u1) 12.2.0, 64-bit", "ef_search": 128, "ack": "autocommit, READ COMMITTED", "transport": "psycopg, connection per worker", "query_plan": [ { "Plan": { "Node Type": "Limit", "Parallel Aware": false, "Async Capable": false, "Startup Cost": 1678.95, "Total Cost": 1696.12, "Plan Rows": 10, "Plan Width": 18, "Plans": [ { "Node Type": "Index Scan", "Parent Relationship": "Outer", "Parallel Aware": false, "Async Capable": false, "Scan Direction": "Forward", "Index Name": "fb2_429e9a_058_hnsw", "Relation Name": "fb2_429e9a_058", "Alias": "fb2_429e9a_058", "Startup Cost": 1678.95, "Total Cost": 173342.7, "Plan Rows": 99983, "Plan Width": 18, "Order By": "(embedding <=> '[-0.08946895,-0.094362676,0.092023626,-0.0040844884,0.043233823,0.00062519487,0.1075197,-0.035732996,0.013746996,-0.024103122,0.0021706754,-0.04720602,0.04787615,-0.02043222,0.058315646,0.081252135,-0.05375015,-0.030799072,-0.0077205584,0.035735283,-0.04474784,0.01614922,-0.06905586,0.04194401,-0.08832908,0.007873556,0.0044992934,0.021968655,0.010953364,0.07212316,0.023185596,0.001176476,0.04725166,-0.03580423,-0.032344446,0.004752835,-0.056179125,-0.0353235,-0.037251893,-0.005143068,-0.027599342,0.024403147,0.031338032,0.0056410627,0.008425455,-0.0087741045,-0.039870497,-0.011831836,0.042621233,0.048142947,0.01603842,0.06745542,-0.038018145,0.064212486,-0.009636566,0.01607475,-0.0027868534,-0.03761097,-0.011889431,-0.01084577,0.083403684,0.028946918,0.061870236,-0.045382615,0.03812586,0.060426526,-0.017003024,0.02328925,-0.093020774,-0.024072766,0.042720392,-0.022444837,-0.011597358,0.01209231,-0.005953629,0.029691145,-0.011711495,-0.053345133,-0.081757374,-0.049303155,0.032193717,-0.038507868,-0.0060101775,-0.031470332,-0.07362443,0.03575578,0.0245484,0.01985519,-0.15116754,0.050386053,-0.0389634,0.07548389,-0.09264168,-0.012602816,-0.0035539444,-0.046416666,0.015100493,0.100691915,-0.037529,0.04600518,-0.021859422,0.014675989,-0.097722545,-0.053200893,-0.017040461,-0.01629616,-0.012339222,0.06543302,0.07252074,-0.044301156,0.03054835,-0.038570553,0.019310065,-0.005527747,-0.029386865,-0.058248505,0.065443166,0.028506057,0.0878254,0.058387812,0.081721246,-0.0005962058,0.0569388,0.00960514,0.011667701,0.014456492,-0.07218306,-3.0444767e-33,0.06687367,-0.019213954,0.058202963,0.025171444,-0.028259544,-0.04171671,0.03127722,0.04865431,-0.0721459,-0.050143186,-0.029907746,0.06248994,-0.05096076,-0.065192886,0.010210447,-0.077256374,-0.043370593,-0.0901922,-0.066522665,0.026072454,-0.022315364,0.075375885,-0.029968103,-0.0050771856,-0.0046955296,0.0065650316,-0.06791897,0.07368851,0.056069646,-0.021026025,-0.011352883,0.038811427,-0.090847075,-0.056490287,-0.039881036,0.032885592,-0.029586185,0.04753608,0.006341778,-0.0034275919,0.03681623,0.09326265,-0.0532089,0.032396164,0.012605483,0.045778517,-0.0006472902,0.029721813,0.0734947,-0.037117425,-0.09356724,-0.0071634455,-0.032400057,0.010453378,0.06286513,0.006053158,0.060524322,0.091866635,0.063640274,0.055919155,0.053222507,-0.019174587,0.089296214,-0.058922265,0.013745364,-0.022907477,0.028347451,-0.040759467,-0.06820364,-0.061398294,-0.046857864,0.0060627847,0.010442015,-0.07662317,0.031950947,-0.107813306,-0.0016125626,-0.033413693,-0.09989722,-0.03580293,-0.036118146,-0.08276185,0.1280713,0.07596603,0.049225595,-0.026312606,0.0031930343,-0.0012469466,0.000872793,-0.040353082,-0.034516506,-0.036047976,0.004999689,-0.01570079,-0.09783203,1.4186912e-33,-0.0130903,0.04742377,0.02889633,0.09809088,0.043639127,-0.0747531,0.03402747,0.029147558,0.019848268,0.111865185,0.04845458,-0.016222177,0.025501896,0.09813583,-0.002833195,0.0038929228,0.048537854,0.014111116,0.036570966,-0.017512968,-0.079755045,0.043452248,0.10843892,0.061968718,-0.015678233,0.0060376576,0.010718569,-0.03907552,0.040307663,0.04553227,0.041731495,0.083179526,-0.0844024,-0.023447752,0.014485363,-0.00788756,-0.11410207,0.039309476,0.011220049,0.12889338,-0.07970701,0.010658111,-0.06657531,-0.022037188,-0.043459363,-0.030291978,0.05803613,0.06109021,-0.09415837,0.030727232,0.011858702,0.013526454,-0.049580045,-0.052486703,0.013218615,0.0050231596,0.07660412,-0.0046343356,-0.0692584,-0.03353194,0.022288544,0.022836983,0.01361394,-0.036331616,0.008194689,0.09226991,0.08229875,-0.047639843,-0.005033471,0.0066081914,-0.106666416,-0.06385503,0.006385333,0.06318358,0.03146985,-0.060764216,-0.043287244,-0.042385958,-0.07963737,-0.043478705,-0.031840656,0.060027804,0.027829666,0.03728562,-0.07014622,0.0649568,-0.052579004,-0.0034938955,-0.15390792,-0.04426162,0.028217599,0.05593145,0.023251446,0.029115746,-0.040321343,-1.8832651e-08,-0.037536032,0.0005056563,0.013919978,-0.038129315,0.061486673,0.005534046,-0.010768211,-0.16527015,0.061414365,0.019963786,-0.019666849,-0.059065994,0.00083181454,-0.016695915,0.01090875,0.02685501,-0.0008743374,-0.060967337,-0.049258173,0.012731965,-0.03973743,-0.0018210232,-0.03797001,0.00035104176,0.04477184,0.04185603,-0.03215796,-0.09103283,0.0467481,0.017810425,0.037437033,0.054782324,-0.0059442874,0.022557084,0.053393807,-0.03746794,-0.07080435,0.10548834,-0.04215435,0.042547524,0.05064807,0.07560324,0.04072184,-0.0011278372,0.00702018,-0.070807114,0.040035766,0.051417697,0.011316928,-0.028955372,0.04197099,-0.0026867494,-0.056857754,0.0761941,0.030621601,-0.052413527,-0.021492323,0.0957355,0.1013111,-0.082977295,-0.036080915,-0.03327806,0.0029404552,-0.03586191]'::vector)", "Filter": "(cohort = 'static'::text)" } ] } } ] } }, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "eligible": true, "protocol": "mixed-mutations-v2" }, "examples": [ { "event": 0, "operation": "insert", "doc_id": "mutation-000000", "scheduled_s": 0.0, "submitted_s": 0.00023754197172820568, "schedule_delay_ms": 0.23754197172820568, "precondition": true, "status": "success", "ack_ms": 24.765333044342697, "write_started_ms": 1.1148330522701144, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 27.25666598416865, "stable_since_ms": 27.25666598416865, "confirmed_ms": 78.57562508434057, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 36.65108303539455, "stable_since_ms": 36.65108303539455, "confirmed_ms": 93.54533301666379, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 36.65108303539455, "confirmed_ms": 93.54533301666379, "search_minus_ack_ms": 11.885749991051853 }, { "event": 1, "operation": "update", "doc_id": "mutation-000001", "scheduled_s": 0.025, "submitted_s": 0.02502449997700751, "schedule_delay_ms": 0.02449997700750689, "precondition": true, "status": "success", "ack_ms": 5.111792008392513, "write_started_ms": 0.0701249809935689, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 11.233916971832514, "stable_since_ms": 11.233916971832514, "confirmed_ms": 62.86241707857698, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 15.821667038835585, "stable_since_ms": 15.821667038835585, "confirmed_ms": 70.33229200169444, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 15.821667038835585, "confirmed_ms": 70.33229200169444, "search_minus_ack_ms": 10.709875030443072 }, { "event": 2, "operation": "delete", "doc_id": "mutation-000002", "scheduled_s": 0.05, "submitted_s": 0.05002270801924169, "schedule_delay_ms": 0.02270801924168786, "precondition": true, "status": "success", "ack_ms": 0.8715839358046651, "write_started_ms": 0.13291696086525917, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 11.708584032021463, "stable_since_ms": 11.708584032021463, "confirmed_ms": 63.8537920312956, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 5.285416962578893, "stable_since_ms": 5.285416962578893, "confirmed_ms": 61.51108397170901, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 11.708584032021463, "confirmed_ms": 63.8537920312956, "search_minus_ack_ms": 4.413833026774228 } ] }, { "run": "fb2_429e9a_059", "trial": 4, "backend": "qdrant", "scenario": "nq-100k-burst", "status": "complete", "eligible": true, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "summary": { "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "burst", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "wall_s": 30.06169404205866, "operations": { "insert": { "count": 130, "success": 130, "success_rate": 1.0, "first_observed_ms": { "min": 8.84545804001391, "p50": 68.37083346908912, "p95": 106.35504775564186, "p99": 112.58217758266254, "max": 120.97662501037121 }, "confirmed_ms": { "min": 106.94324993528426, "p50": 213.15550000872463, "p95": 247.7684456622228, "p99": 253.2044493185822, "max": 258.4090830059722 }, "ack_ms": { "min": 2.239458030089736, "p50": 7.292770489584655, "p95": 11.07331260573119, "p99": 12.463997456943616, "max": 15.494749997742474 }, "search_minus_ack_ms": { "min": 1.9630830502137542, "p50": 59.083228989038616, "p95": 100.48272522981279, "p99": 104.22661617631091, "max": 109.4645420089364 }, "regressions": 0, "confirmed_within_100ms": 0.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "update": { "count": 130, "success": 130, "success_rate": 1.0, "first_observed_ms": { "min": 14.961375040002167, "p50": 68.00758349709213, "p95": 102.87115379469469, "p99": 107.606902110856, "max": 111.9532500160858 }, "confirmed_ms": { "min": 111.53750005178154, "p50": 213.27354153618217, "p95": 244.5864896872081, "p99": 250.842116252752, "max": 259.12795902695507 }, "ack_ms": { "min": 2.410832908935845, "p50": 8.382125466596335, "p95": 13.16034214105457, "p99": 18.678657787386296, "max": 22.500750026665628 }, "search_minus_ack_ms": { "min": 1.4434580225497484, "p50": 57.62637499719858, "p95": 96.8294145714026, "p99": 98.91089040436782, "max": 101.1397079564631 }, "regressions": 0, "confirmed_within_100ms": 0.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "delete": { "count": 130, "success": 130, "success_rate": 1.0, "first_observed_ms": { "min": 13.445292017422616, "p50": 18.294395529665053, "p95": 47.51195623539388, "p99": 52.777505710255355, "max": 67.95654201414436 }, "confirmed_ms": { "min": 104.73000002093613, "p50": 155.07977060042322, "p95": 189.79566668276675, "p99": 204.79811205994335, "max": 213.29004201106727 }, "ack_ms": { "min": 2.2414589766412973, "p50": 6.607187970075756, "p95": 11.30358530790545, "p99": 13.106410659383975, "max": 13.666167040355504 }, "search_minus_ack_ms": { "min": -1.7137079266831279, "p50": 1.1046454310417175, "p95": 42.54592936486006, "p99": 45.38654529838822, "max": 54.48441696353257 }, "regressions": 0, "confirmed_within_100ms": 0.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 } }, "total_operations": 390, "statuses": { "success": 390, "timeout": 0, "error": 0, "invalid_precondition": 0, "client_overload": 0 }, "query": { "count": 300, "success": 300, "ann_recall_at_10": 0.9879999999999999, "ndcg_at_10": 0.7931268675492565, "latency_ms": { "min": 3.0686670215800405, "p50": 8.892229001503438, "p95": 47.127238987013705, "p99": 49.843042922439054, "max": 51.014082971960306 }, "dispatch_delay_ms": { "min": 0.10562501847743988, "p50": 1.2698750360868871, "p95": 2.2135041013825685, "p99": 2.5600216235034163, "max": 7.759292027913034 } }, "diagnostics": { "schedule_delay_ms": { "min": 0.005333055742795523, "p50": 0.3594579990018332, "p95": 0.6320731178850014, "p99": 0.8436612591605211, "max": 6.260750046930497 }, "probe_dispatch_ms": { "min": 0.0237500062212348, "p50": 0.3934170235879719, "p95": 1.4671992743387814, "p99": 2.51128161442466, "max": 15.578333986923099 }, "probe_service_ms": { "min": 0.6706248968839645, "p50": 5.2630414720624685, "p95": 48.83897852851078, "p99": 50.77903799945488, "max": 61.372124939225614 }, "write_service_ms": { "min": 1.8537499709054828, "p50": 6.890458462294191, "p95": 11.401416861917822, "p99": 13.745367418741816, "max": 21.36320900171995 } }, "actual_submission_rate": 13.009906626920976, "metadata": { "backend": "qdrant", "collection": "fb2_429e9a_059", "dataset": { "dataset": "nq", "source": "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/nq.zip", "source_sha256": "2553bf7bfbab47b1436ca00a34bce57320e18e611fd00999a2a3a1b4714be752", "source_md5": "d4d3d2e48787a744b6f6e691ff534307", "records": 124000, "queries": 100, "relevant_documents_retained": 120, "seed": 20261028, "license": "CC-BY-SA-3.0", "license_source": "https://raw.githubusercontent.com/google-research-datasets/natural-questions/master/nq_open/README.md", "attribution": "BEIR and original dataset authors; Wikipedia contributors for NQ, SciFact authors and cited article authors for SciFact.", "transformation": "200-word title+text prefix; seeded reservoir of documents with relevant documents for selected test queries retained at front. Simulated mutations; no historical replay.", "model": "sentence-transformers/all-MiniLM-L6-v2", "model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "dimensions": 384, "normalized": true, "max_tokens": 256, "device": "mps", "files": { "corpus.jsonl": "9dc487d785e6ee493eccaf764daa3bf8720a8f4fc8c406246065e1ad33a91f5d", "queries.json": "5603f52876ada5d36118b65dea490f1d43900bc159e1804e637796eb9115e999", "vectors.npy": "cb7421eb6ca5964231403f0a922843088f6fd0e01cbf67f2a1076cc7101005db", "query_vectors.npy": "5e666f8d966d8db6b0fd93d4963234d2dffa0ee3910b8fad069a8ee201e8b5d4", "UPSTREAM-LICENSE.md": "8b675521da3fa142ac6bda38c9bcfcc8cff0e2d6128be53a062ed55fb53f7b87" } }, "base_count": 100000, "profile": { "dataset": "nq", "base_count": 100000, "rate": 10, "query_rate": 10, "arrival": "burst" }, "config": { "duration_s": 30.0, "rate": 10, "query_rate": 10, "arrival": "burst", "burst_multiplier": 4, "deadline_s": 5, "poll_ms": 10, "stable_ms": 50, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "preloaded_mutable": 260, "started_utc": "2026-10-29T15:58:10Z", "setup_s": 41.7520588330226, "backend_before": { "version": "1.20.0", "collection": { "status": "green", "optimizer_status": "ok", "indexed_vectors_count": 100260, "points_count": 100260, "segments_count": 2, "config": { "params": { "vectors": { "size": 384, "distance": "Cosine" }, "shard_number": 1, "replication_factor": 1, "write_consistency_factor": 1, "on_disk_payload": true }, "hnsw_config": { "m": 16, "ef_construct": 128, "full_scan_threshold": 10, "max_indexing_threads": 0, "on_disk": false }, "optimizer_config": { "deleted_threshold": 0.2, "vacuum_min_vector_number": 1000, "default_segment_number": 0, "max_segment_size": null, "memmap_threshold": null, "indexing_threshold": 1024, "flush_interval_sec": 5, "max_optimization_threads": null, "prevent_unoptimized": null }, "wal_config": { "wal_capacity_mb": 32, "wal_segments_ahead": 0, "wal_retain_closed": 1 }, "quantization_config": null }, "payload_schema": { "cohort": { "data_type": "keyword", "points": 100260 } }, "update_queue": { "length": 0 }, "created_at": "2026-10-29T15:58:10.783Z" }, "ack": "wait=true", "transport": "HTTP REST, pooled httpx" }, "count_before": 100260, "idle_duration_s": 10.0, "idle_queries": { "count": 100, "ann_recall": 0.988, "ndcg": 0.7931268675492565, "latency_ms": { "min": 4.280291963368654, "p50": 10.442896513268352, "p95": 12.510554748587309, "p99": 15.745030863909223, "max": 49.13583304733038 } }, "exact_ndcg": 0.7931268675492565, "resources_before": { "usage_usec": 155755300, "user_usec": 147467569, "system_usec": 8287731, "nice_usec": 114058000, "nr_periods": 3152, "nr_throttled": 243, "throttled_usec": 1137849, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 645685248, "pids": 29 }, "resources_after": { "usage_usec": 162752396, "user_usec": 153708064, "system_usec": 9044332, "nice_usec": 114058000, "nr_periods": 3454, "nr_throttled": 243, "throttled_usec": 1137849, "nr_bursts": 0, "burst_usec": 0, "memory_bytes": 624676864, "pids": 31 }, "count_after": 100260, "backend_after": { "version": "1.20.0", "collection": { "status": "green", "optimizer_status": "ok", "indexed_vectors_count": 100260, "points_count": 100260, "segments_count": 2, "config": { "params": { "vectors": { "size": 384, "distance": "Cosine" }, "shard_number": 1, "replication_factor": 1, "write_consistency_factor": 1, "on_disk_payload": true }, "hnsw_config": { "m": 16, "ef_construct": 128, "full_scan_threshold": 10, "max_indexing_threads": 0, "on_disk": false }, "optimizer_config": { "deleted_threshold": 0.2, "vacuum_min_vector_number": 1000, "default_segment_number": 0, "max_segment_size": null, "memmap_threshold": null, "indexing_threshold": 1024, "flush_interval_sec": 5, "max_optimization_threads": null, "prevent_unoptimized": null }, "wal_config": { "wal_capacity_mb": 32, "wal_segments_ahead": 0, "wal_retain_closed": 1 }, "quantization_config": null }, "payload_schema": { "cohort": { "data_type": "keyword", "points": 100260 } }, "update_queue": { "length": 0 }, "created_at": "2026-10-29T15:58:10.783Z" }, "ack": "wait=true", "transport": "HTTP REST, pooled httpx" } }, "gates": { "arrival_dispatch": true, "probe_dispatch": true, "query_dispatch": true, "ann_recall": true, "preconditions": true, "client_capacity": true, "cohort_integrity": true, "ann_index_used": true }, "eligible": true, "protocol": "mixed-mutations-v2" }, "examples": [ { "event": 0, "operation": "insert", "doc_id": "mutation-000000", "scheduled_s": 0.0, "submitted_s": 0.000167582998983562, "schedule_delay_ms": 0.167582998983562, "precondition": true, "status": "success", "ack_ms": 5.335874971933663, "write_started_ms": 1.0024170624092221, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 14.032916980795562, "stable_since_ms": 14.032916980795562, "confirmed_ms": 67.33854196500033, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 58.34554205648601, "stable_since_ms": 58.34554205648601, "confirmed_ms": 195.4557920107618, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 58.34554205648601, "confirmed_ms": 195.4557920107618, "search_minus_ack_ms": 53.00966708455235 }, { "event": 1, "operation": "update", "doc_id": "mutation-000001", "scheduled_s": 0.025, "submitted_s": 0.025296624982729554, "schedule_delay_ms": 0.2966249827295528, "precondition": true, "status": "success", "ack_ms": 3.4590830327942967, "write_started_ms": 0.5126670002937317, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 13.934249989688396, "stable_since_ms": 13.934249989688396, "confirmed_ms": 67.54525005817413, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 97.20695798750967, "stable_since_ms": 97.20695798750967, "confirmed_ms": 237.4042080482468, "consecutive": 3, "regressions": 0, "requests": 4 }, "first_observed_ms": 97.20695798750967, "confirmed_ms": 237.4042080482468, "search_minus_ack_ms": 93.74787495471537 }, { "event": 2, "operation": "delete", "doc_id": "mutation-000002", "scheduled_s": 0.05, "submitted_s": 0.050273583037778735, "schedule_delay_ms": 0.2735830377787324, "precondition": true, "status": "success", "ack_ms": 3.186083980835974, "write_started_ms": 0.4960419610142708, "write_finished": true, "write_error": null, "probe_errors": [], "control": { "first_ms": 12.9370839567855, "stable_since_ms": 12.9370839567855, "confirmed_ms": 66.00120896473527, "consecutive": 3, "regressions": 0, "requests": 4 }, "search": { "first_ms": 44.25775003619492, "stable_since_ms": 44.25775003619492, "confirmed_ms": 143.1934169959277, "consecutive": 3, "regressions": 0, "requests": 3 }, "first_observed_ms": 44.25775003619492, "confirmed_ms": 143.1934169959277, "search_minus_ack_ms": 41.071666055358946 } ] } ], "calibration": [ { "config": { "duration_s": 1, "rate": 20, "query_rate": 0, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 1, "poll_ms": 5, "stable_ms": 20, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "wall_s": 1.0016757500125095, "operations": { "insert": { "count": 7, "success": 7, "success_rate": 1.0, "first_observed_ms": { "min": 0.1591669861227274, "p50": 5.441791960038245, "p95": 5.744695686735213, "p99": 5.762705490924418, "max": 5.767207941971719 }, "confirmed_ms": { "min": 20.96604206599295, "p50": 26.405166019685566, "p95": 26.72856681747362, "p99": 26.75744697218761, "max": 26.764667010866106 }, "ack_ms": { "min": 0.0867500202730298, "p50": 0.13887498062103987, "p95": 0.40993364527821535, "p99": 0.43115399777889246, "max": 0.4364590859040618 }, "search_minus_ack_ms": { "min": 0.018415972590446472, "p50": 5.1984169986099005, "p95": 5.384291987866163, "p99": 5.412191990762949, "max": 5.419166991487145 }, "regressions": 0, "confirmed_within_100ms": 1.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "update": { "count": 7, "success": 7, "success_rate": 1.0, "first_observed_ms": { "min": 0.0725829740986228, "p50": 5.285458057187498, "p95": 5.518937308806926, "p99": 5.521387478802353, "max": 5.52200002130121 }, "confirmed_ms": { "min": 20.88895905762911, "p50": 25.708750006742775, "p95": 26.577509054914117, "p99": 26.62946908734739, "max": 26.642459095455706 }, "ack_ms": { "min": 0.04983297549188137, "p50": 0.1425830414518714, "p95": 0.23662864696234465, "p99": 0.2412584749981761, "max": 0.24241593200713396 }, "search_minus_ack_ms": { "min": 0.02274999860674143, "p50": 5.142875015735626, "p95": 5.305278720334172, "p99": 5.317588550969958, "max": 5.320666008628905 }, "regressions": 0, "confirmed_within_100ms": 1.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "delete": { "count": 6, "success": 6, "success_rate": 1.0, "first_observed_ms": { "min": 5.217375000938773, "p50": 5.47222956083715, "p95": 5.704687442630529, "p99": 5.746437422931194, "max": 5.7568749180063605 }, "confirmed_ms": { "min": 26.273083058185875, "p50": 26.411459024529904, "p95": 27.35326043330133, "p99": 27.513852017000318, "max": 27.553999912925065 }, "ack_ms": { "min": 0.15049998182803392, "p50": 0.23925001733005047, "p95": 0.43065674253739417, "p99": 0.46989855472929776, "max": 0.47970900777727365 }, "search_minus_ack_ms": { "min": 5.066875019110739, "p50": 5.253000068478286, "p95": 5.298635456711054, "p99": 5.304360669106245, "max": 5.305791972205043 }, "regressions": 0, "confirmed_within_100ms": 1.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 } }, "total_operations": 20, "statuses": { "success": 20, "timeout": 0, "error": 0, "invalid_precondition": 0, "client_overload": 0 }, "query": { "count": 0, "success": 0, "ann_recall_at_10": null, "ndcg_at_10": null, "latency_ms": { "min": null, "p50": null, "p95": null, "p99": null, "max": null }, "dispatch_delay_ms": { "min": null, "p50": null, "p95": null, "p99": null, "max": null } }, "diagnostics": { "schedule_delay_ms": { "min": 0.014874944463394435, "p50": 0.22897949675099172, "p95": 0.5986297386698508, "p99": 0.6031258939765055, "max": 0.6042499328031692 }, "probe_dispatch_ms": { "min": 0.060542020946741104, "p50": 0.35691692028194666, "p95": 0.6420204881578684, "p99": 0.7643539574928582, "max": 1.4822499360889196 }, "probe_service_ms": { "min": 0.0007499475032091141, "p50": 0.004208064638078213, "p95": 0.016250007320195436, "p99": 0.0296254875138402, "max": 0.03291701432317495 }, "write_service_ms": { "min": 0.0009170034900307655, "p50": 0.002333486918359995, "p95": 0.005719193723052741, "p99": 0.006511050742119549, "max": 0.006709014996886253 } }, "actual_submission_rate": 19.996911864205146, "injected_visibility_ms": 0, "injected_ack_ms": 0 }, { "config": { "duration_s": 1, "rate": 20, "query_rate": 0, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 1, "poll_ms": 5, "stable_ms": 20, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "wall_s": 1.001616375055164, "operations": { "insert": { "count": 7, "success": 7, "success_rate": 1.0, "first_observed_ms": { "min": 26.590875000692904, "p50": 26.717292028479278, "p95": 26.833754393737763, "p99": 26.86208447208628, "max": 26.86916699167341 }, "confirmed_ms": { "min": 47.28441603947431, "p50": 47.370666987262666, "p95": 47.84277871949598, "p99": 47.85028854152188, "max": 47.85216599702835 }, "ack_ms": { "min": 0.08854200132191181, "p50": 0.16250007320195436, "p95": 0.2513794461265206, "p99": 0.26380952913314104, "max": 0.26691704988479614 }, "search_minus_ack_ms": { "min": 26.447084033861756, "p50": 26.544458931311965, "p95": 26.62080000154674, "p99": 26.62716002203524, "max": 26.628750027157366 }, "regressions": 0, "confirmed_within_100ms": 1.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "update": { "count": 7, "success": 7, "success_rate": 1.0, "first_observed_ms": { "min": 26.430374942719936, "p50": 26.47570800036192, "p95": 26.59770444734022, "p99": 26.62027453770861, "max": 26.625917060300708 }, "confirmed_ms": { "min": 47.27475007530302, "p50": 47.412000014446676, "p95": 47.75829658610746, "p99": 47.85912660649046, "max": 47.88433411158621 }, "ack_ms": { "min": 0.07599999662488699, "p50": 0.10379205923527479, "p95": 0.22363365860655898, "p99": 0.2486940124072134, "max": 0.25495910085737705 }, "search_minus_ack_ms": { "min": 26.27983398269862, "p50": 26.37095795944333, "p95": 26.419570471625775, "p99": 26.4263804606162, "max": 26.428082957863808 }, "regressions": 0, "confirmed_within_100ms": 1.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "delete": { "count": 6, "success": 6, "success_rate": 1.0, "first_observed_ms": { "min": 26.354582980275154, "p50": 26.440666464623064, "p95": 26.606916973832995, "p99": 26.640816975850612, "max": 26.649291976355016 }, "confirmed_ms": { "min": 46.96937499102205, "p50": 47.39266645628959, "p95": 47.454948275117204, "p99": 47.45932327932678, "max": 47.460417030379176 }, "ack_ms": { "min": 0.10920804925262928, "p50": 0.1536869676783681, "p95": 0.2557603002060205, "p99": 0.28071851120330393, "max": 0.2869580639526248 }, "search_minus_ack_ms": { "min": 26.192415971308947, "p50": 26.29185450496152, "p95": 26.368521415861323, "p99": 26.370171416783705, "max": 26.3705839170143 }, "regressions": 0, "confirmed_within_100ms": 1.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 } }, "total_operations": 20, "statuses": { "success": 20, "timeout": 0, "error": 0, "invalid_precondition": 0, "client_overload": 0 }, "query": { "count": 0, "success": 0, "ann_recall_at_10": null, "ndcg_at_10": null, "latency_ms": { "min": null, "p50": null, "p95": null, "p99": null, "max": null }, "dispatch_delay_ms": { "min": null, "p50": null, "p95": null, "p99": null, "max": null } }, "diagnostics": { "schedule_delay_ms": { "min": 0.0694999936967644, "p50": 0.24947950150816522, "p95": 0.4853694327175564, "p99": 0.5828074477612938, "max": 0.6071669515222283 }, "probe_dispatch_ms": { "min": 0.07262500002980232, "p50": 0.27354201301932335, "p95": 0.5226454697549344, "p99": 0.5749572766944766, "max": 0.6780829280614853 }, "probe_service_ms": { "min": 0.0009160721674561501, "p50": 0.005249457899481058, "p95": 0.012556550791487101, "p99": 0.015944531187415123, "max": 0.040749902836978436 }, "write_service_ms": { "min": 0.0021670712158083916, "p50": 0.004062487278133631, "p95": 0.007260398706421256, "p99": 0.007418504683300853, "max": 0.007458031177520752 } }, "actual_submission_rate": 19.9964409920956, "injected_visibility_ms": 25, "injected_ack_ms": 0 }, { "config": { "duration_s": 1, "rate": 20, "query_rate": 0, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 1, "poll_ms": 5, "stable_ms": 20, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "wall_s": 1.0756221250630915, "operations": { "insert": { "count": 7, "success": 7, "success_rate": 1.0, "first_observed_ms": { "min": 101.52283299248666, "p50": 102.53791697323322, "p95": 103.8494786946103, "p99": 104.20762853231281, "max": 104.29716599173844 }, "confirmed_ms": { "min": 122.01808299869299, "p50": 123.14237502869219, "p95": 124.61067080730572, "p99": 124.80630054371431, "max": 124.85520797781646 }, "ack_ms": { "min": 0.08991698268800974, "p50": 0.13033405411988497, "p95": 0.22885382641106836, "p99": 0.2578035788610577, "max": 0.2650410169735551 }, "search_minus_ack_ms": { "min": 101.41795792151242, "p50": 102.43500000797212, "p95": 103.62442497862503, "p99": 103.95058497553691, "max": 104.03212497476488 }, "regressions": 0, "confirmed_within_100ms": 0.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "update": { "count": 7, "success": 7, "success_rate": 1.0, "first_observed_ms": { "min": 100.3025829559192, "p50": 101.89420892857015, "p95": 103.95285360282287, "p99": 104.45420353440568, "max": 104.57954101730138 }, "confirmed_ms": { "min": 121.4332499075681, "p50": 122.38554109353572, "p95": 124.46280806325376, "p99": 124.78872808627784, "max": 124.87020809203386 }, "ack_ms": { "min": 0.10012497659772635, "p50": 0.13983401004225016, "p95": 0.29525356367230404, "p99": 0.33008355647325505, "max": 0.3387910546734929 }, "search_minus_ack_ms": { "min": 100.15908302739263, "p50": 101.7543749185279, "p95": 103.6856623715721, "p99": 104.12973244441673, "max": 104.24074996262789 }, "regressions": 0, "confirmed_within_100ms": 0.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "delete": { "count": 6, "success": 6, "success_rate": 1.0, "first_observed_ms": { "min": 100.29854206368327, "p50": 102.31120802927762, "p95": 102.8509165334981, "p99": 102.96838334179483, "max": 102.99775004386902 }, "confirmed_ms": { "min": 121.12629204057157, "p50": 122.86308297188953, "p95": 123.60905174864456, "p99": 123.73757673194632, "max": 123.76970797777176 }, "ack_ms": { "min": 0.1005829544737935, "p50": 0.12916600098833442, "p95": 0.14339579502120614, "p99": 0.1438791980035603, "max": 0.14400004874914885 }, "search_minus_ack_ms": { "min": 100.19000002648681, "p50": 102.17581246979535, "p95": 102.71184402517974, "p99": 102.82730241306126, "max": 102.85616701003164 }, "regressions": 0, "confirmed_within_100ms": 0.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 } }, "total_operations": 20, "statuses": { "success": 20, "timeout": 0, "error": 0, "invalid_precondition": 0, "client_overload": 0 }, "query": { "count": 0, "success": 0, "ann_recall_at_10": null, "ndcg_at_10": null, "latency_ms": { "min": null, "p50": null, "p95": null, "p99": null, "max": null }, "dispatch_delay_ms": { "min": null, "p50": null, "p95": null, "p99": null, "max": null } }, "diagnostics": { "schedule_delay_ms": { "min": 0.037167011760153024, "p50": 0.2878125756979044, "p95": 0.6426170747727576, "p99": 0.6817570012062849, "max": 0.6915419828146668 }, "probe_dispatch_ms": { "min": 0.05533394869416952, "p50": 0.3565829829312861, "p95": 0.5840627360157669, "p99": 0.6860011932440104, "max": 0.9718750370666385 }, "probe_service_ms": { "min": 0.0009159557521343231, "p50": 0.0037919962778687477, "p95": 0.008001999231055377, "p99": 0.01426062313839797, "max": 0.022541964426636696 }, "write_service_ms": { "min": 0.0015830155462026596, "p50": 0.0037294812500476837, "p95": 0.006200367351993928, "p99": 0.016206520376726968, "max": 0.01870805863291025 } }, "actual_submission_rate": 19.991434385765963, "injected_visibility_ms": 100, "injected_ack_ms": 0 }, { "config": { "duration_s": 1, "rate": 20, "query_rate": 0, "arrival": "steady", "burst_multiplier": 4, "deadline_s": 1, "poll_ms": 5, "stable_ms": 20, "min_confirmations": 3, "write_workers": 4, "probe_workers": 32, "query_workers": 4, "max_active": 1024 }, "wall_s": 1.0543286659521982, "operations": { "insert": { "count": 7, "success": 7, "success_rate": 1.0, "first_observed_ms": { "min": 5.41787501424551, "p50": 5.6168329901993275, "p95": 5.761371343396604, "p99": 5.7659414364025, "max": 5.767083959653974 }, "confirmed_ms": { "min": 26.382540934719145, "p50": 26.798957958817482, "p95": 26.874929352197796, "p99": 26.879019413609058, "max": 26.880041928961873 }, "ack_ms": { "min": 101.78820800501853, "p50": 104.83666695654392, "p95": 105.41132078506052, "p99": 105.43403054587543, "max": 105.43970798607916 }, "search_minus_ack_ms": { "min": -99.76833302062005, "p50": -99.08862505108118, "p95": -96.53377060312778, "p99": -96.40302051324397, "max": -96.37033299077302 }, "regressions": 0, "confirmed_within_100ms": 1.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "update": { "count": 7, "success": 7, "success_rate": 1.0, "first_observed_ms": { "min": 0.18025003373622894, "p50": 5.569875007495284, "p95": 5.976846325211227, "p99": 5.988236493431032, "max": 5.991084035485983 }, "confirmed_ms": { "min": 21.0027500288561, "p50": 26.780333020724356, "p95": 27.427658822853118, "p99": 27.479698958341032, "max": 27.49270899221301 }, "ack_ms": { "min": 101.03154194075614, "p50": 101.16029100026935, "p95": 104.28951273206621, "p99": 104.37850255984813, "max": 104.40075001679361 }, "search_minus_ack_ms": { "min": -104.09420798532665, "p50": -95.70362500380725, "p95": -95.11953288456425, "p99": -95.10427298722789, "max": -95.1004580128938 }, "regressions": 0, "confirmed_within_100ms": 1.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 }, "delete": { "count": 6, "success": 6, "success_rate": 1.0, "first_observed_ms": { "min": 5.504000000655651, "p50": 5.8204164379276335, "p95": 6.045510206604376, "p99": 6.054801988648251, "max": 6.057124934159219 }, "confirmed_ms": { "min": 26.532792020589113, "p50": 27.223166485782713, "p95": 27.46671822387725, "p99": 27.482676436193287, "max": 27.486665989272296 }, "ack_ms": { "min": 100.56616691872478, "p50": 104.17610447620973, "p95": 105.47769721597433, "p99": 105.50807220861316, "max": 105.51566595677286 }, "search_minus_ack_ms": { "min": -99.5761250378564, "p50": -98.22095854906365, "p95": -95.329718926223, "p99": -95.1156773196999, "max": -95.06216691806912 }, "regressions": 0, "confirmed_within_100ms": 1.0, "confirmed_within_1000ms": 1.0, "errors": 0, "timeouts": 0, "invalid_preconditions": 0 } }, "total_operations": 20, "statuses": { "success": 20, "timeout": 0, "error": 0, "invalid_precondition": 0, "client_overload": 0 }, "query": { "count": 0, "success": 0, "ann_recall_at_10": null, "ndcg_at_10": null, "latency_ms": { "min": null, "p50": null, "p95": null, "p99": null, "max": null }, "dispatch_delay_ms": { "min": null, "p50": null, "p95": null, "p99": null, "max": null } }, "diagnostics": { "schedule_delay_ms": { "min": 0.05129103083167941, "p50": 0.3052705316804183, "p95": 0.6104267435148358, "p99": 0.6156517611816525, "max": 0.6169580155983567 }, "probe_dispatch_ms": { "min": 0.10245793964713812, "p50": 0.4480000352486968, "p95": 0.7360207091551274, "p99": 0.7817788806278257, "max": 0.7968749850988388 }, "probe_service_ms": { "min": 0.0015830155462026596, "p50": 0.009312992915511131, "p95": 0.029499438824132085, "p99": 0.03384837182238696, "max": 0.037207966670393944 }, "write_service_ms": { "min": 100.32674996182323, "p50": 103.55766699649394, "p95": 105.02289378200658, "p99": 105.02887871698476, "max": 105.03037495072931 } }, "actual_submission_rate": 19.998013356421325, "injected_visibility_ms": 0, "injected_ack_ms": 100 } ], "continuations": [ { "resumed_utc": "20261029T024749Z", "completed_prefix": 6, "reason": "Host/session interruption; containers restarted. Completed trials retained; partial trial archived and restarted.", "measurement_source_unchanged": true, "environment_unchanged": true, "containers": [ { "name": "/freshness-v2-qdrant", "started_at": "2026-10-29T02:40:21.60969425Z", "image_id": "sha256:9bfc89f9914fc14876a88c05270685921a55b8877b254a98bcc1cc2f0a424cd2" }, { "name": "/freshness-v2-weaviate", "started_at": "2026-10-29T02:40:21.605728917Z", "image_id": "sha256:e96e2e04ee6f401a8dcdf89adb80b610516e1491a33769dfed6d1318801e1129" }, { "name": "/freshness-v2-pgvector", "started_at": "2026-10-29T02:40:21.611165667Z", "image_id": "sha256:7b822b0aac60967beb1ea5e576b8602c94c300a157d187f385ae3e0da199b90a" } ], "resume_source_sha256": "096a6e11f0bd05d96773e1fb85c8b275605ffc63adca8d2bf59c65d621417baa", "interrupted_attempt": { "run": "fb2_429e9a_006", "trial": 2, "backend": "pgvector", "scenario": "scifact-small", "status": "interrupted", "restarted": true } }, { "resumed_utc": "20261029T154029Z", "completed_prefix": 44, "reason": "Resuming a user-requested pause; completed trials retained.", "measurement_source_unchanged": true, "environment_unchanged": true, "containers": [ { "name": "/freshness-v2-qdrant", "started_at": "2026-10-29T15:39:02.715151083Z", "image_id": "sha256:9bfc89f9914fc14876a88c05270685921a55b8877b254a98bcc1cc2f0a424cd2" }, { "name": "/freshness-v2-weaviate", "started_at": "2026-10-29T15:39:02.8226175Z", "image_id": "sha256:e96e2e04ee6f401a8dcdf89adb80b610516e1491a33769dfed6d1318801e1129" }, { "name": "/freshness-v2-pgvector", "started_at": "2026-10-29T15:39:02.929619208Z", "image_id": "sha256:7b822b0aac60967beb1ea5e576b8602c94c300a157d187f385ae3e0da199b90a" } ], "resume_source_sha256": "41690b01c349d795d0e17d9e0dd7e22108198b11262253f901246724f24deea1", "paused_checkpoint": { "paused_utc": "2026-10-29T03:31:21Z", "reason": "User requested a temporary pause", "completed_prefix": 44, "planned_runs": 60, "checkpoint_pause": true, "measurement_process_stopped": 96406 } } ] }