mirror of
https://mau.dev/maunium/synapse.git
synced 2024-12-14 21:33:53 +01:00
Add OpenTracing for database activity. (#10113)
This adds quite a lot of OpenTracing decoration for database activity. Specifically it adds tracing at four different levels: * emit a span for each "interaction" - ie, the top level database function that we tend to call "transaction", but isn't really, because it can end up as multiple transactions. * emit a span while we hold a database connection open * emit a span for each database transaction - actual actual transaction. * emit a span for each database query. I'm aware this might be quite a lot of overhead, but even just running it on a local Synapse it looks really interesting, and I hope the overhead can be offset just by turning down the sampling frequency and finding other ways of tracing requests of interest (eg, the `force_tracing_for_users` setting).
This commit is contained in:
parent
1d143074c5
commit
9eea4646be
3 changed files with 57 additions and 30 deletions
1
changelog.d/10113.feature
Normal file
1
changelog.d/10113.feature
Normal file
|
@ -0,0 +1 @@
|
||||||
|
Report OpenTracing spans for database activity.
|
|
@ -271,6 +271,12 @@ class SynapseTags:
|
||||||
# HTTP request tag (used to distinguish full vs incremental syncs, etc)
|
# HTTP request tag (used to distinguish full vs incremental syncs, etc)
|
||||||
REQUEST_TAG = "request_tag"
|
REQUEST_TAG = "request_tag"
|
||||||
|
|
||||||
|
# Text description of a database transaction
|
||||||
|
DB_TXN_DESC = "db.txn_desc"
|
||||||
|
|
||||||
|
# Uniqueish ID of a database transaction
|
||||||
|
DB_TXN_ID = "db.txn_id"
|
||||||
|
|
||||||
|
|
||||||
# Block everything by default
|
# Block everything by default
|
||||||
# A regex which matches the server_names to expose traces for.
|
# A regex which matches the server_names to expose traces for.
|
||||||
|
|
|
@ -40,6 +40,7 @@ from twisted.enterprise import adbapi
|
||||||
|
|
||||||
from synapse.api.errors import StoreError
|
from synapse.api.errors import StoreError
|
||||||
from synapse.config.database import DatabaseConnectionConfig
|
from synapse.config.database import DatabaseConnectionConfig
|
||||||
|
from synapse.logging import opentracing
|
||||||
from synapse.logging.context import (
|
from synapse.logging.context import (
|
||||||
LoggingContext,
|
LoggingContext,
|
||||||
current_context,
|
current_context,
|
||||||
|
@ -313,6 +314,13 @@ class LoggingTransaction:
|
||||||
start = time.time()
|
start = time.time()
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
with opentracing.start_active_span(
|
||||||
|
"db.query",
|
||||||
|
tags={
|
||||||
|
opentracing.tags.DATABASE_TYPE: "sql",
|
||||||
|
opentracing.tags.DATABASE_STATEMENT: sql,
|
||||||
|
},
|
||||||
|
):
|
||||||
return func(sql, *args)
|
return func(sql, *args)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
sql_logger.debug("[SQL FAIL] {%s} %s", self.name, e)
|
sql_logger.debug("[SQL FAIL] {%s} %s", self.name, e)
|
||||||
|
@ -525,6 +533,13 @@ class DatabasePool:
|
||||||
exception_callbacks=exception_callbacks,
|
exception_callbacks=exception_callbacks,
|
||||||
)
|
)
|
||||||
try:
|
try:
|
||||||
|
with opentracing.start_active_span(
|
||||||
|
"db.txn",
|
||||||
|
tags={
|
||||||
|
opentracing.SynapseTags.DB_TXN_DESC: desc,
|
||||||
|
opentracing.SynapseTags.DB_TXN_ID: name,
|
||||||
|
},
|
||||||
|
):
|
||||||
r = func(cursor, *args, **kwargs)
|
r = func(cursor, *args, **kwargs)
|
||||||
conn.commit()
|
conn.commit()
|
||||||
return r
|
return r
|
||||||
|
@ -653,6 +668,7 @@ class DatabasePool:
|
||||||
logger.warning("Starting db txn '%s' from sentinel context", desc)
|
logger.warning("Starting db txn '%s' from sentinel context", desc)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
with opentracing.start_active_span(f"db.{desc}"):
|
||||||
result = await self.runWithConnection(
|
result = await self.runWithConnection(
|
||||||
self.new_transaction,
|
self.new_transaction,
|
||||||
desc,
|
desc,
|
||||||
|
@ -718,6 +734,9 @@ class DatabasePool:
|
||||||
with LoggingContext(
|
with LoggingContext(
|
||||||
str(curr_context), parent_context=parent_context
|
str(curr_context), parent_context=parent_context
|
||||||
) as context:
|
) as context:
|
||||||
|
with opentracing.start_active_span(
|
||||||
|
operation_name="db.connection",
|
||||||
|
):
|
||||||
sched_duration_sec = monotonic_time() - start_time
|
sched_duration_sec = monotonic_time() - start_time
|
||||||
sql_scheduling_timer.observe(sched_duration_sec)
|
sql_scheduling_timer.observe(sched_duration_sec)
|
||||||
context.add_database_scheduled(sched_duration_sec)
|
context.add_database_scheduled(sched_duration_sec)
|
||||||
|
@ -725,6 +744,7 @@ class DatabasePool:
|
||||||
if self.engine.is_connection_closed(conn):
|
if self.engine.is_connection_closed(conn):
|
||||||
logger.debug("Reconnecting closed database connection")
|
logger.debug("Reconnecting closed database connection")
|
||||||
conn.reconnect()
|
conn.reconnect()
|
||||||
|
opentracing.log_kv({"message": "reconnected"})
|
||||||
|
|
||||||
try:
|
try:
|
||||||
if db_autocommit:
|
if db_autocommit:
|
||||||
|
|
Loading…
Reference in a new issue