forked from MirrorHub/synapse
Censor redactions in DB after a month
This commit is contained in:
parent
92c1550f4a
commit
a4bf72c30c
|
@ -23,7 +23,7 @@ from functools import wraps
|
|||
from six import iteritems, text_type
|
||||
from six.moves import range
|
||||
|
||||
from canonicaljson import json
|
||||
from canonicaljson import encode_canonical_json, json
|
||||
from prometheus_client import Counter, Histogram
|
||||
|
||||
from twisted.internet import defer
|
||||
|
@ -33,6 +33,7 @@ from synapse.api.constants import EventTypes
|
|||
from synapse.api.errors import SynapseError
|
||||
from synapse.events import EventBase # noqa: F401
|
||||
from synapse.events.snapshot import EventContext # noqa: F401
|
||||
from synapse.events.utils import prune_event_dict
|
||||
from synapse.logging.context import PreserveLoggingContext, make_deferred_yieldable
|
||||
from synapse.logging.utils import log_function
|
||||
from synapse.metrics import BucketCollector
|
||||
|
@ -262,6 +263,13 @@ class EventsStore(
|
|||
|
||||
hs.get_clock().looping_call(read_forward_extremities, 60 * 60 * 1000)
|
||||
|
||||
def _censor_redactions():
|
||||
return run_as_background_process(
|
||||
"_censor_redactions", self._censor_redactions
|
||||
)
|
||||
|
||||
hs.get_clock().looping_call(_censor_redactions, 10 * 60 * 1000)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _read_forward_extremities(self):
|
||||
def fetch(txn):
|
||||
|
@ -1548,6 +1556,84 @@ class EventsStore(
|
|||
(event.event_id, event.redacts),
|
||||
)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _censor_redactions(self):
|
||||
"""Censors all redactions older than a month that haven't been censored.
|
||||
|
||||
By censor we mean update the event_json table with the redacted event.
|
||||
|
||||
Returns:
|
||||
Deferred
|
||||
"""
|
||||
|
||||
if self.stream_ordering_month_ago is None:
|
||||
return
|
||||
|
||||
max_pos = self.stream_ordering_month_ago
|
||||
|
||||
# We fetch all redactions that point to an event that we have that has
|
||||
# a stream ordering from over a month ago, that we haven't yet censored
|
||||
# in the DB.
|
||||
sql = """
|
||||
SELECT er.event_id, redacts FROM redactions
|
||||
INNER JOIN events AS er USING (event_id)
|
||||
INNER JOIN events AS eb ON (er.room_id = eb.room_id AND redacts = eb.event_id)
|
||||
WHERE NOT have_censored
|
||||
AND ? <= er.stream_ordering AND er.stream_ordering <= ?
|
||||
ORDER BY er.stream_ordering ASC
|
||||
LIMIT ?
|
||||
"""
|
||||
|
||||
rows = yield self._execute(
|
||||
"_censor_redactions_fetch", None, sql, -max_pos, max_pos, 100
|
||||
)
|
||||
|
||||
updates = []
|
||||
|
||||
for redaction_id, event_id in rows:
|
||||
redaction_event = yield self.get_event(redaction_id, allow_none=True)
|
||||
original_event = yield self.get_event(
|
||||
event_id, allow_rejected=True, allow_none=True
|
||||
)
|
||||
|
||||
# The SQL above ensures that we have both the redaction and
|
||||
# original event, so if the `get_event` calls return None it
|
||||
# means that the redaction wasn't allowed. Either way we know that
|
||||
# the result won't change so we mark the fact that we've checked.
|
||||
if (
|
||||
redaction_event
|
||||
and original_event
|
||||
and original_event.internal_metadata.is_redacted()
|
||||
):
|
||||
# Redaction was allowed
|
||||
pruned_json = encode_canonical_json(
|
||||
prune_event_dict(original_event.get_dict())
|
||||
)
|
||||
else:
|
||||
# Redaction wasn't allowed
|
||||
pruned_json = None
|
||||
|
||||
updates.append((redaction_id, event_id, pruned_json))
|
||||
|
||||
def _update_censor_txn(txn):
|
||||
for redaction_id, event_id, pruned_json in updates:
|
||||
if pruned_json:
|
||||
self._simple_update_one_txn(
|
||||
txn,
|
||||
table="event_json",
|
||||
keyvalues={"event_id": event_id},
|
||||
updatevalues={"json": pruned_json},
|
||||
)
|
||||
|
||||
self._simple_update_one_txn(
|
||||
txn,
|
||||
table="redactions",
|
||||
keyvalues={"event_id": redaction_id},
|
||||
updatevalues={"have_censored": True},
|
||||
)
|
||||
|
||||
yield self.runInteraction("_update_censor_txn", _update_censor_txn)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def count_daily_messages(self):
|
||||
"""
|
||||
|
|
17
synapse/storage/schema/delta/56/redaction_censor.sql
Normal file
17
synapse/storage/schema/delta/56/redaction_censor.sql
Normal file
|
@ -0,0 +1,17 @@
|
|||
/* Copyright 2019 The Matrix.org Foundation C.I.C.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
ALTER TABLE redactions ADD COLUMN have_censored BOOL NOT NULL DEFAULT false;
|
||||
CREATE INDEX redactions_have_censored ON redactions(event_id) WHERE not have_censored;
|
|
@ -17,6 +17,8 @@
|
|||
|
||||
from mock import Mock
|
||||
|
||||
from canonicaljson import json
|
||||
|
||||
from twisted.internet import defer
|
||||
|
||||
from synapse.api.constants import EventTypes, Membership
|
||||
|
@ -286,3 +288,72 @@ class RedactionTestCase(unittest.HomeserverTestCase):
|
|||
self.assertEqual(
|
||||
fetched.unsigned["redacted_because"].event_id, redaction_event_id2
|
||||
)
|
||||
|
||||
def test_redact_censor(self):
|
||||
"""Test that a redacted event gets censored in the DB after a month
|
||||
"""
|
||||
|
||||
self.get_success(
|
||||
self.inject_room_member(self.room1, self.u_alice, Membership.JOIN)
|
||||
)
|
||||
|
||||
msg_event = self.get_success(self.inject_message(self.room1, self.u_alice, "t"))
|
||||
|
||||
# Check event has not been redacted:
|
||||
event = self.get_success(self.store.get_event(msg_event.event_id))
|
||||
|
||||
self.assertObjectHasAttributes(
|
||||
{
|
||||
"type": EventTypes.Message,
|
||||
"user_id": self.u_alice.to_string(),
|
||||
"content": {"body": "t", "msgtype": "message"},
|
||||
},
|
||||
event,
|
||||
)
|
||||
|
||||
self.assertFalse("redacted_because" in event.unsigned)
|
||||
|
||||
# Redact event
|
||||
reason = "Because I said so"
|
||||
self.get_success(
|
||||
self.inject_redaction(self.room1, msg_event.event_id, self.u_alice, reason)
|
||||
)
|
||||
|
||||
event = self.get_success(self.store.get_event(msg_event.event_id))
|
||||
|
||||
self.assertTrue("redacted_because" in event.unsigned)
|
||||
|
||||
self.assertObjectHasAttributes(
|
||||
{
|
||||
"type": EventTypes.Message,
|
||||
"user_id": self.u_alice.to_string(),
|
||||
"content": {},
|
||||
},
|
||||
event,
|
||||
)
|
||||
|
||||
event_json = self.get_success(
|
||||
self.store._simple_select_one_onecol(
|
||||
table="event_json",
|
||||
keyvalues={"event_id": msg_event.event_id},
|
||||
retcol="json",
|
||||
)
|
||||
)
|
||||
|
||||
self.assert_dict(
|
||||
{"content": {"body": "t", "msgtype": "message"}}, json.loads(event_json)
|
||||
)
|
||||
|
||||
# Advance by 30 days
|
||||
self.reactor.advance(60 * 60 * 24 * 31)
|
||||
self.reactor.advance(60 * 60 * 2)
|
||||
|
||||
event_json = self.get_success(
|
||||
self.store._simple_select_one_onecol(
|
||||
table="event_json",
|
||||
keyvalues={"event_id": msg_event.event_id},
|
||||
retcol="json",
|
||||
)
|
||||
)
|
||||
|
||||
self.assert_dict({"content": {}}, json.loads(event_json))
|
||||
|
|
Loading…
Reference in a new issue