diff --git a/include/ircd/m/vm.h b/include/ircd/m/vm.h index 2ef6e55d8..a5925ac2c 100644 --- a/include/ircd/m/vm.h +++ b/include/ircd/m/vm.h @@ -246,21 +246,6 @@ struct ircd::m::vm::opts /// update the optimized present state table of the room if it is proper. bool present {true}; - /// Toggles whether event may be added to the room head table which means - /// it is considered unreferenced by any other event at this time. It is - /// safe for this to always be true if events are evaluated in order. If - /// `present` is false this should be set to false but they are not tied. - bool room_head {true}; - - /// Toggles whether the prev_events of this event are removed from the - /// room head table, now that this event has referenced them. It is safe - /// for this to always be true. - bool room_head_resolve {true}; - - /// Toggles whether the state btree is updated; this should be consistently - /// true or false for all events in a room. - bool history {true}; - /// Evaluate in EDU mode. Input must not have event_id and none will be /// generated for it. bool edu {false}; diff --git a/matrix/media.cc b/matrix/media.cc new file mode 100644 index 000000000..c26640579 --- /dev/null +++ b/matrix/media.cc @@ -0,0 +1,507 @@ +// The Construct +// +// Copyright (C) The Construct Developers, Authors & Contributors +// Copyright (C) 2016-2020 Jason Volk +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice is present in all copies. The +// full license for this software is available in the LICENSE file. + +decltype(ircd::m::media::log) +ircd::m::media::log +{ + "m.media" +}; + +decltype(ircd::m::media::events_prefetch) +ircd::m::media::events_prefetch +{ + { "name", "ircd.media.file.prefetch.events" }, + { "default", 16L }, +}; + +decltype(ircd::m::media::downloading) +ircd::m::media::downloading; + +decltype(ircd::m::media::downloading_dock) +ircd::m::media::downloading_dock; + +// +// media::file +// + +ircd::m::room::id::buf +ircd::m::media::file::download(const mxc &mxc, + const m::user::id &user_id, + const string_view &remote) +{ + const m::room::id::buf room_id + { + file::room_id(mxc) + }; + + if(remote && my_host(remote)) + return room_id; + + if(!remote && my_host(mxc.server)) + return room_id; + + download(mxc, user_id, room_id, remote); + return room_id; +} + +ircd::m::room +ircd::m::media::file::download(const mxc &mxc, + const m::user::id &user_id, + const m::room::id &room_id, + string_view remote) +try +{ + auto iit + { + downloading.emplace(room_id) + }; + + if(!iit.second) + { + downloading_dock.wait([&room_id] + { + return !downloading.count(room_id); + }); + + return room_id; + } + + const unwind uw{[&iit] + { + downloading.erase(iit.first); + downloading_dock.notify_all(); + }}; + + if(exists(room_id)) + return room_id; + + if(!remote) + remote = mxc.server; + + const unique_buffer buf + { + 16_KiB + }; + + const auto pair + { + download(buf, mxc, remote) + }; + + const auto &head + { + pair.first + }; + + const const_buffer &content + { + pair.second + }; + + char mime_type_buf[64]; + const auto &content_type + { + magic::mime(mime_type_buf, content) + }; + + if(content_type != head.content_type) + log::dwarning + { + log, "Server %s claims thumbnail %s is '%s' but we think it is '%s'", + remote, + mxc.mediaid, + head.content_type, + content_type, + }; + + m::vm::copts vmopts; + const m::room room + { + room_id, &vmopts + }; + + create(room, user_id, "file"); + const unwind_exceptional purge{[&room] + { + m::room::purge(room); + }}; + + const size_t written + { + file::write(room, user_id, content, content_type) + }; + + return room; +} +catch(const ircd::server::unavailable &e) +{ + throw m::error + { + http::BAD_GATEWAY, "M_MEDIA_UNAVAILABLE", + "Server '%s' is not available for media for '%s/%s' :%s", + remote, + mxc.server, + mxc.mediaid, + e.what() + }; +} + +decltype(ircd::m::media::download_timeout) +ircd::m::media::download_timeout +{ + { "name", "ircd.media.download.timeout" }, + { "default", 30L }, +}; + +std::pair +< + ircd::http::response::head, + ircd::unique_buffer +> +ircd::m::media::file::download(const mutable_buffer &buf_, + const mxc &mxc, + string_view remote, + server::request::opts *const opts) +{ + assert(remote || !my_host(mxc.server)); + assert(!remote || !my_host(remote)); + + mutable_buffer buf{buf_}; + fed::request::opts fedopts; + fedopts.remote = remote?: mxc.server; + json::get<"method"_>(fedopts.request) = "GET"; + json::get<"uri"_>(fedopts.request) = fmt::sprintf + { + buf, "/_matrix/media/r0/download/%s/%s", + mxc.server, + mxc.mediaid, + }; + consume(buf, size(json::get<"uri"_>(fedopts.request))); + + //TODO: --- This should use the progress callback to build blocks + fed::request remote_request + { + buf, std::move(fedopts) + }; + + if(!remote_request.wait(seconds(download_timeout), std::nothrow)) + throw m::error + { + http::GATEWAY_TIMEOUT, "M_MEDIA_DOWNLOAD_TIMEOUT", + "Server '%s' did not respond with media for '%s/%s' in time", + remote, + mxc.server, + mxc.mediaid + }; + + const auto &code + { + remote_request.get() + }; + + if(code != http::OK) + return {}; + + parse::buffer pb{remote_request.in.head}; + parse::capstan pc{pb}; + pc.read += size(remote_request.in.head); + return std::pair> + { + pc, std::move(remote_request.in.dynamic) + }; +} + +size_t +ircd::m::media::file::write(const m::room &room, + const m::user::id &user_id, + const const_buffer &content, + const string_view &content_type) +{ + static const size_t BLK_SZ + { + 32_KiB + }; + + static const size_t BLK_ENCODE_BUF_SZ + { + 48_KiB + }; + + static const size_t BLK_ENCODE_BUF_ALIGN + { + 64 + }; + + static_assert + ( + BLK_ENCODE_BUF_SZ >= b64::encode_unpadded_size(BLK_SZ) + ); + + const unique_mutable_buffer blk_encode_buf + { + BLK_ENCODE_BUF_SZ, + BLK_ENCODE_BUF_ALIGN, + }; + + send(room, user_id, "ircd.file.stat", "size", json::members + { + { "value", long(size(content)) } + }); + + send(room, user_id, "ircd.file.stat", "type", json::members + { + { "value", content_type } + }); + + size_t off{0}, wrote{0}; + while(off < size(content)) + { + const size_t blk_sz + { + std::min(size(content) - off, BLK_SZ) + }; + + const const_buffer blk_raw + { + content + off, blk_sz + }; + + const string_view blk + { + b64::encode_unpadded(blk_encode_buf, blk_raw) + }; + + const auto event_id + { + send(room, user_id, "ircd.file.block", json::members + { + { "data.ub64", blk }, + }) + }; + + off += size(blk_raw); + wrote += size(blk); + assert(size(blk) == b64::encode_unpadded_size(blk_raw)); + } + + //assert(wrote == b64::encode_unpadded_size(off)); + assert(off == size(content)); + return off; +} + +size_t +ircd::m::media::file::read(const m::room &room, + const closure &closure) +{ + static const size_t BLK_DECODE_BUF_SZ + { + 64_KiB + }; + + static const size_t BLK_DECODE_BUF_ALIGN + { + 64 + }; + + const unique_mutable_buffer blk_decode_buf + { + BLK_DECODE_BUF_SZ, + BLK_DECODE_BUF_ALIGN, + }; + + static const event::fetch::opts fopts + { + event::keys::include + { + "content", "type" + } + }; + + room::events it + { + room, 1, &fopts + }; + + if(!it) + return 0; + + room::events epf + { + room, 1, &fopts + }; + + size_t + decoded_bytes(0), + encoding_bytes(0), + events_fetched(0), + events_prefetched(0); + for(; it; ++it) + { + for(; epf && events_prefetched < events_fetched + events_prefetch; ++epf) + events_prefetched += epf.prefetch(); + + ++events_fetched; + const m::event &event + { + *it + }; + + if(json::get<"type"_>(event) != "ircd.file.block") + continue; + + const json::object content + { + json::get<"content"_>(event) + }; + + const json::string &blk_encoded + { + content["data.ub64"] // unpadded base64 + }; + + const const_buffer blk + { + b64::decode(blk_decode_buf, blk_encoded) + }; + + #if 0 + log::debug + { + log, "File %s read event_idx:%lu events[fetched:%zu prefetched:%zu] encoded:%zu decoded:%zu total_encoded:%zu total_decoded:%zu", + string_view{room.room_id}, + it.event_idx(), + events_fetched, + events_prefetched, + size(blk_encoded), + size(blk), + encoding_bytes, + decoded_bytes, + }; + #endif + + closure(blk); + decoded_bytes += size(blk); + encoding_bytes += size(blk_encoded); + assert(size(blk) == b64::decode_size(blk_encoded)); + } + + //assert(decoded_bytes == b64::decode_size(encoding_bytes)); + return decoded_bytes; +} + +// +// media::file +// + +ircd::m::room::id::buf +ircd::m::media::file::room_id(const mxc &mxc) +{ + m::room::id::buf ret; + room_id(ret, mxc); + return ret; +} + +ircd::m::room::id +ircd::m::media::file::room_id(room::id::buf &out, + const mxc &mxc) +{ + thread_local char buf[512]; + const auto path + { + mxc.path(buf) + }; + + const sha256::buf hash + { + sha256{path} + }; + + out = + { + b58::encode(buf, hash), my_host() + }; + + return out; +} + +// +// media::mxc +// + +ircd::m::media::mxc::mxc(const string_view &server, + const string_view &mediaid) +:server +{ + split(lstrip(server, "mxc://"), '/').first +} +,mediaid +{ + mediaid?: rsplit(server, '/').second +} +{ + if(unlikely(empty(server))) + throw m::BAD_REQUEST + { + "Invalid MXC: missing server parameter." + }; + + if(unlikely(empty(mediaid))) + throw m::BAD_REQUEST + { + "Invalid MXC: missing mediaid parameter." + }; +} + +ircd::m::media::mxc::mxc(const string_view &uri) +:server +{ + split(lstrip(uri, "mxc://"), '/').first +} +,mediaid +{ + rsplit(uri, '/').second +} +{ + if(unlikely(empty(server))) + throw m::BAD_REQUEST + { + "Invalid MXC: missing server parameter." + }; + + if(unlikely(empty(mediaid))) + throw m::BAD_REQUEST + { + "Invalid MXC: missing mediaid parameter." + }; +} + +ircd::string_view +ircd::m::media::mxc::uri(const mutable_buffer &out) +const +{ + return fmt::sprintf + { + out, "mxc://%s/%s", + server, + mediaid + }; +} + +ircd::string_view +ircd::m::media::mxc::path(const mutable_buffer &out) +const +{ + return fmt::sprintf + { + out, "%s/%s", + server, + mediaid + }; +} diff --git a/matrix/vm_execute.cc b/matrix/vm_execute.cc index 756526661..39cc756ec 100644 --- a/matrix/vm_execute.cc +++ b/matrix/vm_execute.cc @@ -905,12 +905,18 @@ ircd::m::vm::write_append(eval &eval, wopts.interpose = eval.txn.get(); wopts.event_idx = eval.sequence; wopts.json_source = opts.json_source; - wopts.appendix.set(dbs::appendix::ROOM_STATE_SPACE, opts.history); // Don't update or resolve the room head with this shit. - const bool dummy_event(json::get<"type"_>(event) == "org.matrix.dummy_event"); - wopts.appendix.set(dbs::appendix::ROOM_HEAD, opts.room_head && !dummy_event); - wopts.appendix.set(dbs::appendix::ROOM_HEAD_RESOLVE, opts.room_head_resolve); + const bool dummy_event + { + json::get<"type"_>(event) == "org.matrix.dummy_event" + }; + + wopts.appendix.set + ( + dbs::appendix::ROOM_HEAD, + wopts.appendix[dbs::appendix::ROOM_HEAD] && !dummy_event + ); if(opts.present && json::get<"state_key"_>(event)) { @@ -943,7 +949,7 @@ ircd::m::vm::write_append(eval &eval, //XXX const auto &[pass, fail] { - opts.auth && !eval.room_internal? + opts.phase[phase::AUTH_PRES] && !eval.room_internal? room::auth::check_present(event): room::auth::passfail{true, {}} }; @@ -957,8 +963,17 @@ ircd::m::vm::write_append(eval &eval, what(fail), }; - wopts.appendix.set(dbs::appendix::ROOM_STATE, pass); - wopts.appendix.set(dbs::appendix::ROOM_JOINED, pass); + wopts.appendix.set + ( + dbs::appendix::ROOM_STATE, + pass && wopts.appendix[dbs::appendix::ROOM_STATE] + ); + + wopts.appendix.set + ( + dbs::appendix::ROOM_JOINED, + pass && wopts.appendix[dbs::appendix::ROOM_JOINED] + ); } } diff --git a/modules/console.cc b/modules/console.cc index 4a81d2b16..2d3dbb8ec 100644 --- a/modules/console.cc +++ b/modules/console.cc @@ -14356,8 +14356,8 @@ console_cmd__fed__backfill(opt &out, const string_view &line) m::vm::opts vmopts; vmopts.nothrows = -1; - vmopts.room_head = false; - vmopts.room_head_resolve = true; + vmopts.wopts.appendix[m::dbs::appendix::ROOM_HEAD_RESOLVE] = false; + vmopts.wopts.appendix[m::dbs::appendix::ROOM_HEAD] = false; vmopts.phase.set(m::vm::phase::FETCH_PREV, false); vmopts.phase.set(m::vm::phase::FETCH_STATE, false); vmopts.node_id = remote; @@ -14726,8 +14726,8 @@ console_cmd__fed__auth(opt &out, const string_view &line) m::vm::opts vmopts; vmopts.node_id = opts.remote; vmopts.nothrows = -1; - vmopts.room_head = false; - vmopts.room_head_resolve = true; + vmopts.wopts.appendix[m::dbs::appendix::ROOM_HEAD_RESOLVE] = false; + vmopts.wopts.appendix[m::dbs::appendix::ROOM_HEAD] = false; vmopts.phase.set(m::vm::phase::FETCH_PREV, false); vmopts.phase.set(m::vm::phase::FETCH_STATE, false); vmopts.notify_servers = false; diff --git a/modules/media/media.cc b/modules/media/media.cc index 694808635..b02316e6e 100644 --- a/modules/media/media.cc +++ b/modules/media/media.cc @@ -289,7 +289,6 @@ try }; m::vm::copts vmopts; - vmopts.history = false; const m::room room { room_id, &vmopts diff --git a/modules/media/upload.cc b/modules/media/upload.cc index 51a887daf..195b798ae 100644 --- a/modules/media/upload.cc +++ b/modules/media/upload.cc @@ -66,7 +66,6 @@ post__upload(client &client, }; m::vm::copts vmopts; - vmopts.history = false; const m::room room { room_id, &vmopts