ircd:Ⓜ️:vm: Remove legacy options; fix appendix masking.

include/ircd/m/vm.h

@@ -246,21 +246,6 @@ struct ircd::m::vm::opts
 	/// update the optimized present state table of the room if it is proper.
 	bool present {true};
 
-	/// Toggles whether event may be added to the room head table which means
-	/// it is considered unreferenced by any other event at this time. It is
-	/// safe for this to always be true if events are evaluated in order. If
-	/// `present` is false this should be set to false but they are not tied.
-	bool room_head {true};
-
-	/// Toggles whether the prev_events of this event are removed from the
-	/// room head table, now that this event has referenced them. It is safe
-	/// for this to always be true.
-	bool room_head_resolve {true};
-
-	/// Toggles whether the state btree is updated; this should be consistently
-	/// true or false for all events in a room.
-	bool history {true};
-
 	/// Evaluate in EDU mode. Input must not have event_id and none will be
 	/// generated for it.
 	bool edu {false};

matrix/media.cc

@@ -0,0 +1,507 @@
+// The Construct
+//
+// Copyright (C) The Construct Developers, Authors & Contributors
+// Copyright (C) 2016-2020 Jason Volk <jason@zemos.net>
+//
+// Permission to use, copy, modify, and/or distribute this software for any
+// purpose with or without fee is hereby granted, provided that the above
+// copyright notice and this permission notice is present in all copies. The
+// full license for this software is available in the LICENSE file.
+
+decltype(ircd::m::media::log)
+ircd::m::media::log
+{
+	"m.media"
+};
+
+decltype(ircd::m::media::events_prefetch)
+ircd::m::media::events_prefetch
+{
+	{ "name",     "ircd.media.file.prefetch.events" },
+	{ "default",  16L                               },
+};
+
+decltype(ircd::m::media::downloading)
+ircd::m::media::downloading;
+
+decltype(ircd::m::media::downloading_dock)
+ircd::m::media::downloading_dock;
+
+//
+// media::file
+//
+
+ircd::m::room::id::buf
+ircd::m::media::file::download(const mxc &mxc,
+                               const m::user::id &user_id,
+                               const string_view &remote)
+{
+	const m::room::id::buf room_id
+	{
+		file::room_id(mxc)
+	};
+
+	if(remote && my_host(remote))
+		return room_id;
+
+	if(!remote && my_host(mxc.server))
+		return room_id;
+
+	download(mxc, user_id, room_id, remote);
+	return room_id;
+}
+
+ircd::m::room
+ircd::m::media::file::download(const mxc &mxc,
+                               const m::user::id &user_id,
+                               const m::room::id &room_id,
+                               string_view remote)
+try
+{
+	auto iit
+	{
+		downloading.emplace(room_id)
+	};
+
+	if(!iit.second)
+	{
+		downloading_dock.wait([&room_id]
+		{
+			return !downloading.count(room_id);
+		});
+
+		return room_id;
+	}
+
+	const unwind uw{[&iit]
+	{
+		downloading.erase(iit.first);
+		downloading_dock.notify_all();
+	}};
+
+	if(exists(room_id))
+		return room_id;
+
+	if(!remote)
+		remote = mxc.server;
+
+	const unique_buffer<mutable_buffer> buf
+	{
+		16_KiB
+	};
+
+	const auto pair
+	{
+		download(buf, mxc, remote)
+	};
+
+	const auto &head
+	{
+		pair.first
+	};
+
+	const const_buffer &content
+	{
+		pair.second
+	};
+
+	char mime_type_buf[64];
+	const auto &content_type
+	{
+		magic::mime(mime_type_buf, content)
+	};
+
+	if(content_type != head.content_type)
+		log::dwarning
+		{
+			log, "Server %s claims thumbnail %s is '%s' but we think it is '%s'",
+			remote,
+			mxc.mediaid,
+			head.content_type,
+			content_type,
+		};
+
+	m::vm::copts vmopts;
+	const m::room room
+	{
+		room_id, &vmopts
+	};
+
+	create(room, user_id, "file");
+	const unwind_exceptional purge{[&room]
+	{
+		m::room::purge(room);
+	}};
+
+	const size_t written
+	{
+		file::write(room, user_id, content, content_type)
+	};
+
+	return room;
+}
+catch(const ircd::server::unavailable &e)
+{
+	throw m::error
+	{
+		http::BAD_GATEWAY, "M_MEDIA_UNAVAILABLE",
+		"Server '%s' is not available for media for '%s/%s' :%s",
+		remote,
+		mxc.server,
+		mxc.mediaid,
+		e.what()
+	};
+}
+
+decltype(ircd::m::media::download_timeout)
+ircd::m::media::download_timeout
+{
+	{ "name",     "ircd.media.download.timeout" },
+	{ "default",  30L                           },
+};
+
+std::pair
+<
+	ircd::http::response::head,
+	ircd::unique_buffer<ircd::mutable_buffer>
+>
+ircd::m::media::file::download(const mutable_buffer &buf_,
+                               const mxc &mxc,
+                               string_view remote,
+                               server::request::opts *const opts)
+{
+	assert(remote || !my_host(mxc.server));
+	assert(!remote || !my_host(remote));
+
+	mutable_buffer buf{buf_};
+	fed::request::opts fedopts;
+	fedopts.remote = remote?: mxc.server;
+	json::get<"method"_>(fedopts.request) = "GET";
+	json::get<"uri"_>(fedopts.request) = fmt::sprintf
+	{
+		buf, "/_matrix/media/r0/download/%s/%s",
+		mxc.server,
+		mxc.mediaid,
+	};
+	consume(buf, size(json::get<"uri"_>(fedopts.request)));
+
+	//TODO: --- This should use the progress callback to build blocks
+	fed::request remote_request
+	{
+		buf, std::move(fedopts)
+	};
+
+	if(!remote_request.wait(seconds(download_timeout), std::nothrow))
+		throw m::error
+		{
+			http::GATEWAY_TIMEOUT, "M_MEDIA_DOWNLOAD_TIMEOUT",
+			"Server '%s' did not respond with media for '%s/%s' in time",
+			remote,
+			mxc.server,
+			mxc.mediaid
+		};
+
+	const auto &code
+	{
+		remote_request.get()
+	};
+
+	if(code != http::OK)
+		return {};
+
+	parse::buffer pb{remote_request.in.head};
+	parse::capstan pc{pb};
+	pc.read += size(remote_request.in.head);
+	return std::pair<http::response::head, unique_buffer<mutable_buffer>>
+	{
+		pc, std::move(remote_request.in.dynamic)
+	};
+}
+
+size_t
+ircd::m::media::file::write(const m::room &room,
+                            const m::user::id &user_id,
+                            const const_buffer &content,
+                            const string_view &content_type)
+{
+	static const size_t BLK_SZ
+	{
+		32_KiB
+	};
+
+	static const size_t BLK_ENCODE_BUF_SZ
+	{
+		48_KiB
+	};
+
+	static const size_t BLK_ENCODE_BUF_ALIGN
+	{
+		64
+	};
+
+	static_assert
+	(
+		BLK_ENCODE_BUF_SZ >= b64::encode_unpadded_size(BLK_SZ)
+	);
+
+	const unique_mutable_buffer blk_encode_buf
+	{
+		BLK_ENCODE_BUF_SZ,
+		BLK_ENCODE_BUF_ALIGN,
+	};
+
+	send(room, user_id, "ircd.file.stat", "size", json::members
+	{
+		{ "value", long(size(content)) }
+	});
+
+	send(room, user_id, "ircd.file.stat", "type", json::members
+	{
+		{ "value", content_type }
+	});
+
+	size_t off{0}, wrote{0};
+	while(off < size(content))
+	{
+		const size_t blk_sz
+		{
+			std::min(size(content) - off, BLK_SZ)
+		};
+
+		const const_buffer blk_raw
+		{
+			content + off, blk_sz
+		};
+
+		const string_view blk
+		{
+			b64::encode_unpadded(blk_encode_buf, blk_raw)
+		};
+
+		const auto event_id
+		{
+			send(room, user_id, "ircd.file.block", json::members
+			{
+				{ "data.ub64", blk },
+			})
+		};
+
+		off += size(blk_raw);
+		wrote += size(blk);
+		assert(size(blk) == b64::encode_unpadded_size(blk_raw));
+	}
+
+	//assert(wrote == b64::encode_unpadded_size(off));
+	assert(off == size(content));
+	return off;
+}
+
+size_t
+ircd::m::media::file::read(const m::room &room,
+                           const closure &closure)
+{
+	static const size_t BLK_DECODE_BUF_SZ
+	{
+		64_KiB
+	};
+
+	static const size_t BLK_DECODE_BUF_ALIGN
+	{
+		64
+	};
+
+	const unique_mutable_buffer blk_decode_buf
+	{
+		BLK_DECODE_BUF_SZ,
+		BLK_DECODE_BUF_ALIGN,
+	};
+
+	static const event::fetch::opts fopts
+	{
+		event::keys::include
+		{
+			"content", "type"
+		}
+	};
+
+	room::events it
+	{
+		room, 1, &fopts
+	};
+
+	if(!it)
+		return 0;
+
+	room::events epf
+	{
+		room, 1, &fopts
+	};
+
+	size_t
+	decoded_bytes(0),
+	encoding_bytes(0),
+	events_fetched(0),
+	events_prefetched(0);
+	for(; it; ++it)
+	{
+		for(; epf && events_prefetched < events_fetched + events_prefetch; ++epf)
+			events_prefetched += epf.prefetch();
+
+		++events_fetched;
+		const m::event &event
+		{
+			*it
+		};
+
+		if(json::get<"type"_>(event) != "ircd.file.block")
+			continue;
+
+		const json::object content
+		{
+			json::get<"content"_>(event)
+		};
+
+		const json::string &blk_encoded
+		{
+			content["data.ub64"] // unpadded base64
+		};
+
+		const const_buffer blk
+		{
+			b64::decode(blk_decode_buf, blk_encoded)
+		};
+
+		#if 0
+		log::debug
+		{
+			log, "File %s read event_idx:%lu events[fetched:%zu prefetched:%zu] encoded:%zu decoded:%zu total_encoded:%zu total_decoded:%zu",
+			string_view{room.room_id},
+			it.event_idx(),
+			events_fetched,
+			events_prefetched,
+			size(blk_encoded),
+			size(blk),
+			encoding_bytes,
+			decoded_bytes,
+		};
+		#endif
+
+		closure(blk);
+		decoded_bytes += size(blk);
+		encoding_bytes += size(blk_encoded);
+		assert(size(blk) == b64::decode_size(blk_encoded));
+	}
+
+	//assert(decoded_bytes == b64::decode_size(encoding_bytes));
+	return decoded_bytes;
+}
+
+//
+// media::file
+//
+
+ircd::m::room::id::buf
+ircd::m::media::file::room_id(const mxc &mxc)
+{
+	m::room::id::buf ret;
+	room_id(ret, mxc);
+	return ret;
+}
+
+ircd::m::room::id
+ircd::m::media::file::room_id(room::id::buf &out,
+                              const mxc &mxc)
+{
+	thread_local char buf[512];
+	const auto path
+	{
+		mxc.path(buf)
+	};
+
+	const sha256::buf hash
+	{
+		sha256{path}
+	};
+
+	out =
+	{
+		b58::encode(buf, hash), my_host()
+	};
+
+	return out;
+}
+
+//
+// media::mxc
+//
+
+ircd::m::media::mxc::mxc(const string_view &server,
+                         const string_view &mediaid)
+:server
+{
+	split(lstrip(server, "mxc://"), '/').first
+}
+,mediaid
+{
+	mediaid?: rsplit(server, '/').second
+}
+{
+	if(unlikely(empty(server)))
+		throw m::BAD_REQUEST
+		{
+			"Invalid MXC: missing server parameter."
+		};
+
+	if(unlikely(empty(mediaid)))
+		throw m::BAD_REQUEST
+		{
+			"Invalid MXC: missing mediaid parameter."
+		};
+}
+
+ircd::m::media::mxc::mxc(const string_view &uri)
+:server
+{
+	split(lstrip(uri, "mxc://"), '/').first
+}
+,mediaid
+{
+	rsplit(uri, '/').second
+}
+{
+	if(unlikely(empty(server)))
+		throw m::BAD_REQUEST
+		{
+			"Invalid MXC: missing server parameter."
+		};
+
+	if(unlikely(empty(mediaid)))
+		throw m::BAD_REQUEST
+		{
+			"Invalid MXC: missing mediaid parameter."
+		};
+}
+
+ircd::string_view
+ircd::m::media::mxc::uri(const mutable_buffer &out)
+const
+{
+	return fmt::sprintf
+	{
+		out, "mxc://%s/%s",
+		server,
+		mediaid
+	};
+}
+
+ircd::string_view
+ircd::m::media::mxc::path(const mutable_buffer &out)
+const
+{
+	return fmt::sprintf
+	{
+		out, "%s/%s",
+		server,
+		mediaid
+	};
+}

matrix/vm_execute.cc

@@ -905,12 +905,18 @@ ircd::m::vm::write_append(eval &eval,
 	wopts.interpose = eval.txn.get();
 	wopts.event_idx = eval.sequence;
 	wopts.json_source = opts.json_source;
-	wopts.appendix.set(dbs::appendix::ROOM_STATE_SPACE, opts.history);
 
 	// Don't update or resolve the room head with this shit.
-	const bool dummy_event(json::get<"type"_>(event) == "org.matrix.dummy_event");
-	wopts.appendix.set(dbs::appendix::ROOM_HEAD, opts.room_head && !dummy_event);
-	wopts.appendix.set(dbs::appendix::ROOM_HEAD_RESOLVE, opts.room_head_resolve);
+	const bool dummy_event
+	{
+		json::get<"type"_>(event) == "org.matrix.dummy_event"
+	};
+
+	wopts.appendix.set
+	(
+		dbs::appendix::ROOM_HEAD,
+		wopts.appendix[dbs::appendix::ROOM_HEAD] && !dummy_event
+	);
 
 	if(opts.present && json::get<"state_key"_>(event))
 	{
@@ -943,7 +949,7 @@ ircd::m::vm::write_append(eval &eval,
 			//XXX
 			const auto &[pass, fail]
 			{
-				opts.auth && !eval.room_internal?
+				opts.phase[phase::AUTH_PRES] && !eval.room_internal?
 					room::auth::check_present(event):
 					room::auth::passfail{true, {}}
 			};
@@ -957,8 +963,17 @@ ircd::m::vm::write_append(eval &eval,
 					what(fail),
 				};
 
-			wopts.appendix.set(dbs::appendix::ROOM_STATE, pass);
-			wopts.appendix.set(dbs::appendix::ROOM_JOINED, pass);
+			wopts.appendix.set
+			(
+				dbs::appendix::ROOM_STATE,
+				pass && wopts.appendix[dbs::appendix::ROOM_STATE]
+			);
+
+			wopts.appendix.set
+			(
+				dbs::appendix::ROOM_JOINED,
+				pass && wopts.appendix[dbs::appendix::ROOM_JOINED]
+			);
 		}
 	}
 

modules/console.cc

@@ -14356,8 +14356,8 @@ console_cmd__fed__backfill(opt &out, const string_view &line)
 
 	m::vm::opts vmopts;
 	vmopts.nothrows = -1;
-	vmopts.room_head = false;
-	vmopts.room_head_resolve = true;
+	vmopts.wopts.appendix[m::dbs::appendix::ROOM_HEAD_RESOLVE] = false;
+	vmopts.wopts.appendix[m::dbs::appendix::ROOM_HEAD] = false;
 	vmopts.phase.set(m::vm::phase::FETCH_PREV, false);
 	vmopts.phase.set(m::vm::phase::FETCH_STATE, false);
 	vmopts.node_id = remote;
@@ -14726,8 +14726,8 @@ console_cmd__fed__auth(opt &out, const string_view &line)
 		m::vm::opts vmopts;
 		vmopts.node_id = opts.remote;
 		vmopts.nothrows = -1;
-		vmopts.room_head = false;
-		vmopts.room_head_resolve = true;
+		vmopts.wopts.appendix[m::dbs::appendix::ROOM_HEAD_RESOLVE] = false;
+		vmopts.wopts.appendix[m::dbs::appendix::ROOM_HEAD] = false;
 		vmopts.phase.set(m::vm::phase::FETCH_PREV, false);
 		vmopts.phase.set(m::vm::phase::FETCH_STATE, false);
 		vmopts.notify_servers = false;

modules/media/media.cc

@@ -289,7 +289,6 @@ try
 	};
 
 	m::vm::copts vmopts;
-	vmopts.history = false;
 	const m::room room
 	{
 		room_id, &vmopts

modules/media/upload.cc

@@ -66,7 +66,6 @@ post__upload(client &client,
 	};
 
 	m::vm::copts vmopts;
-	vmopts.history = false;
 	const m::room room
 	{
 		room_id, &vmopts