Compare commits

...

10 Commits

9 changed files with 464 additions and 130 deletions

View File

@ -1,9 +1,8 @@
#
# Build docker images
#
name: Docker Images
env:
ctor_id: ${{ vars.DOCKER_ID }}
ctor_url: https://github.com/${{github.repository}}
on:
push:
branches: [master]
@ -15,7 +14,12 @@ concurrency:
group: ${{ github.ref }}
cancel-in-progress: true
env:
ctor_id: ${{ vars.DOCKER_ID }}
ctor_url: https://github.com/${{github.repository}}
jobs:
# Build the base-feature intermediate images (cached and not shipped).
base:
uses: ./.github/workflows/docker_prime.yml
with:
@ -26,6 +30,7 @@ jobs:
machines: ${{vars.DOCKER_MACHINES}}
test: ${{contains(github.events.push.commits[0].message, '[ci test]')}}
# Build the full-feature intermediate images (cached and not shipped).
full:
uses: ./.github/workflows/docker_prime.yml
needs: [base]
@ -37,6 +42,7 @@ jobs:
machines: ${{vars.DOCKER_MACHINES}}
test: ${{contains(github.events.push.commits[0].message, '[ci test]')}}
# Build the leaf images (shipped and not cached)
built:
needs: [base, full]
runs-on: ${{matrix.machine}}

View File

@ -1,3 +1,9 @@
#
# Build intermediate images
#
# Called to build lower-layer images which other images depend on. These are
# cached for use by the next layer but not shipped to users.
#
name: Docker Images Prime
on:
@ -5,32 +11,38 @@ on:
inputs:
id:
type: string
description: Dockerhub acct/repo identity.
url:
type: string
description: Git repository for checkout.
features:
type: string
description: JSON array of feature-set names to build images for.
distros:
type: string
description: JSON array of operating system distros to build for.
machines:
type: string
description: JSON array of machines to build for.
test:
type: boolean
default: false
required: false
description: Echo all docker commands rather than invoking them.
concurrency:
group: ${{github.workflow}}-${{inputs.features}}
cancel-in-progress: false
env:
ctor_id: ${{inputs.id}}
ctor_url: ${{inputs.url}}
concurrency:
group: ${{github.run_id}}
cancel-in-progress: true
jobs:
prime:
runs-on: ${{matrix.machine}}
strategy:
fail-fast: false
fail-fast: true
matrix:
feature: ${{fromJSON(inputs.features)}}
distro: ${{fromJSON(inputs.distros)}}

View File

@ -131,7 +131,7 @@ build()
# Leaf build; unique to each iteration.
tag="$ctor_acct/$ctor_repo:${distro}-${feature}-built-${toolchain}-${machine}"
arg="$args -t $tag $BASEDIR/${dist_name}/built"
arg="$args --no-cache -t $tag $BASEDIR/${dist_name}/built"
eval "$cmd build $arg"
if test $? -ne 0; then return 1; fi
if test $stage = "built"; then return 0; fi

View File

@ -192,8 +192,8 @@ ircd::cl::envs
{ "default", "true" },
},
{
{ "name", "MESA_GLSL_CACHE_DISABLE" },
{ "default", "true" },
{ "name", "MESA_SHADER_CACHE_DISABLE" },
{ "default", "true" },
},
{
{ "name", "AMD_DEBUG" },

View File

@ -2358,15 +2358,20 @@ ircd::db::seek(domain::const_iterator_base &it,
{
switch(p)
{
// This is inefficient as per RocksDB's prefix impl.
case pos::BACK:
{
// This is inefficient as per RocksDB's prefix impl. unknown why
// a seek to NEXT is still needed after walking back one.
char buf[512];
string_view key;
assert(bool(it)); do
{
assert(size(it.it->key()) <= sizeof(buf));
key = string_view(buf, copy(buf, slice(it.it->key())));
}
while(seek(it, pos::NEXT));
if(seek(it, pos::PREV))
seek(it, pos::NEXT);
return bool(it);
assert(key);
return seek(it, key);
}
default:
@ -5003,24 +5008,23 @@ ircd::db::_seek(const vector_view<_read_op> &op,
namespace ircd::db
{
static rocksdb::Iterator &_seek_(rocksdb::Iterator &, const pos &);
static rocksdb::Iterator &_seek_(rocksdb::Iterator &, const string_view &);
static rocksdb::Iterator &_seek_lower_(rocksdb::Iterator &, const string_view &);
static rocksdb::Iterator &_seek_upper_(rocksdb::Iterator &, const string_view &);
static bool _seek(database::column &, const pos &, const rocksdb::ReadOptions &, rocksdb::Iterator &it);
static bool _seek(database::column &, const string_view &, const rocksdb::ReadOptions &, rocksdb::Iterator &it);
static rocksdb::Iterator &_seek_(rocksdb::Iterator &, const string_view &, const bool lte);
static bool _seek(database::column &, const pos &, const rocksdb::ReadOptions &, rocksdb::Iterator &it, const bool lte);
static bool _seek(database::column &, const string_view &, const rocksdb::ReadOptions &, rocksdb::Iterator &it, const bool lte);
}
std::unique_ptr<rocksdb::Iterator>
ircd::db::seek(column &column,
const string_view &key,
const gopts &opts)
const gopts &opts,
const bool lte)
{
database &d(column);
database::column &c(column);
std::unique_ptr<rocksdb::Iterator> ret;
const auto ropts(make_opts(opts));
seek(c, key, ropts, ret);
seek(c, key, ropts, ret, lte);
return ret;
}
@ -5029,7 +5033,8 @@ bool
ircd::db::seek(database::column &c,
const pos &p,
const rocksdb::ReadOptions &opts,
std::unique_ptr<rocksdb::Iterator> &it)
std::unique_ptr<rocksdb::Iterator> &it,
const bool lte)
{
const ctx::uninterruptible ui;
const ctx::stack_usage_assertion sua;
@ -5041,31 +5046,33 @@ ircd::db::seek(database::column &c,
it.reset(d.d->NewIterator(opts, cf));
}
return _seek(c, p, opts, *it);
return _seek(c, p, opts, *it, lte);
}
bool
ircd::db::_seek(database::column &c,
const string_view &p,
const rocksdb::ReadOptions &opts,
rocksdb::Iterator &it)
rocksdb::Iterator &it,
const bool lte)
try
{
util::timer timer{util::timer::nostart};
if constexpr(RB_DEBUG_DB_SEEK)
timer = util::timer{};
_seek_(it, p);
_seek_(it, p, lte);
database &d(*c.d);
if constexpr(RB_DEBUG_DB_SEEK)
log::debug
{
log, "[%s] %lu:%lu SEEK %s %s in %ld$us '%s'",
log, "[%s] %lu:%lu SEEK[%s] %s %s in %ld$us '%s'",
name(d),
sequence(d),
sequence(opts.snapshot),
valid(it)? "VALID" : "INVALID",
lte? "LTE"_sv: "GTE"_sv,
valid(it)? "VALID"_sv: "INVALID"_sv,
it.status().ok()? "OK"s: it.status().ToString(),
timer.at<microseconds>().count(),
name(c)
@ -5078,11 +5085,12 @@ catch(const error &e)
const database &d(*c.d);
log::critical
{
log, "[%s][%s] %lu:%lu SEEK key :%s",
log, "[%s][%s] %lu:%lu SEEK[%s] key :%s",
name(d),
name(c),
sequence(d),
sequence(opts.snapshot),
lte? "LTE"_sv: "GTE"_sv,
e.what(),
};
@ -5093,7 +5101,8 @@ bool
ircd::db::_seek(database::column &c,
const pos &p,
const rocksdb::ReadOptions &opts,
rocksdb::Iterator &it)
rocksdb::Iterator &it,
const bool)
try
{
bool valid_it;
@ -5115,7 +5124,7 @@ try
sequence(d),
sequence(opts.snapshot),
reflect(p),
valid_it? "VALID" : "INVALID",
valid_it? "VALID"_sv: "INVALID"_sv,
it.status().ok()? "OK"s: it.status().ToString(),
timer.at<microseconds>().count(),
name(c)
@ -5134,41 +5143,27 @@ catch(const error &e)
sequence(d),
sequence(opts.snapshot),
reflect(p),
it.Valid()? "VALID" : "INVALID",
it.Valid()? "VALID"_sv: "INVALID"_sv,
e.what(),
};
throw;
}
/// Seek to entry NOT GREATER THAN key. That is, equal to or less than key
rocksdb::Iterator &
ircd::db::_seek_lower_(rocksdb::Iterator &it,
const string_view &sv)
{
assert(!ctx::interruptible());
it.SeekForPrev(slice(sv));
return it;
}
/// Seek to entry NOT LESS THAN key. That is, equal to or greater than key
rocksdb::Iterator &
ircd::db::_seek_upper_(rocksdb::Iterator &it,
const string_view &sv)
{
assert(!ctx::interruptible());
it.Seek(slice(sv));
return it;
}
/// Defaults to _seek_upper_ because it has better support from RocksDB.
rocksdb::Iterator &
ircd::db::_seek_(rocksdb::Iterator &it,
const string_view &sv)
const string_view &sv,
const bool lte)
{
return _seek_upper_(it, sv);
assert(!ctx::interruptible());
if(lte)
it.SeekForPrev(slice(sv));
else
it.Seek(slice(sv));
return it;
}
rocksdb::Iterator &
@ -5179,11 +5174,24 @@ ircd::db::_seek_(rocksdb::Iterator &it,
switch(p)
{
case pos::NEXT: it.Next(); break;
case pos::PREV: it.Prev(); break;
case pos::FRONT: it.SeekToFirst(); break;
case pos::BACK: it.SeekToLast(); break;
default:
case pos::NEXT:
assert(valid(it));
it.Next();
break;
case pos::PREV:
assert(valid(it));
it.Prev();
break;
case pos::FRONT:
it.SeekToFirst();
break;
case pos::BACK:
it.SeekToLast();
break;
case pos::END:
{
it.SeekToLast();
@ -5192,6 +5200,10 @@ ircd::db::_seek_(rocksdb::Iterator &it,
break;
}
default:
assert(false);
break;
}
return it;

View File

@ -134,10 +134,8 @@ namespace ircd::db
static void valid_eq_or_throw(const rocksdb::Iterator &, const string_view &);
// [GET] iterator seek suite
template<class pos> static bool seek(database::column &, const pos &, const rocksdb::ReadOptions &, std::unique_ptr<rocksdb::Iterator> &it);
static std::unique_ptr<rocksdb::Iterator> seek(column &, const gopts &);
static std::unique_ptr<rocksdb::Iterator> seek(column &, const string_view &key, const gopts &);
static std::vector<row::value_type> seek(database &, const gopts &);
template<class pos> static bool seek(database::column &, const pos &, const rocksdb::ReadOptions &, std::unique_ptr<rocksdb::Iterator> &it, const bool lte = false);
static std::unique_ptr<rocksdb::Iterator> seek(column &, const string_view &key, const gopts &, const bool lte = false);
static std::pair<string_view, string_view> operator*(const rocksdb::Iterator &);
// [GET] read suite

View File

@ -13312,9 +13312,9 @@ console_cmd__user__read(opt &out, const string_view &line)
"user_id", "room_id", "limit"
}};
const m::user::id user_id
const auto user_id
{
param.at("user_id")
param.at("user_id", "*"_sv)
};
const auto room_id
@ -13324,6 +13324,11 @@ console_cmd__user__read(opt &out, const string_view &line)
m::room::id::buf{}
};
const bool all_users
{
param["user_id"] == "*"
};
const bool all_rooms
{
param["room_id"] == "*"
@ -13340,14 +13345,9 @@ console_cmd__user__read(opt &out, const string_view &line)
param["room_id"] == "***"
};
size_t limit
ssize_t limit
{
param.at("limit", 32UL)
};
const m::user::room user_room
{
user_id
param.at("limit", 32L)
};
const m::event::closure each_event{[&out]
@ -13411,19 +13411,35 @@ console_cmd__user__read(opt &out, const string_view &line)
out << std::endl;
}};
if(all_rooms)
if(all_rooms && !all_users)
{
const m::user::room user_room
{
user_id
};
const m::room::state state
{
user_room
};
state.for_each("ircd.read", each_event);
state.for_each("ircd.read", m::event::closure_bool{[&each_event, &limit]
(const m::event &event)
{
each_event(event);
return --limit > 0;
}});
return true;
}
if(eye_track)
if(eye_track && !all_users)
{
const m::user::room user_room
{
user_id
};
const m::room::type type
{
user_room, "ircd.read"
@ -13440,14 +13456,19 @@ console_cmd__user__read(opt &out, const string_view &line)
if(likely(event.valid))
each_event(event);
return --limit;
return --limit > 0;
});
return true;
}
if(fully_read)
if(fully_read && !all_users)
{
const m::user::room user_room
{
user_id
};
const m::room::type type
{
user_room, "ircd.account_data!", { -1UL, -1UL }, true
@ -13468,29 +13489,55 @@ console_cmd__user__read(opt &out, const string_view &line)
return true;
each_event(event);
return --limit;
return --limit > 0;
});
return true;
}
const m::room::state::space space
if(!all_users)
{
user_room
};
space.for_each("ircd.read", room_id, [&each_event, &limit]
(const auto &type, const auto &state_key, const auto &depth, const auto &event_idx) -> bool
{
const m::event::fetch event
const m::user::room user_room
{
std::nothrow, event_idx
user_id
};
if(likely(event.valid))
each_event(event);
const m::room::state::space space
{
user_room
};
return --limit;
space.for_each("ircd.read", room_id, [&each_event, &limit]
(const auto &type, const auto &state_key, const auto &depth, const auto &event_idx) -> bool
{
const m::event::fetch event
{
std::nothrow, event_idx
};
if(likely(event.valid))
each_event(event);
return --limit > 0;
});
}
const m::events::range range
{
-1UL, 0UL
};
m::events::for_each(range, [&each_event, &limit]
(const m::event::idx &seq, const m::event &event)
{
if(json::get<"type"_>(event) != "ircd.read")
return true;
if(!my(event))
return true;
each_event(event);
return --limit > 0;
});
return true;
@ -15058,7 +15105,7 @@ console_cmd__feds__head(opt &out, const string_view &line)
if(prev_event.valid)
out << pretty_oneline(prev_event);
else
out << string_view{prev_event_id};
out << result.request->room_id << ' ' << prev_event_id;
out << std::endl;
}

View File

@ -40,7 +40,21 @@ struct command_result
string_view msgtype {"m.notice"};
};
conf::item<bool>
static conf::item<size_t>
watch_limit
{
{ "name", "ircd.m.command.watch.limit" },
{ "default", 256 },
};
static conf::item<bool>
watch_opers
{
{ "name", "ircd.m.command.watch.opers" },
{ "default", true },
};
static conf::item<bool>
command_typing
{
{ "name", "ircd.m.command.typing" },
@ -55,6 +69,30 @@ execute_command(const mutable_buffer &buf,
const m::event::id &reply_to,
const bool public_response);
static void
watch_command(const m::user::id &user_id,
const m::room::id &room_id,
const m::event::id &reply_id,
const m::event::id &response_id,
const m::room::id &response_room,
const m::user::id &response_sender,
const string_view &response_type,
const string_view &cmd,
const bool &public_response,
const milliseconds &watch_delay);
static const json::value
undef_val
{
string_view{nullptr}, json::STRING
};
static const json::value
html_format
{
"org.matrix.custom.html", json::STRING
};
void
handle_command(const m::event &event,
m::vm::eval &eval)
@ -105,6 +143,18 @@ try
if(!is_command)
return;
const json::string reply_to
{
content["reply_id"]
};
const auto reply_id
{
reply_to?
m::event::id::buf{reply_to}:
m::event::id::buf{}
};
// View of the command string without prefix
string_view input
{
@ -120,29 +170,43 @@ try
startswith(input, '!')
};
if(public_response)
input = lstrip(input, '!');
const bool command_watch
{
startswith(input, "watch ")
&& (!watch_opers || is_oper(user))
};
milliseconds watch_delay {0ms};
if(command_watch)
{
const auto delay
{
lex_cast<float>(token(input, ' ', 1))
};
watch_delay = milliseconds
{
long(delay * 1000.0)
};
input = tokens_after(input, ' ', 1);
}
const string_view cmd
{
lstrip(input, '!')
};
const json::string reply_to
{
content["reply_id"]
};
const auto reply_id
{
reply_to?
m::event::id{reply_to}:
m::event::id{}
input
};
log::debug
{
m::log, "Server command from %s in %s public:%b replying:%s :%s",
m::log, "Server command from %s in %s public:%b watch:%ld replying:%s :%s",
string_view{room_id},
string_view{user.user_id},
public_response,
watch_delay.count(),
string_view{reply_id}?: "false"_sv,
cmd
};
@ -175,7 +239,7 @@ try
public_response? "m.room.message" : "ircd.cmd.result"
};
const auto &response_event_id
const auto &command_event_id
{
public_response?
string_view{event_id}:
@ -192,16 +256,6 @@ try
alt?: "no alt text"_sv, json::STRING
};
static const json::value undef_val
{
string_view{nullptr}, json::STRING
};
static const json::value html_format
{
"org.matrix.custom.html", json::STRING
};
char relates_buf[576], reply_buf[288];
json::object in_reply_to, relates_to {json::empty_object};
if(event_id)
@ -219,16 +273,52 @@ try
});
}
m::send(response_room, response_sender, response_type,
const auto response_id
{
{ "msgtype", msgtype?: "m.notice" },
{ "format", html? html_format: undef_val },
{ "body", content_body },
{ "formatted_body", html? html_val: undef_val },
{ "room_id", room_id },
{ "input", input },
{ "m.relates_to", relates_to },
});
m::send(response_room, response_sender, response_type,
{
{ "msgtype", msgtype?: "m.notice" },
{ "format", html? html_format: undef_val },
{ "body", content_body },
{ "formatted_body", html? html_val: undef_val },
{ "room_id", room_id },
{ "input", cmd },
{ "m.relates_to", relates_to },
})
};
if(command_watch)
ctx::context
{
"watch", ctx::context::POST | ctx::context::DETACH,
[
user_id(m::user::id::buf(user.user_id)),
room_id(m::room::id::buf(room_id)),
reply_id(m::event::id::buf(reply_id)),
response_id(m::event::id::buf(response_id)),
response_room(m::room::id::buf(response_room)),
response_sender(m::user::id::buf(response_sender)),
response_type(std::string(response_type)),
cmd(std::string(cmd)),
public_response,
watch_delay
]
{
watch_command
(
user_id,
room_id,
reply_id,
response_id,
response_room,
response_sender,
response_type,
cmd,
public_response,
watch_delay
);
}
};
}
catch(const std::exception &e)
{
@ -258,6 +348,90 @@ catch(const std::exception &e)
};
}
void
watch_command(const m::user::id &user_id,
const m::room::id &room_id,
const m::event::id &reply_id,
const m::event::id &response_id,
const m::room::id &response_room,
const m::user::id &response_sender,
const string_view &response_type,
const string_view &cmd,
const bool &public_response,
const milliseconds &watch_delay)
{
const auto annotation_id
{
public_response?
m::annotate(response_room, response_sender, response_id, "▶️"_sv):
m::event::id::buf{}
};
const unwind deannotate{[&]
{
if(annotation_id && !m::redacted(annotation_id))
m::redact(response_room, response_sender, annotation_id, "cleared");
}};
const unique_buffer<mutable_buffer> buf
{
56_KiB
};
for(size_t i(0); i < size_t(watch_limit); ++i)
{
sleep(watch_delay);
if(m::redacted(response_id))
break;
if(annotation_id && m::redacted(annotation_id))
break;
const auto &[html, alt, msgtype]
{
execute_command(buf, user_id, room_id, cmd, reply_id, public_response)
};
if(!html && !alt)
break;
const json::value html_val
{
html, json::STRING
};
const json::value content_body
{
alt?: "no alt text"_sv, json::STRING
};
m::send(response_room, response_sender, response_type,
{
{ "body", content_body },
{ "format", html? html_format: undef_val },
{ "formatted_body", html? html_val: undef_val },
{ "input", cmd },
{ "msgtype", msgtype?: "m.notice" },
{ "room_id", room_id },
{ "m.new_content", json::members
{
{ "body", content_body },
{ "format", html? html_format: undef_val },
{ "formatted_body", html? html_val: undef_val },
{ "input", cmd },
{ "msgtype", msgtype?: "m.notice" },
{ "room_id", room_id },
}},
{ "m.relates_to", json::members
{
{ "event_id", response_id },
{ "rel_type", "m.replace" },
}}
});
}
}
static command_result
command__summarize(const mutable_buffer &buf,
const m::user &user,

View File

@ -848,6 +848,91 @@ github_markdown(unique_const_buffer &buf,
);
}
static bool
github_hook_for_each(const string_view &repo,
const function_bool<json::object> &closure)
{
unique_const_buffer buf;
const json::array response
{
github_request
(
buf, "GET", repo, "hooks"
)
};
for(const json::object hook : response)
if(!closure(hook))
return false;
return true;
}
static void
github_hook_ping(const string_view &repo,
const string_view &hook)
{
unique_const_buffer buf;
github_request
(
buf, "POST", repo, "hooks/%s/pings",
hook
);
}
static void
github_hook_ping(const string_view &repo)
{
github_hook_for_each(repo, [&repo]
(const json::object &hook)
{
const json::string id
{
hook["id"]
};
github_hook_ping(repo, id);
});
}
static bool
github_hook_shot_for_each(const string_view &repo,
const string_view &hook,
const bool &redelivery,
const function_bool<json::object> &closure)
{
unique_const_buffer buf;
const json::array response
{
github_request
(
//TODO: pagination token
buf, "GET", repo, "hooks/%s/deliveries?per_page=100",
hook
)
};
for(const json::object shot : response)
if(!closure(shot))
return false;
return true;
}
static void
github_hook_shot_retry(const string_view &repo,
const string_view &hook,
const string_view &id)
{
unique_const_buffer buf;
github_request
(
buf, "POST", repo, "hooks/%s/deliveries/%s/attempts",
hook,
id
);
}
static bool
github_run_for_each_jobs(const string_view &repo,
const string_view &run_id,