MirrorHub/construct
0
0
Fork 0
mirror of https://github.com/matrix-construct/construct synced 2024-12-02 03:32:52 +01:00
Code Issues Releases Wiki Activity
construct/ircd/gpt_pipe.cc

1525 lines
29 KiB
C++
Raw Blame History

// Tensor Construct
//
// Copyright (C) Matrix Construct Developers, Authors & Contributors
// Copyright (C) 2016-2021 Jason Volk <jason@zemos.net>
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice is present in all copies. The
// full license for this software is available in the LICENSE file.
namespace ircd::gpt::pipe
{
static void profile_dumplog(pipe::exec &);
extern conf::item<size_t> queue_cycles;
extern const ircd::run::changed handle_quit;
static ircd::cl::exec::opts
send_opts_opts, send_ctrl_opts, send_coil_opts, send_head_opts,
anode_opts, negative_opts, positive_opts, cathode_opts,
lmhead_opts, lmamax_opts, backprop_opts, recv_ctrl_opts;
}
decltype(ircd::gpt::pipe::queue_cycles)
ircd::gpt::pipe::queue_cycles
{
{ "name", "ircd.gpt.pipe.queue" },
{ "default", 1L, },
};
decltype(ircd::gpt::pipe::default_model)
ircd::gpt::pipe::default_model;
decltype(ircd::gpt::pipe::default_code)
ircd::gpt::pipe::default_code;
decltype(ircd::gpt::pipe::default_desc)
ircd::gpt::pipe::default_desc;
decltype(ircd::gpt::pipe::handle_quit)
ircd::gpt::pipe::handle_quit
{
run::level::QUIT, pipe::fini
};
void
ircd::gpt::pipe::init()
{
const gpt::model::decoder &default_model
{
*gpt::model::default_model
};
assert(!pipe::default_model);
pipe::default_model = new pipe::model
{
default_model, default_model.word
};
pipe::default_code = new pipe::code
{
};
pipe::default_desc = new pipe::desc
{
*pipe::default_code, *pipe::default_model
};
//XXX
send_ctrl_opts.flush = true;
send_ctrl_opts.nice = 1;
lmamax_opts.flush = true;
lmamax_opts.nice = 2;
recv_ctrl_opts.flush = true;
log::debug
{
log, "Pipe initialized from model:%p data:%p code:%p desc:%p",
&default_model,
pipe::default_model,
pipe::default_code,
pipe::default_desc,
};
}
void
ircd::gpt::pipe::fini()
noexcept
{
const auto pending
{
cl::work::list.size()
};
if(pending)
log::warning
{
log, "Waiting for %zu pending tasks to leave the pipe...",
pending,
};
cl::sync();
delete default_desc; default_desc = nullptr;
delete default_code; default_code = nullptr;
delete default_model; default_model = nullptr;
}
//
// pipe
//
void
ircd::gpt::pipe::generate(task &task)
{
assert(pipe::default_model);
assert(task.opts);
const auto &opts
{
*task.opts
};
assert(task.ctrl);
auto &ctrl
{
*task.ctrl
};
ctrl.epic.cycle = 0;
ctrl.epic.host_tsc = prof::cycles();
const auto tokens(ctrl.tokens.count);
const auto epoch(ctrl.epic.epoch);
volatile auto cycle(ctrl.epic.cycle);
std::deque<pipe::exec> list;
for(; cycle < opts.limit; ++cycle)
{
// When the release/acquire bits are set the control pages are sent
// and received; only set on first and last iterations of this loop.
const bool
rel(cycle == 0),
acq(cycle + 1 >= opts.limit || ctx::interruption_requested());
// Enqueue the cycle's commands
list.emplace_back
(
task, tokens + cycle, rel, acq
);
if(ctx::interruption_requested())
if(acq || termination(ctx::cur()))
break;
// Enqueue consecutive repetitions of our kernel batch before waiting
// on the first; based on the configuration. XXX get from ircd::cl
if(list.size() <= pipe::queue_cycles)
continue;
// Profiling branch
if((false))
{
auto &ex(list.front());
profile_dumplog(ex);
}
// Destructing the front of the queue waits for completion by yielding
// this ircd::ctx.
list.pop_front();
}
// Wait for all unfinished
list.clear();
assert(ctrl.magic == 0xC7012C70);
assert(ctrl.epic.cycle == cycle || ctx::interruption_requested());
this_ctx::interruption_point();
}
void
ircd::gpt::pipe::profile_dumplog(pipe::exec &exec)
{
constexpr size_t coils
{
sizeof(exec.coil) / sizeof(cl::exec)
};
for(size_t i(0); i < coils; ++i)
{
exec.coil[i].wait();
const auto &pro
{
exec.coil[i].profile()
};
char tmbuf[4][32] {{0}};
log::logf
{
log, log::level::DEBUG,
"coil:%-2lu %8s %8s %8s %8s",
i,
util::pretty(tmbuf[0], si(pro[0]), 1),
util::pretty(tmbuf[1], si(pro[1]), 1),
util::pretty(tmbuf[2], si(pro[2]), 1),
util::pretty(tmbuf[3], si(pro[3]), 1),
};
}
}
//
// pipe::exec
//
ircd::gpt::pipe::exec::exec(task &task,
const size_t tokens,
const bool release,
const bool acquire)
:desc
{
default_desc
}
,send_opts
{
reinterpret_cast<const char *>(task.opts),
release?
sizeof(gpt::opts):
0
}
,send_ctrl
{
reinterpret_cast<const char *>(task.ctrl),
release?
sizeof(gpt::ctrl):
0
}
,send_coil
{
reinterpret_cast<const char *>(gpt::model::default_model),
release && desc->model->invalid?
(sizeof(gpt::model::block) * 12 + sizeof(gpt::model::norm)):
0
}
,send_head
{
reinterpret_cast<const char *>(&gpt::model::default_model->word),
release && desc->model->invalid?
sizeof(gpt::model::embed):
0
}
,recv_ctrl
{
reinterpret_cast<char *>(task.ctrl),
acquire?
sizeof(gpt::ctrl):
0
}
,range_full
{
{ tokens * 192UL, 0, },
{ 192UL, 0, },
}
,range_last
{
{ 1 * 192UL, 0 },
{ 192UL, 0 },
{ (tokens - 1) * 192UL, 0 },
}
,range_lm_embed
{
release?
range_full:
range_last
}
,range_negative
{
release?
range_full:
range_last
}
,range_positive
{
release?
range_full:
range_last
}
,range_lm_norm
{
range_last
}
,range_lm_logit
{
{ 786 * 64UL, 0 }, // align_up(50257) / 64
{ 64UL, 0 },
}
,range_lm_logsm
{
{ 1 * 256UL, 0 },
{ 256UL, 0 },
}
,range_lm_select
{
{ 1 * 256UL, 0 },
{ 256UL, 0 },
}
,release_opts
{
desc->opts, send_opts, send_opts_opts,
}
,release_ctrl
{
desc->ctrl, send_ctrl, send_ctrl_opts
}
,release_coil
{
desc->model->decode->master[0], send_coil, send_coil_opts
}
,release_head
{
desc->model->embed->master[0], send_head, send_head_opts
}
,lm_embed
{
desc->lm_embed, range_lm_embed, anode_opts
}
,coil
{
{ desc->layer[0x00]->negative, range_negative, negative_opts },
{ desc->layer[0x00]->positive, range_positive, positive_opts },
{ desc->layer[0x01]->negative, range_negative, negative_opts },
{ desc->layer[0x01]->positive, range_positive, positive_opts },
{ desc->layer[0x02]->negative, range_negative, negative_opts },
{ desc->layer[0x02]->positive, range_positive, positive_opts },
{ desc->layer[0x03]->negative, range_negative, negative_opts },
{ desc->layer[0x03]->positive, range_positive, positive_opts },
{ desc->layer[0x04]->negative, range_negative, negative_opts },
{ desc->layer[0x04]->positive, range_positive, positive_opts },
{ desc->layer[0x05]->negative, range_negative, negative_opts },
{ desc->layer[0x05]->positive, range_positive, positive_opts },
{ desc->layer[0x06]->negative, range_negative, negative_opts },
{ desc->layer[0x06]->positive, range_positive, positive_opts },
{ desc->layer[0x07]->negative, range_negative, negative_opts },
{ desc->layer[0x07]->positive, range_positive, positive_opts },
{ desc->layer[0x08]->negative, range_negative, negative_opts },
{ desc->layer[0x08]->positive, range_positive, positive_opts },
{ desc->layer[0x09]->negative, range_negative, negative_opts },
{ desc->layer[0x09]->positive, range_positive, positive_opts },
{ desc->layer[0x0a]->negative, range_negative, negative_opts },
{ desc->layer[0x0a]->positive, range_positive, positive_opts },
{ desc->layer[0x0b]->negative, range_negative, negative_opts },
{ desc->layer[0x0b]->positive, range_positive, positive_opts },
}
,lm_norm
{
desc->lm_norm, range_lm_norm, cathode_opts
}
,lm_logit
{
desc->lm_logit, range_lm_logit, lmhead_opts
}
,lm_logsm
{
desc->lm_logsm, range_lm_logsm, lmhead_opts
}
,lm_select
{
desc->lm_select, range_lm_select, lmamax_opts
}
,acquire_ctrl
{
desc->ctrl, recv_ctrl, recv_ctrl_opts
}
{
if(release && desc->model->invalid)
desc->model->invalid = false;
}
ircd::gpt::pipe::exec::~exec()
noexcept
{
}
//
// code
//
decltype(ircd::gpt::pipe::code::default_path)
ircd::gpt::pipe::code::default_path
{
{ "name", "ircd.gpt.pipe.code.path" },
};
decltype(ircd::gpt::pipe::code::default_opts)
ircd::gpt::pipe::code::default_opts
{
{ "name", "ircd.gpt.pipe.code.opts" },
{ "default", string_view
{
" -cl-strict-aliasing"
" -cl-no-signed-zeros"
" -cl-finite-math-only"
" -cl-unsafe-math-optimizations"
" -cl-fast-relaxed-math"
" -cl-mad-enable"
" -cl-single-precision-constant"
//" -cl-fp32-correctly-rounded-divide-sqrt"
" -cl-kernel-arg-info"
}}
};
ircd::gpt::pipe::code::code()
:cl::code{[]
{
const string_view code_path
{
default_path
};
const fs::fd fd
{
code_path
};
const std::string read
{
fs::read(fd)
};
const string_view bin
{
read
};
const vector_view<const string_view> bins
(
&bin, 1
);
const auto opts
{
fmt::snstringf
{
4096, "%s -I%s",
string_view{default_opts},
string_view{fs::base::include},
}
};
return cl::code
{
bins, opts
};
}()}
{
}
ircd::gpt::pipe::code::~code()
noexcept
{
}
//
// pipe::desc
//
ircd::gpt::pipe::desc::desc(pipe::code &code,
pipe::model &model)
:model
{
&model
}
,code
{
&code
}
,state
{
0
+ 12 * 512 * 3 * 768 * sizeof(float),
mutable_buffer{},
}
,master
{
0
+ 512 * 768 * sizeof(float)
+ 65536 * sizeof(float)
+ 65536 * sizeof(float)
+ 65536 * sizeof(float)
,mutable_buffer{}
}
,accum
{
master,
{
512 * 768 * sizeof(float),
off_t(0),
},
}
,logit
{
master,
{
65536 * sizeof(float),
accum.offset() + off_t(accum.size()),
},
}
,logsm
{
master,
{
65536 * sizeof(float),
logit.offset() + off_t(logit.size()),
},
}
,ctrl
{
sizeof(gpt::ctrl),
mutable_buffer{}
}
,opts
{
sizeof(gpt::opts),
const_buffer{}
}
,lm_embed
{
code,
"ircd_gpt_lm_embed",
ctrl,
opts,
accum,
model.embed->pos.param,
model.embed->token.param,
}
,lm_norm
{
code,
"ircd_gpt_lm_norm",
ctrl,
opts,
accum,
model.decode->norm.bias.param,
model.decode->norm.weight.param,
}
,lm_logit
{
code,
"ircd_gpt_lm_logit",
ctrl,
opts,
logit,
accum,
model.embed->token.param,
}
,lm_logsm
{
code,
"ircd_gpt_lm_logsm",
ctrl,
opts,
logsm,
logit,
}
,lm_select
{
code,
"ircd_gpt_lm_select",
ctrl,
opts,
logsm,
logit,
}
,lm_norm_backprop
{
code,
"ircd_gpt_norm_prop",
ctrl,
opts,
model.decode->norm.bias.param,
model.decode->norm.bias.moment[0],
model.decode->norm.bias.moment[1],
model.decode->norm.weight.param,
model.decode->norm.weight.moment[0],
model.decode->norm.weight.moment[1],
}
,lm_embed_backprop
{
code,
"ircd_gpt_lm_embed_prop",
ctrl,
opts,
model.embed->pos.param,
model.embed->pos.moment[0],
model.embed->pos.moment[1],
model.embed->token.param,
model.embed->token.moment[0],
model.embed->token.moment[1],
}
,layer
{
std::make_unique<struct desc::layer>(*this, 0x00),
std::make_unique<struct desc::layer>(*this, 0x01),
std::make_unique<struct desc::layer>(*this, 0x02),
std::make_unique<struct desc::layer>(*this, 0x03),
std::make_unique<struct desc::layer>(*this, 0x04),
std::make_unique<struct desc::layer>(*this, 0x05),
std::make_unique<struct desc::layer>(*this, 0x06),
std::make_unique<struct desc::layer>(*this, 0x07),
std::make_unique<struct desc::layer>(*this, 0x08),
std::make_unique<struct desc::layer>(*this, 0x09),
std::make_unique<struct desc::layer>(*this, 0x0a),
std::make_unique<struct desc::layer>(*this, 0x0b),
}
{
}
//
// pipe::desc::layer
//
ircd::gpt::pipe::desc::layer::layer(pipe::desc &desc,
const int laynum)
:state
{
desc.state,
{
512 * 3 * 768 * sizeof(float),
laynum * 512 * 3 * 768 * sizeof(float),
}
}
,negative
{
*desc.code,
"ircd_gpt_attn_fcon",
desc.ctrl,
desc.opts,
state,
desc.accum,
desc.model->decode->block[laynum].attn.norm.bias.param,
desc.model->decode->block[laynum].attn.norm.weight.param,
desc.model->decode->block[laynum].attn.fcon.bias.param,
desc.model->decode->block[laynum].attn.fcon.weight.param,
}
,positive
{
*desc.code,
"ircd_gpt_coil",
desc.ctrl,
desc.opts,
desc.accum,
state,
desc.model->decode->block[laynum].attn.proj.bias.param,
desc.model->decode->block[laynum].attn.proj.weight.param,
desc.model->decode->block[laynum].ffnn.norm.bias.param,
desc.model->decode->block[laynum].ffnn.norm.weight.param,
desc.model->decode->block[laynum].ffnn.fcon.bias.param,
desc.model->decode->block[laynum].ffnn.fcon.weight.param,
desc.model->decode->block[laynum].ffnn.proj.bias.param,
desc.model->decode->block[laynum].ffnn.proj.weight.param,
}
,backattn
{
*desc.code,
"ircd_gpt_coil_prop_attn",
desc.ctrl,
desc.opts,
desc.model->decode->block[laynum].attn.norm.bias.param,
desc.model->decode->block[laynum].attn.norm.bias.moment[0],
desc.model->decode->block[laynum].attn.norm.bias.moment[1],
desc.model->decode->block[laynum].attn.norm.weight.param,
desc.model->decode->block[laynum].attn.norm.weight.moment[0],
desc.model->decode->block[laynum].attn.norm.weight.moment[1],
desc.model->decode->block[laynum].attn.fcon.bias.param,
desc.model->decode->block[laynum].attn.fcon.bias.moment[0],
desc.model->decode->block[laynum].attn.fcon.bias.moment[1],
desc.model->decode->block[laynum].attn.fcon.weight.param,
desc.model->decode->block[laynum].attn.fcon.weight.moment[0],
desc.model->decode->block[laynum].attn.fcon.weight.moment[1],
desc.model->decode->block[laynum].attn.proj.bias.param,
desc.model->decode->block[laynum].attn.proj.bias.moment[0],
desc.model->decode->block[laynum].attn.proj.bias.moment[1],
desc.model->decode->block[laynum].attn.proj.weight.param,
desc.model->decode->block[laynum].attn.proj.weight.moment[0],
desc.model->decode->block[laynum].attn.proj.weight.moment[1],
}
,backffnn
{
*desc.code,
"ircd_gpt_coil_prop_ffnn",
desc.ctrl,
desc.opts,
desc.model->decode->block[laynum].ffnn.norm.bias.param,
desc.model->decode->block[laynum].ffnn.norm.bias.moment[0],
desc.model->decode->block[laynum].ffnn.norm.bias.moment[1],
desc.model->decode->block[laynum].ffnn.norm.weight.param,
desc.model->decode->block[laynum].ffnn.norm.weight.moment[0],
desc.model->decode->block[laynum].ffnn.norm.weight.moment[1],
desc.model->decode->block[laynum].ffnn.fcon.bias.param,
desc.model->decode->block[laynum].ffnn.fcon.bias.moment[0],
desc.model->decode->block[laynum].ffnn.fcon.bias.moment[1],
desc.model->decode->block[laynum].ffnn.fcon.weight.param,
desc.model->decode->block[laynum].ffnn.fcon.weight.moment[0],
desc.model->decode->block[laynum].ffnn.fcon.weight.moment[1],
desc.model->decode->block[laynum].ffnn.proj.bias.param,
desc.model->decode->block[laynum].ffnn.proj.bias.moment[0],
desc.model->decode->block[laynum].ffnn.proj.bias.moment[1],
desc.model->decode->block[laynum].ffnn.proj.weight.param,
desc.model->decode->block[laynum].ffnn.proj.weight.moment[0],
desc.model->decode->block[laynum].ffnn.proj.weight.moment[1],
}
{
}
///////////////////////////////////////////////////////////////////////////////
//
// model
//
//
// pipe::model::model
//
ircd::gpt::pipe::model::model(gpt::model::decoder &decoder,
gpt::model::embed &embed)
:decode
{
std::make_unique<model::decoder>(decoder)
}
,embed
{
std::make_unique<model::language>(embed)
}
{
}
ircd::gpt::pipe::model::model(const gpt::model::decoder &decoder,
const gpt::model::embed &embed)
:decode
{
std::make_unique<model::decoder>(decoder)
}
,embed
{
std::make_unique<model::language>(embed)
}
{
}
ircd::gpt::pipe::model::~model()
noexcept
{
}
//
// pipe::model::decoder
//
ircd::gpt::pipe::model::decoder::decoder(gpt::model::decoder &decoder)
:master
{
// params
{
sizeof(gpt::model::block) * 12 + sizeof(gpt::model::norm), mutable_buffer
{
reinterpret_cast<char *>(decoder.layer),
sizeof(decoder.layer) + sizeof(decoder.f)
}
},
// first moment
{
sizeof(gpt::model::block) * 12 + sizeof(gpt::model::norm),
mutable_buffer{}
},
// second moment
{
sizeof(gpt::model::block) * 12 + sizeof(gpt::model::norm),
mutable_buffer{}
},
}
,block
{
{ master, sizeof(gpt::model::block) * 0x00, decoder.layer[0x00], 0x00, },
{ master, sizeof(gpt::model::block) * 0x01, decoder.layer[0x01], 0x01, },
{ master, sizeof(gpt::model::block) * 0x02, decoder.layer[0x02], 0x02, },
{ master, sizeof(gpt::model::block) * 0x03, decoder.layer[0x03], 0x03, },
{ master, sizeof(gpt::model::block) * 0x04, decoder.layer[0x04], 0x04, },
{ master, sizeof(gpt::model::block) * 0x05, decoder.layer[0x05], 0x05, },
{ master, sizeof(gpt::model::block) * 0x06, decoder.layer[0x06], 0x06, },
{ master, sizeof(gpt::model::block) * 0x07, decoder.layer[0x07], 0x07, },
{ master, sizeof(gpt::model::block) * 0x08, decoder.layer[0x08], 0x08, },
{ master, sizeof(gpt::model::block) * 0x09, decoder.layer[0x09], 0x09, },
{ master, sizeof(gpt::model::block) * 0x0a, decoder.layer[0x0a], 0x0a, },
{ master, sizeof(gpt::model::block) * 0x0b, decoder.layer[0x0b], 0x0b, },
}
,norm
{
master,
off_t(sizeof(gpt::model::block) * 12),
mutable_buffer{decoder.f.bias},
mutable_buffer{decoder.f.weight},
}
{
}
ircd::gpt::pipe::model::decoder::decoder(const gpt::model::decoder &decoder)
:master
{
// params
{
sizeof(gpt::model::block) * 12 + sizeof(gpt::model::norm), const_buffer
{
reinterpret_cast<const char *>(decoder.layer),
sizeof(decoder.layer) + sizeof(decoder.f)
}
},
}
,block
{
{ master, sizeof(gpt::model::block) * 0x00, decoder.layer[0x00], 0x00, },
{ master, sizeof(gpt::model::block) * 0x01, decoder.layer[0x01], 0x01, },
{ master, sizeof(gpt::model::block) * 0x02, decoder.layer[0x02], 0x02, },
{ master, sizeof(gpt::model::block) * 0x03, decoder.layer[0x03], 0x03, },
{ master, sizeof(gpt::model::block) * 0x04, decoder.layer[0x04], 0x04, },
{ master, sizeof(gpt::model::block) * 0x05, decoder.layer[0x05], 0x05, },
{ master, sizeof(gpt::model::block) * 0x06, decoder.layer[0x06], 0x06, },
{ master, sizeof(gpt::model::block) * 0x07, decoder.layer[0x07], 0x07, },
{ master, sizeof(gpt::model::block) * 0x08, decoder.layer[0x08], 0x08, },
{ master, sizeof(gpt::model::block) * 0x09, decoder.layer[0x09], 0x09, },
{ master, sizeof(gpt::model::block) * 0x0a, decoder.layer[0x0a], 0x0a, },
{ master, sizeof(gpt::model::block) * 0x0b, decoder.layer[0x0b], 0x0b, },
}
,norm
{
master,
off_t(sizeof(gpt::model::block) * 12),
const_buffer{decoder.f.bias},
const_buffer{decoder.f.weight},
}
{
}
ircd::gpt::pipe::model::decoder::~decoder()
noexcept
{
}
//
// pipe::model::language
//
ircd::gpt::pipe::model::language::language(gpt::model::embed &embed)
:master
{
// params
{
sizeof(embed), mutable_buffer
{
reinterpret_cast<char *>(&embed),
sizeof(embed),
}
},
// first moment
{
sizeof(embed), mutable_buffer{},
},
// second moment
{
sizeof(embed), mutable_buffer{},
},
}
,pos
{
master, 0, mutable_buffer{embed.pos}
}
,token
{
master, sizeof(embed.pos), mutable_buffer{embed.token}
}
{
}
ircd::gpt::pipe::model::language::language(const gpt::model::embed &embed)
:master
{
{
sizeof(embed), const_buffer
{
reinterpret_cast<const char *>(&embed),
sizeof(embed),
}
},
}
,pos
{
master, 0, const_buffer{embed.pos}
}
,token
{
master, sizeof(embed.pos), const_buffer{embed.token}
}
{
}
ircd::gpt::pipe::model::language::language(cl::data *const master,
const off_t offset,
gpt::model::embed &embed)
:pos
{
master, offset, mutable_buffer{embed.pos}
}
,token
{
master, offset + off_t(sizeof(embed.pos)), mutable_buffer{embed.token}
}
{
}
ircd::gpt::pipe::model::language::language(cl::data *const master,
const off_t offset,
const gpt::model::embed &embed)
:pos
{
master, offset, const_buffer{embed.pos}
}
,token
{
master, offset + off_t(sizeof(embed.pos)), const_buffer{embed.token}
}
{
}
ircd::gpt::pipe::model::language::~language()
noexcept
{
}
//
// pipe::model::block
//
ircd::gpt::pipe::model::block::block(gpt::model::block &block,
const size_t layer)
:master
{
// params
{
sizeof(block), mutable_buffer
{
reinterpret_cast<char *>(&block), sizeof(block)
}
},
// first moment
{
sizeof(block),
mutable_buffer{}
},
// second moment
{
sizeof(block),
mutable_buffer{}
},
}
,attn
{
master,
0,
block.ln1,
block.attn,
}
,ffnn
{
master,
off_t(sizeof(block.ln1) + sizeof(block.attn)),
block.ln2,
block.ffnn,
}
{
}
ircd::gpt::pipe::model::block::block(const gpt::model::block &block,
const size_t layer)
:master
{
// params
{
sizeof(block), const_buffer
{
reinterpret_cast<const char *>(&block), sizeof(block)
}
}
}
,attn
{
master,
0,
block.ln1,
block.attn,
}
,ffnn
{
master,
off_t(sizeof(block.ln1) + sizeof(block.attn)),
block.ln2,
block.ffnn,
}
{
}
ircd::gpt::pipe::model::block::block(cl::data *const master,
const off_t offset,
gpt::model::block &block,
const size_t layer)
:attn
{
master,
offset,
block.ln1,
block.attn,
}
,ffnn
{
master,
offset + off_t(sizeof(block.ln1) + sizeof(block.attn)),
block.ln2,
block.ffnn,
}
{
}
ircd::gpt::pipe::model::block::block(cl::data *const master,
const off_t offset,
const gpt::model::block &block,
const size_t layer)
:attn
{
master,
offset,
block.ln1,
block.attn,
}
,ffnn
{
master,
offset + off_t(sizeof(block.ln1) + sizeof(block.attn)),
block.ln2,
block.ffnn,
}
{
}
//
// pipe::model::ffnn
//
ircd::gpt::pipe::model::ffnn::ffnn(cl::data *const master,
const off_t offset,
gpt::model::norm &norm,
gpt::model::ffnn &ffnn)
:norm
{
master,
offset,
mutable_buffer{norm.bias},
mutable_buffer{norm.weight},
}
,fcon
{
master,
offset + off_t(sizeof(norm)),
mutable_buffer{ffnn.fc_bias},
mutable_buffer{ffnn.fc_weight},
}
,proj
{
master,
offset + off_t(sizeof(norm) + sizeof(ffnn.fc_bias) + sizeof(ffnn.fc_weight)),
mutable_buffer{ffnn.proj_bias},
mutable_buffer{ffnn.proj_weight},
}
{
always_assert
(
ircd::data(const_buffer{ffnn.proj_weight})
==
ircd::data(const_buffer{norm.bias}) +
sizeof(norm) +
sizeof(ffnn.fc_bias) +
sizeof(ffnn.fc_weight) +
ircd::size(const_buffer{ffnn.proj_bias})
);
}
ircd::gpt::pipe::model::ffnn::ffnn(cl::data *const master,
const off_t offset,
const gpt::model::norm &norm,
const gpt::model::ffnn &ffnn)
:norm
{
master,
offset,
const_buffer{norm.bias},
const_buffer{norm.weight},
}
,fcon
{
master,
offset + off_t(sizeof(norm)),
const_buffer{ffnn.fc_bias},
const_buffer{ffnn.fc_weight},
}
,proj
{
master,
offset + off_t(sizeof(norm) + sizeof(ffnn.fc_bias) + sizeof(ffnn.fc_weight)),
const_buffer{ffnn.proj_bias},
const_buffer{ffnn.proj_weight},
}
{
always_assert
(
ircd::data(const_buffer{ffnn.proj_weight})
==
ircd::data(const_buffer{norm.bias}) +
sizeof(norm) +
sizeof(ffnn.fc_bias) +
sizeof(ffnn.fc_weight) +
ircd::size(const_buffer{ffnn.proj_bias})
);
}
//
// pipe::model::attn
//
ircd::gpt::pipe::model::attn::attn(cl::data *const master,
const off_t offset,
gpt::model::norm &norm,
gpt::model::attn &attn)
:norm
{
master,
offset,
mutable_buffer{norm.bias},
mutable_buffer{norm.weight},
}
,fcon
{
master,
offset + off_t(sizeof(norm)),
mutable_buffer{attn.attn_bias},
mutable_buffer{attn.attn_weight},
}
,proj
{
master,
offset + off_t(sizeof(norm) + sizeof(attn.attn_bias) + sizeof(attn.attn_weight)),
mutable_buffer{attn.proj_bias},
mutable_buffer{attn.proj_weight},
}
{
always_assert
(
ircd::data(const_buffer{attn.proj_weight})
==
ircd::data(const_buffer{norm.bias}) +
sizeof(norm) +
sizeof(attn.attn_bias) +
sizeof(attn.attn_weight) +
ircd::size(const_buffer{attn.proj_bias})
);
}
ircd::gpt::pipe::model::attn::attn(cl::data *const master,
const off_t offset,
const gpt::model::norm &norm,
const gpt::model::attn &attn)
:norm
{
master,
offset,
const_buffer{norm.bias},
const_buffer{norm.weight},
}
,fcon
{
master,
offset + off_t(sizeof(norm)),
const_buffer{attn.attn_bias},
const_buffer{attn.attn_weight},
}
,proj
{
master,
offset + off_t(sizeof(norm) + sizeof(attn.attn_bias) + sizeof(attn.attn_weight)),
const_buffer{attn.proj_bias},
const_buffer{attn.proj_weight},
}
{
always_assert
(
ircd::data(const_buffer{attn.proj_weight})
==
ircd::data(const_buffer{norm.bias}) +
sizeof(norm) +
sizeof(attn.attn_bias) +
sizeof(attn.attn_weight) +
ircd::size(const_buffer{attn.proj_bias})
);
}
//
// pipe::model::tensor
//
ircd::gpt::pipe::model::tensor::tensor(cl::data *const master,
const off_t offset,
const mutable_buffer &bias,
const mutable_buffer &weight)
:bias
{
master,
offset,
bias,
}
,weight
{
master,
off_t(offset + ircd::size(bias)),
weight,
}
{
}
ircd::gpt::pipe::model::tensor::tensor(cl::data *const master,
const off_t offset,
const const_buffer &bias,
const const_buffer &weight)
:bias
{
master,
offset,
bias,
}
,weight
{
master,
off_t(offset + ircd::size(bias)),
weight,
}
{
}
//
// pipe::model::matrix
//
ircd::gpt::pipe::model::matrix::matrix(cl::data *const master,
const off_t offset,
const mutable_buffer &param)
:param
{
master[0],
{
ircd::size(param),
offset,
},
}
,moment
{
// first moment
{
master[1],
{
ircd::size(param),
offset,
},
},
// second moment
{
master[2],
{
ircd::size(param),
offset,
},
},
}
{
}
ircd::gpt::pipe::model::matrix::matrix(cl::data *const master,
const off_t offset,
const const_buffer &param)
:param
{
master[0],
{
ircd::size(param), // size
offset, // offset
},
}
{
}
///////////////////////////////////////////////////////////////////////////////
//
// pipe::code
//
decltype(ircd::gpt::pipe::code::default_path)
ircd::gpt::pipe::code::default_path
{
{ "name", "ircd.gpt.pipe.code.path" },
};
decltype(ircd::gpt::pipe::code::cache_path)
ircd::gpt::pipe::code::cache_path
{
{ "name", "ircd.gpt.pipe.code.cache.path" },
};
decltype(ircd::gpt::pipe::code::default_opts)
ircd::gpt::pipe::code::default_opts
{
{ "name", "ircd.gpt.pipe.code.opts" },
{ "default", string_view
{
" -cl-strict-aliasing"
" -cl-no-signed-zeros"
" -cl-finite-math-only"
" -cl-unsafe-math-optimizations"
" -cl-fast-relaxed-math"
" -cl-mad-enable"
" -cl-single-precision-constant"
//" -cl-fp32-correctly-rounded-divide-sqrt"
" -cl-kernel-arg-info"
" -O3"
}}
};
ircd::gpt::pipe::code::code()
:cl::code{[]
{
const auto opts
{
fmt::snstringf
{
4096, "%s -I%s",
string_view{default_opts},
string_view{fs::base::include},
}
};
const auto cache_path
{
make_cache_path(fs::path_scratch)
};
if(cache_path)
if(fs::is_reg(cache_path))
return from_cache(opts, cache_path);
return from_source(opts);
}()}
{
put_cache();
}
ircd::gpt::pipe::code::~code()
noexcept
{
}
bool
ircd::gpt::pipe::code::put_cache()
{
const auto cache_path
{
make_cache_path(fs::path_scratch)
};
if(!cache_path)
return false;
if(fs::is_reg(cache_path))
return false;
set_cache(cache_path);
return true;
}
void
ircd::gpt::pipe::code::set_cache(const string_view &path)
{
const unique_mutable_buffer cache_buf
{
this->bins_size()
};
mutable_buffer _cache_bufs[1]
{
cache_buf
};
const auto cache_bufs
{
this->bin(_cache_bufs)
};
const fs::fd fd
{
path, std::ios::out
};
const auto written
{
fs::write(fd, cache_bufs[0])
};
assert(!empty(written));
}
ircd::cl::code
ircd::gpt::pipe::code::from_source(const string_view &opts)
{
const string_view code_path
{
default_path
};
const fs::fd fd
{
code_path
};
const std::string read
{
fs::read(fd)
};
const string_view src
{
read
};
const vector_view<const string_view> srcs
(
&src, 1
);
return cl::code
{
srcs, opts
};
}
ircd::cl::code
ircd::gpt::pipe::code::from_cache(const string_view &opts,
const string_view &cache_path)
{
const fs::fd fd
{
cache_path
};
const std::string read
{
fs::read(fd)
};
const const_buffer bin
{
read
};
const vector_view<const const_buffer> bins
(
&bin, 1
);
return cl::code
{
bins, opts
};
}
ircd::string_view
ircd::gpt::pipe::code::make_cache_path(const mutable_buffer &buf)
{
if(!cache_path)
return {};
const string_view &src_path
{
default_path
};
const string_view &src_fname
{
fs::filename(fs::path_scratch, src_path)
};
const string_view &cache_fname
{
fs::extension(fs::name_scratch, src_fname, ".nir")
};
const string_view parts[]
{
string_view{cache_path},
string_view{cache_fname}
};
return fs::path(buf, fs::path_views{parts});
}
Reference in a new issue View git blame Copy permalink