MirrorHub/construct
0
0
Fork 0
mirror of https://github.com/matrix-construct/construct synced 2024-11-16 15:00:51 +01:00
Code Issues Releases Wiki Activity
construct/ircd/gpt_model.cc

672 lines
17 KiB
C++
Raw Blame History

// Tensor Construct
//
// Copyright (C) Matrix Construct Developers, Authors & Contributors
// Copyright (C) 2016-2021 Jason Volk <jason@zemos.net>
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice is present in all copies. The
// full license for this software is available in the LICENSE file.
namespace ircd::gpt::model
{
using init_func = void (*)(decoder &, const string_view &, const size_t &, const json::array &);
using init_handler = std::pair<string_view, init_func>;
static void
init_f_weight(decoder &, const string_view &, const size_t &, const json::array &),
init_f_bias(decoder &, const string_view &, const size_t &, const json::array &),
init_wpe_weight(decoder &, const string_view &, const size_t &, const json::array &),
init_wte_weight(decoder &, const string_view &, const size_t &, const json::array &),
init_h_ffnn_fc_weight(decoder &, const string_view &, const size_t &, const json::array &),
init_h_ffnn_fc_bias(decoder &, const string_view &, const size_t &, const json::array &),
init_h_ffnn_proj_weight(decoder &, const string_view &, const size_t &, const json::array &),
init_h_ffnn_proj_bias(decoder &, const string_view &, const size_t &, const json::array &),
init_h_ln_1_weight(decoder &, const string_view &, const size_t &, const json::array &),
init_h_ln_1_bias(decoder &, const string_view &, const size_t &, const json::array &),
init_h_ln_2_weight(decoder &, const string_view &, const size_t &, const json::array &),
init_h_ln_2_bias(decoder &, const string_view &, const size_t &, const json::array &),
init_h_attn_attn_weight(decoder &, const string_view &, const size_t &, const json::array &),
init_h_attn_attn_bias(decoder &, const string_view &, const size_t &, const json::array &),
init_h_attn_proj_weight(decoder &, const string_view &, const size_t &, const json::array &),
init_h_attn_proj_bias(decoder &, const string_view &, const size_t &, const json::array &),
init_h_attn_bias(decoder &, const string_view &, const size_t &, const json::array &);
static bool init_dataset(const string_view &);
static bool init_from_cache(const string_view &);
static void init_from_json_handle(decoder &, const init_handler &, const size_t &);
static void init_from_json(const string_view &, const string_view &);
static void init(), fini() noexcept;
extern const init_handler
manifest[],
manifest_h[];
extern conf::item<bool>
cache_locked,
cache_shared,
cache_hugepage;
extern conf::item<std::string>
path,
cache_path,
dataset_path;
static fs::map
default_model_shm,
default_dataset_shm;
static std::unique_ptr<decoder> default_model_res;
}
decltype(ircd::gpt::model::manifest_h)
ircd::gpt::model::manifest_h
{
{ "h.%u.mlp.c_fc.weight.json", init_h_ffnn_fc_weight, },
{ "h.%u.mlp.c_fc.bias.json", init_h_ffnn_fc_bias, },
{ "h.%u.mlp.c_proj.weight.json", init_h_ffnn_proj_weight, },
{ "h.%u.mlp.c_proj.bias.json", init_h_ffnn_proj_bias, },
{ "h.%u.ln_1.weight.json", init_h_ln_1_weight, },
{ "h.%u.ln_1.bias.json", init_h_ln_1_bias, },
{ "h.%u.ln_2.weight.json", init_h_ln_2_weight, },
{ "h.%u.ln_2.bias.json", init_h_ln_2_bias, },
{ "h.%u.attn.c_attn.weight.json", init_h_attn_attn_weight, },
{ "h.%u.attn.c_attn.bias.json", init_h_attn_attn_bias, },
{ "h.%u.attn.c_proj.weight.json", init_h_attn_proj_weight, },
{ "h.%u.attn.c_proj.bias.json", init_h_attn_proj_bias },
{ "h.%u.attn.bias.json", init_h_attn_bias, },
};
decltype(ircd::gpt::model::manifest)
ircd::gpt::model::manifest
{
{ "ln_f.weight.json", init_f_weight, },
{ "ln_f.bias.json", init_f_bias, },
{ "wpe.weight.json", init_wpe_weight },
{ "wte.weight.json", init_wte_weight },
};
decltype(ircd::gpt::model::cache_locked)
ircd::gpt::model::cache_locked
{
{ "name", "ircd.gpt.model.cache.locked" },
{ "default", false },
};
decltype(ircd::gpt::model::cache_shared)
ircd::gpt::model::cache_shared
{
{ "name", "ircd.gpt.model.cache.shared" },
{ "default", false },
};
decltype(ircd::gpt::model::cache_hugepage)
ircd::gpt::model::cache_hugepage
{
{ "name", "ircd.gpt.model.cache.hugepage" },
{ "default", true },
};
decltype(ircd::gpt::model::cache_path)
ircd::gpt::model::cache_path
{
{ "name", "ircd.gpt.model.cache.path" },
{ "default", "model.cache.localhost" },
};
decltype(ircd::gpt::model::dataset_path)
ircd::gpt::model::dataset_path
{
{ "name", "ircd.gpt.model.dataset.path" },
{ "default", string_view{} },
};
decltype(ircd::gpt::model::path)
ircd::gpt::model::path
{
{
{ "name", "ircd.gpt.model.path" },
{ "default", string_view{} },
},
init
};
decltype(ircd::gpt::model::default_model)
ircd::gpt::model::default_model;
decltype(ircd::gpt::model::default_dataset)
ircd::gpt::model::default_dataset;
decltype(ircd::gpt::model::default_data)
ircd::gpt::model::default_data;
void
ircd::gpt::model::init()
{
if(!model::path)
return;
if(!init_from_cache(model::cache_path))
init_from_json(model::cache_path, model::path);
if(model::dataset_path)
init_dataset(model::dataset_path);
}
void
ircd::gpt::model::fini()
noexcept
{
default_model = nullptr;
default_model_shm = {};
default_dataset = nullptr;
default_data.clear();
default_dataset_shm = {};
}
bool
ircd::gpt::model::init_from_cache(const string_view &cache_path)
{
if(!fs::is_reg(cache_path))
return false;
const auto size
{
fs::size(cache_path)
};
if(unlikely(size != sizeof(model::decoder)))
throw error
{
"Cached model `%s' size %zu differs from %zu.",
cache_path,
size,
sizeof(model::decoder),
};
const fs::fd fd
{
cache_path, std::ios::in //| std::ios::out
};
fs::map::opts map_opts
{
std::ios::in | std::ios::out
};
map_opts.locked = bool(cache_locked);
map_opts.shared = bool(cache_shared);
map_opts.huge2mb = bool(cache_hugepage);
default_model_shm = fs::map
{
fd, map_opts, sizeof(decoder)
};
default_model = reinterpret_cast<decoder *>
(
data(default_model_shm)
);
char pbuf[48];
log::info
{
log, "model(%p) mapped cached model `%s' %s",
data(default_model_shm),
cache_path,
pretty(pbuf, iec(size)),
};
return true;
}
void
ircd::gpt::model::init_from_json(const string_view &cache_path,
const string_view &model_path)
{
util::timer stopwatch;
auto decoder(std::make_unique<model::decoder>());
memset(decoder.get(), 0x0, sizeof(model::decoder));
// Load the top level files, vocab etc
for(size_t i(0); i < 4; ++i)
init_from_json_handle(*decoder, manifest[i], 0);
// Load the transformer files by layer
const size_t layers {12};
for(size_t i(0); i < layers; ++i)
for(size_t j(0); j < 13; ++j)
init_from_json_handle(*decoder, manifest_h[j], i);
const const_buffer src
{
reinterpret_cast<char *>(decoder.get()), sizeof(model::decoder)
};
const auto wrote
{
fs::write(cache_path, src)
};
char pbuf[2][48];
log::info
{
log, "model(%p) parsed `%s' cached %s to `%s' in %s",
decoder.get(),
model_path,
pretty(pbuf[0], iec(size(wrote))),
cache_path,
stopwatch.pretty(pbuf[1]),
};
default_model_res = std::move(decoder);
default_model = default_model_res.get();
}
void
ircd::gpt::model::init_from_json_handle(decoder &d,
const init_handler &handler,
const size_t &layer)
{
const auto &[fmt, func]
{
handler
};
char namebuf[128];
const string_view path_part[2]
{
model::path, fmt::sprintf
{
namebuf, fmt, layer
}
};
const auto path
{
fs::path(fs::path_scratch, path_part)
};
fs::fd::opts fdopts;
fdopts.sequential = true;
const fs::fd fd
{
path, fdopts
};
// mmap of the file
const fs::map map
{
fd
};
// Each file is a JSON array at the top level.
const json::array matrix
{
map
};
// Readable name for logging
const auto &name
{
path_part[1]
};
if(likely(func))
func(d, name, layer, matrix);
// Check for interrupt after long operation
ctx::interruption_point();
log::info
{
log, "model(%p) loaded layer:%zu :%s",
&d,
layer,
name,
};
}
bool
ircd::gpt::model::init_dataset(const string_view &path)
{
if(!fs::is_reg(path))
return false;
const auto size
{
fs::size(path)
};
const fs::fd fd
{
path
};
fs::map::opts map_opts;
map_opts.huge2mb = bool(cache_hugepage);
default_dataset_shm = fs::map
{
fd, map_opts, size
};
default_dataset = string_view
(
default_dataset_shm
);
size_t checkpoint(0);
default_data.resize(260000); //TODO: XXX
ircd::tokens(default_dataset, '\n', [&checkpoint]
(const string_view &line)
{
default_data.at(checkpoint++) = line;
});
char pbuf[48];
log::info
{
log, "dataset(%p) mapped `%s' %s @%lu",
data(default_dataset_shm),
path,
pretty(pbuf, iec(size)),
checkpoint,
};
return true;
}
void
ircd::gpt::model::init_wpe_weight(decoder &d,
const string_view &name,
const size_t &layer,
const json::array &mat)
{
size_t i(0);
for(const json::array vec : mat)
{
size_t j(0);
for(const auto &elem : vec)
d.word.pos[i][j++] = lex_cast<float>(elem);
always_assert(j == sizeof(d.word.pos[i]) / sizeof(float));
++i;
}
}
void
ircd::gpt::model::init_wte_weight(decoder &d,
const string_view &name,
const size_t &layer,
const json::array &mat)
{
size_t i(0);
for(const json::array vec : mat)
{
size_t j(0);
for(const auto &elem : vec)
d.word.token[i][j++] = lex_cast<float>(elem);
always_assert(j == sizeof(d.word.token[i]) / sizeof(float));
++i;
}
}
void
ircd::gpt::model::init_f_weight(decoder &d,
const string_view &name,
const size_t &layer,
const json::array &vec)
{
size_t i(0);
for(const auto &elem : vec)
d.f.weight[i++] = lex_cast<float>(elem);
always_assert(i == sizeof(d.f.weight) / sizeof(float));
}
void
ircd::gpt::model::init_f_bias(decoder &d,
const string_view &name,
const size_t &layer,
const json::array &vec)
{
size_t i(0);
for(const auto &elem : vec)
d.f.bias[i++] = lex_cast<float>(elem);
always_assert(i == sizeof(d.f.bias) / sizeof(float));
}
void
ircd::gpt::model::init_h_ffnn_fc_weight(decoder &d,
const string_view &name,
const size_t &layer,
const json::array &mat)
{
size_t i(0);
for(const json::array vec : mat)
{
size_t j(0);
for(const auto &elem : vec)
d.layer[layer].ffnn.fc_weight[i][j++] = lex_cast<float>(elem);
always_assert(j == sizeof(d.layer[layer].ffnn.fc_weight[i]) / sizeof(float));
++i;
}
always_assert
(
i == sizeof(d.layer[layer].ffnn.fc_weight)
/ sizeof(d.layer[layer].ffnn.fc_weight[0])
);
}
void
ircd::gpt::model::init_h_ffnn_fc_bias(decoder &d,
const string_view &name,
const size_t &layer,
const json::array &vec)
{
size_t i(0);
for(const auto &elem : vec)
d.layer[layer].ffnn.fc_bias[i++] = lex_cast<float>(elem);
always_assert(i == sizeof(d.layer[layer].ffnn.fc_bias) / sizeof(float));
}
void
ircd::gpt::model::init_h_ffnn_proj_weight(decoder &d,
const string_view &name,
const size_t &layer,
const json::array &mat)
{
size_t i(0);
for(const json::array vec : mat)
{
size_t j(0);
for(const auto &elem : vec)
d.layer[layer].ffnn.proj_weight[i][j++] = lex_cast<float>(elem);
always_assert(j == sizeof(d.layer[layer].ffnn.proj_weight[i]) / sizeof(float));
++i;
}
always_assert
(
i == sizeof(d.layer[layer].ffnn.proj_weight)
/ sizeof(d.layer[layer].ffnn.proj_weight[0])
);
}
void
ircd::gpt::model::init_h_ffnn_proj_bias(decoder &d,
const string_view &name,
const size_t &layer,
const json::array &vec)
{
size_t i(0);
for(const auto &elem : vec)
d.layer[layer].ffnn.proj_bias[i++] = lex_cast<float>(elem);
always_assert(i == sizeof(d.layer[layer].ffnn.proj_bias) / sizeof(float));
}
void
ircd::gpt::model::init_h_ln_1_weight(decoder &d,
const string_view &name,
const size_t &layer,
const json::array &vec)
{
size_t i(0);
for(const auto &elem : vec)
d.layer[layer].ln1.weight[i++] = lex_cast<float>(elem);
always_assert(i == sizeof(d.layer[layer].ln1.weight) / sizeof(float));
}
void
ircd::gpt::model::init_h_ln_1_bias(decoder &d,
const string_view &name,
const size_t &layer,
const json::array &vec)
{
size_t i(0);
for(const auto &elem : vec)
d.layer[layer].ln1.bias[i++] = lex_cast<float>(elem);
always_assert(i == sizeof(d.layer[layer].ln1.bias) / sizeof(float));
}
void
ircd::gpt::model::init_h_ln_2_weight(decoder &d,
const string_view &name,
const size_t &layer,
const json::array &vec)
{
size_t i(0);
for(const auto &elem : vec)
d.layer[layer].ln2.weight[i++] = lex_cast<float>(elem);
always_assert(i == sizeof(d.layer[layer].ln2.weight) / sizeof(float));
}
void
ircd::gpt::model::init_h_ln_2_bias(decoder &d,
const string_view &name,
const size_t &layer,
const json::array &vec)
{
size_t i(0);
for(const auto &elem : vec)
d.layer[layer].ln2.bias[i++] = lex_cast<float>(elem);
always_assert(i == sizeof(d.layer[layer].ln2.bias) / sizeof(float));
}
void
ircd::gpt::model::init_h_attn_attn_weight(decoder &d,
const string_view &name,
const size_t &layer,
const json::array &mat)
{
size_t i(0);
for(const json::array vec : mat)
{
size_t j(0);
for(const auto &elem : vec)
d.layer[layer].attn.attn_weight[i][j++] = lex_cast<float>(elem);
always_assert(j == sizeof(d.layer[layer].attn.attn_weight[i]) / sizeof(float));
++i;
}
always_assert
(
i == sizeof(d.layer[layer].attn.attn_weight)
/ sizeof(d.layer[layer].attn.attn_weight[0])
);
}
void
ircd::gpt::model::init_h_attn_attn_bias(decoder &d,
const string_view &name,
const size_t &layer,
const json::array &vec)
{
size_t i(0);
for(const auto &elem : vec)
d.layer[layer].attn.attn_bias[i++] = lex_cast<float>(elem);
always_assert(i == sizeof(d.layer[layer].attn.attn_bias) / sizeof(float));
}
void
ircd::gpt::model::init_h_attn_proj_weight(decoder &d,
const string_view &name,
const size_t &layer,
const json::array &mat)
{
size_t i(0);
for(const json::array vec : mat)
{
size_t j(0);
for(const auto &elem : vec)
d.layer[layer].attn.proj_weight[i][j++] = lex_cast<float>(elem);
always_assert(j == sizeof(d.layer[layer].attn.proj_weight[i]) / sizeof(float));
++i;
}
always_assert
(
i == sizeof(d.layer[layer].attn.proj_weight)
/ sizeof(d.layer[layer].attn.proj_weight[0])
);
}
void
ircd::gpt::model::init_h_attn_proj_bias(decoder &d,
const string_view &name,
const size_t &layer,
const json::array &vec)
{
size_t i(0);
for(const auto &elem : vec)
d.layer[layer].attn.proj_bias[i++] = lex_cast<float>(elem);
always_assert(i == sizeof(d.layer[layer].attn.proj_bias) / sizeof(float));
}
void
ircd::gpt::model::init_h_attn_bias(decoder &d,
const string_view &name,
const size_t &layer,
const json::array &mat)
{
for(const json::array dim0 : mat)
{
for(const json::array dim1 : dim0)
{
size_t k(0);
for(const json::array dim2 : dim1)
{
size_t l(0);
for(const auto &elem : dim2)
{
always_assert(elem == "1.0" || elem == "0.0");
d.layer[layer].attn.bias[k][l++] = startswith(elem, '1');
}
++k;
}
always_assert
(
k == sizeof(d.layer[layer].attn.bias)
/ sizeof(d.layer[layer].attn.bias[0])
);
}
}
}
Reference in a new issue View git blame Copy permalink