From 01aec21d25b41a3add54c869022f89dd11e7684f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Verschelde?= Date: Wed, 20 Oct 2021 13:47:50 +0200 Subject: [PATCH] i18n: Only include editor translations above a threshold This reduces the size of the editor binaries significantly, as we otherwise embed all WIP translations, including ones with very low completion ratios, and end up paying for the size of all `msgid`s for each locale. Cf. https://github.com/godotengine/godot-proposals/issues/3421 for details. The thresholds used are: - 30% for the editor interface (should already include most common strings while more obscure ones like UndoRedo action names might be untranslated). - 10% for the class reference: this is a HUGE resource and 10% is already a lot of useful content, especially if focused on the most used APIs. This currently reduces the size of the editor binary by 17% on Linux. The list will be synced manually every now and then. (cherry picked from commit 8425c589911f8ffc006990f4149859e175ae93a9) --- doc/translations/Makefile | 21 +++++++++++++++++++++ editor/SCsub | 21 ++++++++++++++------- editor/translations/Makefile | 21 +++++++++++++++++++++ 3 files changed, 56 insertions(+), 7 deletions(-) diff --git a/doc/translations/Makefile b/doc/translations/Makefile index ceee79ec95..ab08e107d0 100644 --- a/doc/translations/Makefile +++ b/doc/translations/Makefile @@ -21,3 +21,24 @@ merge: check: @for po in $(POFILES); do msgfmt -c $$po -o /dev/null; done + +# Generate completion ratio from statistics string such as: +# 2775 translated messages, 272 fuzzy translations, 151 untranslated messages. +# First number can be 0, second and third numbers are only present if non-zero. +include-list: + @list=""; \ + threshold=0.10; \ + for po in $(POFILES); do \ + res=`msgfmt --statistics $$po -o /dev/null 2>&1 | sed 's/[^0-9,]*//g'`; \ + complete=`cut -d',' -f1 <<< $$res`; \ + fuzzy_or_untranslated=`cut -d',' -f2 <<< $$res`; \ + untranslated_maybe=`cut -d',' -f3 <<< $$res`; \ + if [ -z "$$fuzzy_or_untranslated" ]; then fuzzy_or_untranslated=0; fi; \ + if [ -z "$$untranslated_maybe" ]; then untranslated_maybe=0; fi; \ + incomplete=`expr $$fuzzy_or_untranslated + $$untranslated_maybe`; \ + if `awk "BEGIN {exit !($$complete / ($$complete + $$incomplete) > $$threshold)}"`; then \ + lang=`basename $$po .po`; \ + list+="$$lang,"; \ + fi; \ + done; \ + echo $$list; diff --git a/editor/SCsub b/editor/SCsub index e8d417319a..6b18eeae93 100644 --- a/editor/SCsub +++ b/editor/SCsub @@ -5,7 +5,6 @@ Import("env") env.editor_sources = [] import os -import os.path import glob import editor_builders @@ -59,7 +58,7 @@ if env["tools"]: else: docs += Glob(d + "/*.xml") # Custom. - _make_doc_data_class_path(os.path.join(env.Dir("#").abspath, "editor")) + _make_doc_data_class_path(env.Dir("#editor").abspath) docs = sorted(docs) env.Depends("#editor/doc_data_compressed.gen.h", docs) @@ -69,10 +68,17 @@ if env["tools"]: env.Run(editor_builders.make_doc_header, "Generating documentation header."), ) - path = env.Dir(".").abspath + # Editor interface and class reference translations incur a significant size + # cost for the editor binary (see godot-proposals#3421). + # To limit it, we only include translations with a high enough completion + # ratio (30% for the editor UI, 10% for the class reference). + # Generated with `make include-list` for each resource. # Editor translations - tlist = glob.glob(path + "/translations/*.po") + to_include = ( + "ar,bg,bn,ca,cs,de,el,eo,es_AR,es,fi,fr,gl,he,hu,id,it,ja,ko,ms,nb,nl,pl,pt_BR,pt,ro,ru,sk,sv,th,tr,uk,vi,zh_CN,zh_TW" + ).split(",") + tlist = [env.Dir("#editor/translations").abspath + "/" + f + ".po" for f in to_include] env.Depends("#editor/editor_translations.gen.h", tlist) env.CommandNoCache( "#editor/editor_translations.gen.h", @@ -81,7 +87,8 @@ if env["tools"]: ) # Documentation translations - tlist = glob.glob(env.Dir("#doc").abspath + "/translations/*.po") + to_include = "es,fr,ja,zh_CN".split(",") + tlist = [env.Dir("#doc/translations").abspath + "/" + f + ".po" for f in to_include] env.Depends("#editor/doc_translations.gen.h", tlist) env.CommandNoCache( "#editor/doc_translations.gen.h", @@ -90,8 +97,8 @@ if env["tools"]: ) # Fonts - flist = glob.glob(path + "/../thirdparty/fonts/*.ttf") - flist.extend(glob.glob(path + "/../thirdparty/fonts/*.otf")) + flist = glob.glob(env.Dir("#thirdparty").abspath + "/fonts/*.ttf") + flist.extend(glob.glob(env.Dir("#thirdparty").abspath + "/fonts/*.otf")) flist.sort() env.Depends("#editor/builtin_fonts.gen.h", flist) env.CommandNoCache( diff --git a/editor/translations/Makefile b/editor/translations/Makefile index 1843114f06..82b3d49c59 100644 --- a/editor/translations/Makefile +++ b/editor/translations/Makefile @@ -18,3 +18,24 @@ merge: check: @for po in $(POFILES); do msgfmt -c $$po -o /dev/null; done + +# Generate completion ratio from statistics string such as: +# 2775 translated messages, 272 fuzzy translations, 151 untranslated messages. +# First number can be 0, second and third numbers are only present if non-zero. +include-list: + @list=""; \ + threshold=0.30; \ + for po in $(POFILES); do \ + res=`msgfmt --statistics $$po -o /dev/null 2>&1 | sed 's/[^0-9,]*//g'`; \ + complete=`cut -d',' -f1 <<< $$res`; \ + fuzzy_or_untranslated=`cut -d',' -f2 <<< $$res`; \ + untranslated_maybe=`cut -d',' -f3 <<< $$res`; \ + if [ -z "$$fuzzy_or_untranslated" ]; then fuzzy_or_untranslated=0; fi; \ + if [ -z "$$untranslated_maybe" ]; then untranslated_maybe=0; fi; \ + incomplete=`expr $$fuzzy_or_untranslated + $$untranslated_maybe`; \ + if `awk "BEGIN {exit !($$complete / ($$complete + $$incomplete) > $$threshold)}"`; then \ + lang=`basename $$po .po`; \ + list+="$$lang,"; \ + fi; \ + done; \ + echo $$list;