From b88566cc0ca8e6a3c1dac07d46ec6a6702e010f1 Mon Sep 17 00:00:00 2001
From: "Wladimir J. van der Laan" <laanwj@gmail.com>
Date: Wed, 30 Apr 2014 09:46:04 +0200
Subject: [PATCH] devtools: add a script to fetch and postprocess translations

Run this script from the root of the repository to update all translations from transifex.
It will do the following automatically:

- create a transifex configuration file
- fetch all translations
- post-process them into valid and committable format

Conflicts:

	contrib/devtools/README.md
---
 contrib/devtools/README.md              | 35 ++++++++++++-
 contrib/devtools/update-translations.py | 66 +++++++++++++++++++++++++
 doc/translation_process.md              |  7 +--
 3 files changed, 104 insertions(+), 4 deletions(-)
 create mode 100755 contrib/devtools/update-translations.py

diff --git a/contrib/devtools/README.md b/contrib/devtools/README.md
index f0d25fd7a..0a4e2aa39 100644
--- a/contrib/devtools/README.md
+++ b/contrib/devtools/README.md
@@ -46,4 +46,37 @@ For example a file changed in 2014 (with 2014 being the current year):
 ```// Copyright (c) 2009-2013 The Bitcoin developers```
 
 would be changed to:
-```// Copyright (c) 2009-2014 The Bitcoin developers```
\ No newline at end of file
+```// Copyright (c) 2009-2014 The Bitcoin developers```
+
+symbol-check.py
+==================
+
+A script to check that the (Linux) executables produced by gitian only contain
+allowed gcc, glibc and libstdc++ version symbols.  This makes sure they are
+still compatible with the minimum supported Linux distribution versions.
+
+Example usage after a gitian build:
+
+    find ../gitian-builder/build -type f -executable | xargs python contrib/devtools/symbol-check.py 
+
+If only supported symbols are used the return value will be 0 and the output will be empty.
+
+If there are 'unsupported' symbols, the return value will be 1 a list like this will be printed:
+
+    .../64/test_bitcoin: symbol memcpy from unsupported version GLIBC_2.14
+    .../64/test_bitcoin: symbol __fdelt_chk from unsupported version GLIBC_2.15
+    .../64/test_bitcoin: symbol std::out_of_range::~out_of_range() from unsupported version GLIBCXX_3.4.15
+    .../64/test_bitcoin: symbol _ZNSt8__detail15_List_nod from unsupported version GLIBCXX_3.4.15
+
+update-translations.py
+=======================
+
+Run this script from the root of the repository to update all translations from transifex.
+It will do the following automatically:
+
+- fetch all translations
+- post-process them into valid and committable format
+- add missing translations to the build system (TODO)
+
+See doc/translation-process.md for more information.
+
diff --git a/contrib/devtools/update-translations.py b/contrib/devtools/update-translations.py
new file mode 100755
index 000000000..1950a4267
--- /dev/null
+++ b/contrib/devtools/update-translations.py
@@ -0,0 +1,66 @@
+#!/usr/bin/python
+# Copyright (c) 2014 Wladimir J. van der Laan
+# Distributed under the MIT/X11 software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+'''
+Run this script from the root of the repository to update all translations from
+transifex.
+It will do the following automatically:
+
+- fetch all translations using the tx tool
+- post-process them into valid and committable format
+  - remove invalid control characters
+  - remove location tags (makes diffs less noisy)
+
+TODO:
+- auto-add new translations to the build system according to the translation process
+- remove 'unfinished' translation items
+'''
+from __future__ import division, print_function
+import subprocess
+import re
+import sys
+import os
+
+# Name of transifex tool
+TX = 'tx'
+# Name of source language file
+SOURCE_LANG = 'bitcoin_en.ts'
+# Directory with locale files
+LOCALE_DIR = 'src/qt/locale'
+
+def check_at_repository_root():
+    if not os.path.exists('.git'):
+        print('No .git directory found')
+        print('Execute this script at the root of the repository', file=sys.stderr)
+        exit(1)
+
+def fetch_all_translations():
+    if subprocess.call([TX, 'pull', '-f']):
+        print('Error while fetching translations', file=sys.stderr)
+        exit(1)
+
+def postprocess_translations():
+    print('Postprocessing...')
+    for filename in os.listdir(LOCALE_DIR):
+        # process only language files, and do not process source language
+        if not filename.endswith('.ts') or filename == SOURCE_LANG: 
+            continue
+        filepath = os.path.join(LOCALE_DIR, filename)
+        with open(filepath, 'rb') as f:
+            data = f.read()
+        # remove non-allowed control characters
+        data = re.sub('[\x00-\x09\x0b\x0c\x0e-\x1f]', '', data)
+        data = data.split('\n')
+        # strip locations from non-origin translation
+        # location tags are used to guide translators, they are not necessary for compilation
+        # TODO: actually process XML instead of relying on Transifex's one-tag-per-line output format
+        data = [line for line in data if not '<location' in line]
+        with open(filepath, 'wb') as f:
+            f.write('\n'.join(data))
+
+if __name__ == '__main__':
+    check_at_repository_root()
+    fetch_all_translations()
+    postprocess_translations()
+
diff --git a/doc/translation_process.md b/doc/translation_process.md
index 20fdf6ddc..912c1af27 100644
--- a/doc/translation_process.md
+++ b/doc/translation_process.md
@@ -71,14 +71,15 @@ We are using https://transifex.com as a frontend for translating the client.
 https://www.transifex.com/projects/p/bitcoin/resource/tx/
 
 The "Transifex client" (see: http://support.transifex.com/customer/portal/topics/440187-transifex-client/articles)
-will help with fetching new translations from Transifex. The Transifex configuration (`.tx/config`)
+is used to fetch new translations from Transifex. The configuration for this client (`.tx/config`)
 is part of the repository.
 
-It is also possible to directly download new translations one by one from the Transifex website.
+Do not directly download translations one by one from the Transifex website, as we do a few
+postprocessing steps before committing the translations.
 
 ### Fetching new translations
 
-1. `tx pull -a`
+1. `python contrib/devtools/update-translations.py`
 2. update `src/qt/bitcoin.qrc` manually or via
    `ls src/qt/locale/*ts|xargs -n1 basename|sed 's/\(bitcoin_\(.*\)\).ts/<file alias="\2">locale\/\1.qm<\/file>/'`
 3. update `src/qt/Makefile.am` manually or via