From a8cc56a1447265f3b3d9b449ba4c847a90c44a12 Mon Sep 17 00:00:00 2001 From: Anand Doshi Date: Thu, 17 Sep 2015 15:09:43 +0530 Subject: [PATCH] [fix] Use streaming in download translations --- bench/utils.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/bench/utils.py b/bench/utils.py index ee622f0d..bbea53c5 100644 --- a/bench/utils.py +++ b/bench/utils.py @@ -547,7 +547,7 @@ def update_translations_p(args): print 'Download failed for', args[0], args[1] def download_translations_p(): - pool = multiprocessing.Pool(8) + pool = multiprocessing.Pool(4) langs = get_langs() apps = ('frappe', 'erpnext') @@ -566,7 +566,7 @@ def get_langs(): lang_file = 'apps/frappe/frappe/data/languages.txt' with open(lang_file) as f: lang_data = f.read() - langs = [line.split('\t')[0] for line in lang_data.splitlines()] + langs = [line.split()[0] for line in lang_data.splitlines()] langs.remove('en') return langs @@ -574,12 +574,18 @@ def get_langs(): def update_translations(app, lang): translations_dir = os.path.join('apps', app, app, 'translations') csv_file = os.path.join(translations_dir, lang + '.csv') - r = requests.get("https://translate.erpnext.com/files/{}-{}.csv".format(app, lang)) + url = "https://translate.erpnext.com/files/{}-{}.csv".format(app, lang) + r = requests.get(url, stream=True) r.raise_for_status() - with open(csv_file, 'wb') as f: - f.write(r.text.encode('utf-8')) - print 'downloaded for', app, lang + with open(csv_file, 'wb') as f: + for chunk in r.iter_content(chunk_size=1024): + # filter out keep-alive new chunks + if chunk: + f.write(chunk) + f.flush() + + print 'downloaded for', app, lang def print_output(p): while p.poll() is None: