bug 985: move from Translingo to Pootle (translations.cacert.org)

- new Makefile
- rework translation selection code
- remove unused files

Signed-off-by: Michael Tänzer <neo@nhng.de>

7 files changed, 274 insertions, 184 deletions

diff --git a/Makefile b/Makefile
deleted file mode 100644
index 914d979..0000000
--- a/Makefile
+++ /dev/null
@@ -1,7 +0,0 @@
-all:
-	xgettext -s -o messages.po --no-wrap --foreign-user includes/*.php www/*.php pages/account/*.php pages/index/*.php pages/wot/*.php pages/gpg/*.php pages/disputes/*.php pages/help/*.php pages/disputes/*.php scripts/removedead.php
-	perl cacertupload.pl
-	cd locale; php make.php
-
-other: all
-	cat messages.po|sed "s/CHARSET/iso-8859-1/"|sed "s/PACKAGE VERSION/CAcert/"|sed "s/This file is put in the public domain./This file is distributed under the same license as the CAcert package./"|sed "s/# SOME DESCRIPTIVE TITLE.//" > messages.po
diff --git a/cacertupload.pl b/cacertupload.pl
deleted file mode 100755
index 991570b..0000000
--- a/cacertupload.pl
+++ /dev/null
@@ -1,59 +0,0 @@
-#!/usr/bin/perl
-
-#LibreSSL - CAcert web application
-#Copyright (C) 2004-2008  CAcert Inc.
-#
-#This program is free software; you can redistribute it and/or modify
-#it under the terms of the GNU General Public License as published by
-#the Free Software Foundation; version 2 of the License.
-#
-#This program is distributed in the hope that it will be useful,
-#but WITHOUT ANY WARRANTY; without even the implied warranty of
-#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#GNU General Public License for more details.
-#
-#You should have received a copy of the GNU General Public License
-#along with this program; if not, write to the Free Software
-#Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
-
-use LWP::UserAgent;
-$ua = LWP::UserAgent->new(agent => 'Translingo Client 1.0');
-use HTTP::Request::Common qw(POST);
-
-my $translingo_password;
-my $translingo_account;
-
-# Read Account&Password from file
-eval `cat password.dat`;
-
-$ua->cookie_jar({});
-$ua->timeout(10000);
-
-my $req = POST 'http://translingo.cacert.org/login.php',
-[
-];
-# ggf. Referer faken
-$req->referer('http://translingo.cacert.org/');
- $ua->request($req)->as_string;
-
-# 1.Test - Umgebung
-my $req = POST 'http://translingo.cacert.org/login.php',
-[
- username => $translingo_account,
- password => $translingo_password,
- submit => 'Login',
-];
-# ggf. Referer faken
-$req->referer('http://translingo.cacert.org/');
-$ua->request($req)->as_string;
-
-# 2.Test - FileUpload
-my $req = POST 'http://translingo.cacert.org/upload.php',
-Content_Type => 'form-data',
-Content => [
- project => '1',
- fileformat => '1',
- pofile   => ["messages.po" => "messages.po", 'Content_Type' => "application/x-gettext"],
-];
-print $ua->request($req)->as_string;
-
diff --git a/includes/general.php b/includes/general.php
index ebdf20e..0231b21 100644
--- a/includes/general.php
+++ b/includes/general.php
@@ -70,50 +70,44 @@
 		}
 	}
 
-	$lang = "";
-	if(array_key_exists("lang",$_REQUEST))
-	  $lang=mysql_escape_string(substr(trim($_REQUEST['lang']), 0, 5));
-	if($lang != "")
-		$_SESSION['_config']['language'] = $lang;
-
-	//if($_SESSION['profile']['id'] == 1 && 1 == 2)
-	//	echo $_SESSION['_config']['language'];
-
+	
+	///////////////////////
+	// Language Handling //
+	///////////////////////
+	
 	$_SESSION['_config']['translations'] = array(
-				"ar_JO" => "العربية",
-				"bg_BG" => "Български",
-				"cs_CZ" => "Čeština",
-				"da_DK" => "Dansk",
-				"de_DE" => "Deutsch",
-				"el_GR" => "Ελληνικά",
-				"en_AU" => "English",
-				"eo_EO" => "Esperanto",
-				"es_ES" => "Español",
-				"fa_IR" => "Farsi",
-				"fi_FI" => "Suomi",
-				"fr_FR" => "Français",
-				"he_IL" => "עברית",
-				"hr_HR" => "Hrvatski",
-				"hu_HU" => "Magyar",
-				"is_IS" => "Íslenska",
-				"it_IT" => "Italiano",
-				"ja_JP" => "日本語",
-				"ka_GE" => "Georgian",
-				"nl_NL" => "Nederlands",
-				"pl_PL" => "Polski",
-				"pt_PT" => "Português",
-				"pt_BR" => "Português Brasileiro",
-				"ru_RU" => "Русский",
-				"ro_RO" => "Română",
-				"sv_SE" => "Svenska",
-				"tr_TR" => "Türkçe",
-				"zh_CN" => "中文(简体)");
-
-        $value=array();
+				"ar" => "العربية",
+				"bg" => "Български",
+				"cs" => "Čeština",
+				"da" => "Dansk",
+				"de" => "Deutsch",
+				"el" => "Ελληνικά",
+				"en" => "English",
+				"es" => "Español",
+				"fi" => "Suomi",
+				"fr" => "Français",
+				"hu" => "Magyar",
+				"it" => "Italiano",
+				"ja" => "日本語",
+				"lv" => "Latviešu",
+				"nl" => "Nederlands",
+				"pl" => "Polski",
+				"pt" => "Português",
+				"pt-br" => "Português Brasileiro",
+				"ru" => "Русский",
+				"sv" => "Svenska",
+				"tr" => "Türkçe",
+				"zh-cn" => "中文(简体)",
+				"zh-tw" => "中文(臺灣)");
+
+	
 
 	if(!(array_key_exists('language',$_SESSION['_config']) && $_SESSION['_config']['language'] != ""))
 	{
-		$bits = explode(",", strtolower(str_replace(" ", "", mysql_real_escape_string(array_key_exists('HTTP_ACCEPT_LANGUAGE',$_SERVER)?$_SERVER['HTTP_ACCEPT_LANGUAGE']:""))));
+		$languages=array();
+		
+		// parse Accept-Language header
+		$bits = explode(",", strtolower(str_replace(" ", "", array_key_exists('HTTP_ACCEPT_LANGUAGE',$_SERVER)?$_SERVER['HTTP_ACCEPT_LANGUAGE']:"")));
 		foreach($bits as $lang)
 		{
 			$b = explode(";", $lang);
@@ -121,59 +115,79 @@
 				$c = floatval(substr($b[1], 2));
 			else
 				$c = 1;
-			$value["$c"] = trim($b[0]);
+			
+			if ($c != 0)
+			{
+				$languages["$c"] = trim($b[0]);
+			}
 		}
+		
+		// check if there is an explicit language given as parameter
+		if(array_key_exists("lang",$_REQUEST) && trim($_REQUEST["lang"]) != "")
+		{
+			// higher priority as those values in the header
+			$languages["2.0"] = strtolower(trim($_REQUEST["lang"]));
+		}
+		
+		// fallback, qvalues may only have three digits after the .
+		$languages["0.0005"] = "en";
+		
+		// remove any non-conforming values (that's why we don't need to
+		// mysql_real_escape() or escapeshellarg() but take care of the *)
+		// spec: ( ( 1*8ALPHA *( "-" 1*8ALPHA ) ) | "*" )
+		$languages = preg_filter('/^(?:[a-zA-Z]{1,8}(?:-[a-zA-Z]{1,8})*|\*)$/',
+				'$0', $languages);
 
-		krsort($value);
-
-		reset($value);
+		krsort($languages, SORT_NUMERIC);
 
-		foreach($value as $key => $val)
+		$all_translations = scandir($_SESSION['_config']['filepath']."/locale/");
+		foreach($languages as $qvalue => $lang)
 		{
-			$val = substr(escapeshellarg($val), 1, -1);
-			$short = substr($val, 0, 2);
-			if($val == "en" || $short == "en")
+			$lang = strtr($lang, '-', '_');
+			$lang_length = strlen($lang);
+			$chosen_translation = "";
+			foreach ($all_translations as $translation)
 			{
-				$_SESSION['_config']['language'] = "en";
-				break;
-			}
-			if(file_exists($_SESSION['_config']['filepath']."/locale/$val/LC_MESSAGES/messages.mo"))
-			{
-				$_SESSION['_config']['language'] = $val;
-				break;
+				$translation_lower = strtolower($translation);
+				if ($translation_lower === $lang ||
+						substr($translation, 0, $lang_length + 1) === $lang.'_')
+				{
+					$chosen_translation = $translation;
+					break;
+				}
 			}
-			if(file_exists($_SESSION['_config']['filepath']."/locale/$short/LC_MESSAGES/messages.mo"))
+			if ($chosen_translation !== "")
 			{
-				$_SESSION['_config']['language'] = $short;
-				break;
+				if(file_exists($_SESSION['_config']['filepath']."/locale/$chosen_translation/LC_MESSAGES/messages.mo"))
+				{
+					$_SESSION['_config']['language'] = $lang;
+					break;
+				}
 			}
 		}
 	}
-	if(!array_key_exists('_config',$_SESSION) || !array_key_exists('language',$_SESSION['_config']) || strlen($_SESSION['_config']['language']) != 5)
+	if(!array_key_exists('_config',$_SESSION) ||
+			!array_key_exists('language',$_SESSION['_config']) ||
+			$_SESSION['_config']['language'] == "" )
 	{
-		$lang = array_key_exists('language',$_SESSION['_config'])?$_SESSION['_config']['language']:"";
-		$_SESSION['_config']['language'] = "en_AU";
-		foreach($_SESSION['_config']['translations'] as $key => $val)
-		{
-			if(substr($lang, 0, 2) == substr($key, 0, 2))
-			{
-				$_SESSION['_config']['language'] = $val;
-				break;
-			}
-		}
+		$_SESSION['_config']['language'] = "en";
+		trigger_error('$_SESSION[\'_config\'][\'language\'] was not set after '.
+				'the translation search', E_USER_WARNING);
 	}
 
 	$_SESSION['_config']['recode'] = "html..latin-1";
-	if($_SESSION['_config']['language'] == "zh_CN")
+	if($_SESSION['_config']['language'] === "zh_CN" ||
+			$_SESSION['_config']['language'] === "zh_TW")
 	{
 		$_SESSION['_config']['recode'] = "html..gb2312";
-	} else if($_SESSION['_config']['language'] == "pl_PL" || $_SESSION['_config']['language'] == "hu_HU") {
+	} else if($_SESSION['_config']['language'] == "pl" ||
+			$_SESSION['_config']['language'] == "hu") {
 		$_SESSION['_config']['recode'] = "html..ISO-8859-2";
-	} else if($_SESSION['_config']['language'] == "ja_JP") {
+	} else if($_SESSION['_config']['language'] == "ja") {
 		$_SESSION['_config']['recode'] = "html..SHIFT-JIS";
-	} else if($_SESSION['_config']['language'] == "ru_RU") {
+	} else if($_SESSION['_config']['language'] == "ru") {
 		$_SESSION['_config']['recode'] = "html..ISO-8859-5";
-	} else if($_SESSION['_config']['language'] == "lt_LT") {
+	} else if($_SESSION['_config']['language'] == "lt") {
 		$_SESSION['_config']['recode'] = "html..ISO-8859-13";
 	}
 
@@ -183,10 +197,6 @@
 	bindtextdomain($domain, $_SESSION['_config']['filepath']."/locale");
 	textdomain($domain);
 
-	//if($_SESSION['profile']['id'] == -1)
-	//	echo $_SESSION['_config']['language']." - ".$_SESSION['_config']['filepath']."/locale";
-
-
         if(array_key_exists('profile',$_SESSION) && is_array($_SESSION['profile']) && array_key_exists('id',$_SESSION['profile']) && $_SESSION['profile']['id'] > 0)
 	{
 		$locked = mysql_fetch_assoc(mysql_query("select `locked` from `users` where `id`='".$_SESSION['profile']['id']."'"));
diff --git a/locale/.gitignore b/locale/.gitignore
index 94fd7c6..ac93223 100644
--- a/locale/.gitignore
+++ b/locale/.gitignore
@@ -1,5 +1,6 @@
 # Language files are imported from translingo
 # => Ignore them
-# Use make.php if you need new ones
+# Use make if you need new ones
 *.po
+*.pot
 *.mo
diff --git a/locale/Makefile b/locale/Makefile
new file mode 100644
index 0000000..b703fb2
--- /dev/null
+++ b/locale/Makefile
@@ -0,0 +1,184 @@
+#
+# This Makefile will download the translations from our translation server (if
+# they don't exist yet) and compile them. Try target help for more information
+#
+
+################################################################################
+###                                 Download                                 ###
+################################################################################
+
+DOWNLOAD_SERVER := translations.cacert.org
+PO_URL_TEMPLATE := http://$(DOWNLOAD_SERVER)/export/cacert/%/messages.po
+
+# Only use languages that have 10% or more of translated strings
+AUTO_LANGS := \
+en \
+de \
+nl \
+pt_BR \
+fr \
+sv \
+it \
+es \
+hu \
+fi \
+ja \
+bg \
+pt \
+da \
+pl \
+zh_CN \
+ru \
+lv \
+cs \
+zh_TW \
+el \
+tr \
+ar \
+
+LANGS := \
+ar \
+bg \
+cs \
+da \
+de \
+el \
+en \
+es \
+fa \
+fi \
+fr \
+he \
+hr \
+hu \
+id \
+is \
+it \
+ja \
+ka \
+ko \
+lv \
+nb \
+nl \
+pl \
+pt \
+pt_BR \
+ro \
+ru \
+sl \
+sv \
+th \
+tr \
+uk \
+zh_CN \
+zh_TW \
+
+
+PO_FILE_TEMPLATE := %/messages.po
+MO_FILE_TEMPLATE := %/LC_MESSAGES/messages.mo
+
+
+# target: all - Build locales downloading po files
+.PHONY: all
+all: $(AUTO_LANGS)
+
+
+# target: help - Display callable targets
+.PHONY: help
+help:
+	@egrep "^# target:" [Mm]akefile
+
+
+# target: clean - remove the build directories
+RM := rm -rf
+.PHONY: clean
+clean:
+	-$(RM) $(LANGS:%=%/)
+
+
+# target: <lang> - build this particular language
+.PHONY: $(LANGS)
+$(LANGS): %: $(MO_FILE_TEMPLATE)
+
+
+$(LANGS:%=$(MO_FILE_TEMPLATE)): $(MO_FILE_TEMPLATE): $(PO_FILE_TEMPLATE)
+	mkdir -p $(@D)
+#filter obsolete translations
+	grep --invert-match '^#~ ' $< | \
+		msgfmt --check --output-file $@ -
+
+
+.PHONY: $(LANGS:%=$(PO_FILE_TEMPLATE))
+$(LANGS:%=$(PO_FILE_TEMPLATE)):
+	mkdir -p $(@D)
+	wget --output-document - '$(@:$(PO_FILE_TEMPLATE)=$(PO_URL_TEMPLATE))' | \
+		# convert UTF-8 characters to HTML entities \
+		php -r 'while (!feof(STDIN)) echo mb_convert_encoding(fgets(STDIN), "HTML-ENTITIES", "UTF-8");' \
+		> $@
+
+
+
+
+################################################################################
+###                                 Upload                                   ###
+################################################################################
+
+UPLOAD_SERVER := $(DOWNLOAD_SERVER)
+SSH_USER := critical
+SSH_OPTIONS :=
+SCP_OPTIONS := $(SSH_OPTIONS)
+
+FILE_OWNER := www-data
+
+POT_UPLOAD_PATH := /var/www/Pootle/po/cacert/templates/messages.pot
+MANAGE_PY := /var/www/Pootle/manage.py
+
+VERSION := Production
+DESCRITPION := LibreSSL - CAcert web application (localisation files)
+COPYRIGHT_YEAR := 2004-$(shell date +\%Y)
+PACKAGE := LibreSSL
+
+GETTEXT_FILE_PATTERN := \
+../CommModule/client.pl \
+../includes/*.php \
+../includes/*/*.php \
+../pages/*/*.php \
+../scripts/*.php \
+../www/*.php \
+../www/*/*.php \
+# ../tverify/*.php \
+# ../tverify/*/*.php \
+
+GETTEXT_FILES := $(wildcard $(GETTEXT_FILE_PATTERN))
+
+# target: template - create the gettext template file, if you want to upload it
+# target:            onto the translation server you can directly use the
+# target:            target "upload"
+.PHONY: template
+template: messages.pot
+
+# target: template.clean - remove anything that was created during the build of
+# target:                  the template file
+.PHONY: template.clean
+template.clean:
+	-$(RM) messages.pot
+
+
+# target: upload - upload the template to the translation server
+.PHONY: upload
+upload: messages.pot
+	scp $(SCP_OPTIONS) messages.pot $(SSH_USER)@$(UPLOAD_SERVER):$(POT_UPLOAD_PATH)
+	ssh $(SSH_OPTIONS) $(SSH_USER)@$(UPLOAD_SERVER) "sudo -u $(FILE_OWNER) pootle-update cacert"
+
+# target: upload.clean - remove anything that was created during the upload
+.PHONY: upload.clean
+upload.clean: template.clean
+
+messages.pot: $(GETTEXT_FILES)
+	xgettext --output - --sort-by-file --copyright-holder "CAcert Inc." \
+		--package-name "CAcert" --package-version "$(VERSION)" \
+		--msgid-bugs-address "translations-admin@cacert.org" $^ | \
+		# replace place holders in the lines before the first msgid\
+		sed '1,/^msgid/ { s/SOME DESCRIPTIVE TITLE/$(DESCRITPION)/; s/YEAR/$(COPYRIGHT_YEAR)/; s/PACKAGE/$(PACKAGE)/ }' \
+		> $@
+
diff --git a/locale/make.php b/locale/make.php
deleted file mode 100755
index 2ffd5a0..0000000
--- a/locale/make.php
+++ /dev/null
@@ -1,37 +0,0 @@
-#!/usr/bin/php -q
-<?
-	$lang = array(  "bg" => "bg_BG", "da" => "da_DK", "de" => "de_DE", "es" => "es_ES",
-			"fr" => "fr_FR", "fi" => "fi_FI", "he" => "he_IL", "hr" => "hr_HR",
-			"hu" => "hu_HU", "id" => "id_ID", "it" => "it_IT", "ja" => "ja_JP",
-			"nl" => "nl_NL", "lv" => "lv_LV", "pt" => "pt_PT", "pt_BR" => "pt_BR",
-			"ro" => "ro_RO", "ru" => "ru_RU", "fa" => "fa_IR", "sv" => "sv_SE",
-			"tr" => "tr_TR", "zh_CN" => "zh_CN", "zh_TW" => "zh_TW", "ar" => "ar_SY",
-			"el" => "el_GR", "tl" => "tl_PH", "pl" => "pl_PL", "cs" => "cs_CZ",
-			"ka" => "ka_GE", "is" => "is_IS", "ko" => "ko_KR", "nb" => "nb_NO",
-			"sl" => "sl_SI", "th" => "th_TH", "uk" => "uk_UA");
-
-	if($argc > 1)
-	{
-		foreach($argv as $key)
-		{
-			$val = $lang[$key];
-			if($val != "")
-			{
-				echo `mkdir -p $key/LC_MESSAGES/`;
-				$do = `wget -O $key/messages.po "http://translingo.cacert.org/export2.php?pid=1&editlanguage=$val" 2>&1`;
-echo $do;
-				$do = `msgfmt -o $key/LC_MESSAGES/messages.mo $key/messages.po 2>&1`;
-echo $do;
-			}
-		}
-	} else {
-		foreach($lang as $key => $val)
-		{
-		echo `mkdir -p $key/LC_MESSAGES/`;
-		$do = `wget -O $key/messages.po "http://translingo.cacert.org/export2.php?pid=1&editlanguage=$val" 2>&1`;
-echo $do;
-		$do = `msgfmt -o $key/LC_MESSAGES/messages.mo $key/messages.po 2>&1`;
-echo $do;
-		}
-	}
-?>
diff --git a/password.dat.sample b/password.dat.sample
deleted file mode 100644
index f9bbb55..0000000
--- a/password.dat.sample
+++ /dev/null
@@ -1,2 +0,0 @@
-$translingo_password = 'ThePassword';
-$translingo_account = 'TheAccount';