Context implemenation and some test cases

lib/cli.js

@@ -26,7 +26,7 @@ function gen(sources) {
   var result;
   var lang = opts.language.toLowerCase();
   if (!opts.keyword) {
-    opts.keyword = ['gettext'];
+    opts.keyword = ['gettext', 'ngettext', 'pgettext', 'npgettext', 'dgettext', 'dngettext', 'dpgettext', 'dnpgettext'];
   }
   if (opts.keyword.indexOf('gettext') === -1) {
     // when called from the cli, gettext should always be one of the default keywords

lib/jsxgettext.js

@@ -4,13 +4,13 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
-var fs   = require('fs');
+var fs = require('fs');
 var path = require('path');
 
 var parser = require('acorn-jsx');
 var walk = require('acorn/dist/walk');
 var gettextParser = require('gettext-parser');
-var regExpEscape  = require('escape-string-regexp');
+var regExpEscape = require('escape-string-regexp');
 
 var walkBase = Object.assign({}, walk.base, {
   JSXElement: function (node, st, c) {
@@ -39,7 +39,7 @@ var walkBase = Object.assign({}, walk.base, {
     c(node.expression, st);
   },
 
-  JSXEmptyExpression: function () {}
+  JSXEmptyExpression: function () { }
 });
 
 function isStringLiteral(node) {
@@ -51,11 +51,18 @@ function isStrConcatExpr(node) {
   var right = node.right;
 
   return node.type === "BinaryExpression" && node.operator === '+' && (
-      (isStringLiteral(left) || isStrConcatExpr(left)) &&
-      (isStringLiteral(right) || isStrConcatExpr(right))
+    (isStringLiteral(left) || isStrConcatExpr(left)) &&
+    (isStringLiteral(right) || isStrConcatExpr(right))
   );
 }
 
+// check if the args (object or array of objects) are strings
+function areArgsString(args) {
+  return args && [].concat(args).every(function (arg) {
+    return arg && (isStringLiteral(arg) || isStrConcatExpr(arg));
+  });
+}
+
 function getTranslatable(node, options) {
   // must be a call expression with arguments
   if (!node.arguments)
@@ -74,7 +81,7 @@ function getTranslatable(node, options) {
     if (callee.property.name === 'call') {
       prop = callee.object.property;
       funcName = callee.object.name || prop && (prop.name || prop.value);
-      node.arguments = node.arguments.slice( 1 );  // skip context object
+      node.arguments = node.arguments.slice(1);  // skip context object
       arg = node.arguments[0];
     } else {
       funcName = callee.property.name;
@@ -84,12 +91,30 @@ function getTranslatable(node, options) {
   if (options.keyword.indexOf(funcName) === -1)
     return false;
 
-  // If the gettext function's name starts with "n" (i.e. ngettext or n_) and its first 2 arguments are strings, we regard it as a plural function
-  if (arg && funcName.substr(0, 1) === "n" && (isStrConcatExpr(arg) || isStringLiteral(arg)) && node.arguments[1] && (isStrConcatExpr(node.arguments[1]) || isStringLiteral(node.arguments[1])))
+  // Domain is not used during extraction, so ignore it
+  if (funcName.substr(0, 1) === "d") {
+    funcName = funcName.substr(1, funcName.length);
+    node.arguments.splice(0, 1);
+    arg = node.arguments[0];
+  }
+
+  // Always return array of context and translatables i.e. [context, textToTranslate1, textToTranslate2 ...]
+  // If the gettext function's name starts with "np" (i.e. npgettext or np_) and its 3 arguments are strings and last argument is a number, we regard it as context
+  // npgettext is mentioned in https://www.gnu.org/software/gettext/manual/gettext.html#Language-specific-options
+  if (funcName.substr(0, 2) === "np" && areArgsString(node.arguments.slice(0, 3)))
+    return [arg, node.arguments[1], node.arguments[2]];
+
+  // If the gettext function's name starts with "n" (i.e. ngettext or n_) and its first 2 arguments are strings and last argument is a number, we regard it as a plural function
+  if (funcName.substr(0, 1) === "n" && areArgsString(node.arguments.slice(0, 2)))
+    return [null, arg, node.arguments[1]];
+
+  // If the gettext function's name starts with "p" (i.e. pgettext or p_) and its 2 arguments are strings, we regard it as context 
+  if (funcName.substr(0, 1) === "p" && areArgsString(node.arguments.slice(1)))
     return [arg, node.arguments[1]];
 
-  if (arg && (isStrConcatExpr(arg) || isStringLiteral(arg)))
-    return arg;
+  // else it is gettext if its 1st argument is string
+  if (areArgsString(node.arguments[0]))
+    return [null, arg];
 
   if (options.sanity)
     throw new Error("Could not parse translatable: " + JSON.stringify(arg, null, 2));
@@ -132,40 +157,39 @@ function parse(sources, options) {
     poJSON = {
       charset: "utf-8",
       headers: headers,
-      translations: {'': {'': {comments: {flag: 'fuzzy'}}} }
+      translations: { '': { '': { comments: { flag: 'fuzzy' } } } }
     };
   }
 
   var translations;
 
   try {
     poJSON.headers["pot-creation-date"] = new Date().toISOString().replace('T', ' ').replace(/:\d{2}.\d{3}Z/, '+0000');
-
-    // Always use the default context for now
-    // TODO: Take into account different contexts
-    translations = poJSON.translations[''];
+    translations = poJSON.translations;
   } catch (err) {
     if (useExisting)
       throw new Error("An error occurred while using the provided PO file. Please make sure it is valid by using `msgfmt -c`.");
     else
       throw err;
   }
 
-  if( options.keyword ) {
+  if (options.keyword) {
     Object.keys(options.keyword).forEach(function (index) {
-      options.keyword.push('n' + options.keyword[index]);
+      ['n', 'p', 'np', 'd', 'dn', 'dp', 'dnp'].forEach(function (keyword) {
+        options.keyword.push(keyword + options.keyword[index]);
+      });
     });
   }
   else {
-    options.keyword = ['gettext', 'ngettext'];
+    options.keyword = ['gettext', 'ngettext', 'pgettext', 'npgettext', 'dgettext', 'dngettext', 'dpgettext', 'dnpgettext'];
   }
   var tagName = options.addComments || "L10n:";
   var commentRegex = new RegExp([
     "^\\s*" + regExpEscape(tagName), // The "TAG" provided externally or "L10n:" by default
     "^\\/" // The "///" style comments which is the xgettext standard
   ].join("|"));
   Object.keys(sources).forEach(function (filename) {
-    var source   = sources[filename].replace(/^#.*/, ''); // strip leading hash-bang
+    var source = sources[filename].replace(/^#.*/, ''); // strip leading hash-bang
     var astComments = [];
     var parserOptions = Object.assign({}, {
       ecmaVersion: 6,
@@ -178,13 +202,13 @@ function parse(sources, options) {
           return;
 
         astComments.push({
-          line : line,
+          line: line,
           value: text
         });
       },
       locations: true
     }, options.parserOptions && JSON.parse(options.parserOptions));
-    var ast      = parser.parse(source, parserOptions);
+    var ast = parser.parse(source, parserOptions);
 
     // finds comments that end on the previous line
     function findComments(comments, line) {
@@ -196,38 +220,48 @@ function parse(sources, options) {
       }).filter(Boolean).join('\n');
     }
 
-     walk.simple(ast, {'CallExpression': function (node) {
-        var arg = getTranslatable(node, options);
-        if (!arg)
+    walk.simple(ast, {
+      'CallExpression': function (node) {
+        var args = getTranslatable(node, options);
+        if (!args)
           return;
-
-        var msgid = arg;
-        if( arg.constructor === Array )
-            msgid = arg[0];
+        var msgCtxt = args[0] ? extractStr(args[0]) : "";
+        var msgid = args[1];
         var str = extractStr(msgid);
         var line = node.loc.start.line;
         var comments = findComments(astComments, line);
 
         var ref = filename + ':' + line;
-        if (!translations[str]) {
-          translations[str] = {
+        if (!translations[msgCtxt])
+          translations[msgCtxt] = {};
+        if (!translations[msgCtxt][str]) {
+          translations[msgCtxt][str] = {
             msgid: str,
             msgstr: [],
             comments: {
               extracted: comments,
               reference: ref
             }
           };
-          if( arg.constructor === Array ) {
-              translations[str].msgid_plural = extractStr(arg[1]);
-              translations[str].msgstr = ['', ''];
+          // if it's npgettext
+          if (args.length === 3 && args[0]) {
+            translations[msgCtxt][str].msgctxt = msgCtxt;
+            translations[msgCtxt][str].msgid_plural = extractStr(args[2]);
+            translations[msgCtxt][str].msgstr = ['', ''];
+          } else if (args.length === 3 && !args[0]) {
+            // if it's ngettext
+            translations[msgCtxt][str].msgid_plural = extractStr(args[2]);
+            translations[msgCtxt][str].msgstr = ['', ''];
+          } else if (args.length === 2 && args[0]) {
+            // if it's pgettext
+            translations[msgCtxt][str].msgctxt = msgCtxt;
+            translations[msgCtxt][str].msgstr = ['', ''];
           }
         } else {
-          if(translations[str].comments) {
-            translations[str].comments.reference +=  '\n' + ref;
-          }
+          if (translations[msgCtxt][str].comments)
+            translations[msgCtxt][str].comments.reference += '\n' + ref;
           if (comments)
-            translations[str].comments.extracted += '\n' + comments;
+            translations[msgCtxt][str].comments.extracted += '\n' + comments;
         }
       }
     }, walkBase);
@@ -237,16 +271,18 @@ function parse(sources, options) {
       return item && arr.indexOf(item) === i;
     }
 
-    Object.keys(translations).forEach(function (msgid) {
-      var comments = translations[msgid].comments;
-
+    function extractComments(msgctxt, msgid) {
+      var comments = translations[msgctxt][msgid].comments;
       if (!comments)
         return;
-
       if (comments.reference)
         comments.reference = comments.reference.split('\n').filter(dedupeNCoalesce).join('\n');
       if (comments.extracted)
         comments.extracted = comments.extracted.split('\n').filter(dedupeNCoalesce).join('\n');
+    }
+
+    Object.keys(translations).forEach(function (msgctxt) {
+      Object.keys(translations[msgctxt]).forEach(extractComments.bind(null, msgctxt));
     });
   });
 

test/inputs/anonymous_functions.js

@@ -9,3 +9,7 @@ var testObj = {
 testObj.somemethod('I shall not pass');
 testObj.gettext("I'm gonna get translated, yay!");
 testObj.ngettext("I'm also gonna get translated!", "I'm the plural form!", 2);
+testObj.pgettext("context1", "I am translated in context!");
+testObj.pgettext("context2", "I am translated in context!");
+testObj.npgettext("context3", "I am also translated in context!", "I'm the plural form!", 2);
+testObj.npgettext("context4", "I am also translated in context!", "I'm the plural form!", 2);

test/inputs/example.swig

@@ -8,7 +8,7 @@ foobar
 
 {% set foobar = gettext("Test gettext directly") %}
 {% set foobar = gettext("Test with additional params on new line", {
-	'foo': 'bar'
+  'foo': 'bar'
 }) %}
 
 {% blocktrans with name=name url=url %}

test/inputs/filter.ejs

@@ -1,2 +1,7 @@
 <%=: gettext("this is a localizable string") | capitalize %>
 <%=: ngettext("this is a localizable singular string", "this is a localizable plural string", 2) | capitalize %>
+<%=: pgettext("context_1","this is a localizable string in context") | capitalize %>
+<%=: npgettext("context_1", "this is a localizable singular string in context", "this is a localizable plural string in context", 2) | capitalize %>
+<%=: pgettext("context_2","this is a localizable string in context") | capitalize %>
+<%=: npgettext("context_2", "this is a localizable singular string in context", "this is a localizable plural string in context", 2) | capitalize %>
+

test/inputs/include.ejs

@@ -1,3 +1,12 @@
 <% include this/include/syntax/is/kinda/dumb %>
+<%# gettext("this is a non localizable comment string") %>
 <%= gettext("this is a localizable string") %>
 <%= ngettext("this is a localizable singular string", "this is a localizable plural string", 2) %>
+<%= pgettext("context_1","this is a localizable string in context") %>
+<%= npgettext("context_1", "this is a localizable singular string in context", "this is a localizable plural string in context", 2) %>
+<%= pgettext("context_2","this is a localizable string in context") %>
+<%= npgettext("context_2", "this is a localizable singular string in context", "this is a localizable plural string in context", 2) %>
+<%= dgettext("domain", "this is a localizable string in domain") %>
+<%= dngettext("domain", "this is a localizable singular string in domain", "this is a localizable plural string in domain", 2) %>
+<%= dpgettext("domain", "context", "this is a localizable string in domain and context") %>
+<%= dnpgettext("domain", "context", "this is a localizable singular string in domain and context", "this is a localizable plural string in domain and context", 2) %>

test/inputs/raw.ejs

@@ -1,3 +1,9 @@
 <%== gettext("this is a raw localizable string") %>
 <%== gettext("this is a raw localizable string") %>
 <%== ngettext("this is a raw localizable singular string", "this is a raw localizable plural string", 2) %>
+<%=: pgettext("context_1","this is a localizable string in context") %>
+<%=: npgettext("context_1", "this is a localizable singular string in context", "this is a localizable plural string in context", 2) %>
+<%=: pgettext("context_2","this is a localizable string in context") %>
+<%=: npgettext("context_2", "this is a localizable singular string in context", "this is a localizable plural string in context", 2) %>
+
+

test/outputs/anonymous_functions.pot

@@ -6,4 +6,28 @@ msgstr ""
 msgid "I'm also gonna get translated!"
 msgid_plural "I'm the plural form!"
 msgstr[0] ""
+msgstr[1] ""
+
+#: inputs/anonymous_functions.js:12
+msgctxt "context1"
+msgid "I am translated in context!"
+msgstr ""
+
+#: inputs/anonymous_functions.js:13
+msgctxt "context2"
+msgid "I am translated in context!"
+msgstr ""
+
+#: inputs/anonymous_functions.js:14
+msgctxt "context3"
+msgid "I am also translated in context!"
+msgid_plural "I'm the plural form!"
+msgstr[0] ""
+msgstr[1] ""
+
+#: inputs/anonymous_functions.js:15
+msgctxt "context4"
+msgid "I am also translated in context!"
+msgid_plural "I'm the plural form!"
+msgstr[0] ""
 msgstr[1] ""

test/tests/ejs.js

@@ -2,6 +2,7 @@
 
 var fs = require('fs');
 var path = require('path');
+var utils = require('../utils');
 
 var jsxgettext = require('../../lib/jsxgettext');
 var ejs = require('../../lib/parsers/ejs').ejs;
@@ -20,6 +21,15 @@ exports['test ejs'] = function (assert, cb) {
               'localizable strings are extracted');
     assert.ok(result.indexOf('this is a localizable plural string') !== -1,
               'localizable plural strings are extracted');
+    var singleLineResult = utils.getSingleLineString(result);
+    assert.ok(singleLineResult.indexOf('msgctxt "context_1"msgid "this is a localizable string in context"') !== -1, 'localizable string in context 1 are extracted');    
+    assert.ok(singleLineResult.indexOf('msgctxt "context_1"msgid "this is a localizable singular string in context"msgid_plural "this is a localizable plural string in context"') !== -1, 'localizable plural strings in context 1 are extracted');
+    assert.ok(singleLineResult.indexOf('msgctxt "context_2"msgid "this is a localizable string in context"') !== -1, 'localizable string in context 2 are extracted');
+    assert.ok(singleLineResult.indexOf('msgctxt "context_1"msgid "this is a localizable singular string in context"msgid_plural "this is a localizable plural string in context"') !== -1, 'localizable plural strings in context 2 are extracted');
+    assert.ok(singleLineResult.indexOf('this is a localizable string in domain') !== -1, 'localizable string in domain are extracted');
+    assert.ok(singleLineResult.indexOf('this is a localizable singular string in domain') !== -1, 'localizable plural strings in domain are extracted');
+    assert.ok(singleLineResult.indexOf('msgctxt "context"msgid "this is a localizable string in domain and context"') !== -1, 'localizable string in domain and context are extracted');
+    assert.ok(singleLineResult.indexOf('msgctxt "context"msgid "this is a localizable singular string in domain and context"') !== -1, 'localizable plural strings in domain and context are extracted');
     cb();
   });
 };

test/tests/ejs_filter.js

@@ -2,6 +2,7 @@
 
 var fs = require('fs');
 var path = require('path');
+var utils = require('../utils');
 
 var jsxgettext = require('../../lib/jsxgettext');
 var ejs = require('../../lib/parsers/ejs').ejs;
@@ -20,6 +21,11 @@ exports['test ejs'] = function (assert, cb) {
               'localizable strings are extracted');
     assert.ok(result.indexOf('this is a localizable plural string') !== -1,
               'localizable plural strings are extracted');
+    var singleLineResult = utils.getSingleLineString(result);
+    assert.ok(singleLineResult.indexOf('msgctxt "context_1"msgid "this is a localizable string in context"') !== -1, 'localizable string in context 1 are extracted');    
+    assert.ok(singleLineResult.indexOf('msgctxt "context_1"msgid "this is a localizable singular string in context"msgid_plural "this is a localizable plural string in context"') !== -1, 'localizable plural strings in context 1 are extracted');
+    assert.ok(singleLineResult.indexOf('msgctxt "context_2"msgid "this is a localizable string in context"') !== -1, 'localizable string in context 2 are extracted');
+    assert.ok(singleLineResult.indexOf('msgctxt "context_1"msgid "this is a localizable singular string in context"msgid_plural "this is a localizable plural string in context"') !== -1, 'localizable plural strings in context 2 are extracted');
     cb();
   });
 };

test/tests/ejs_raw.js

@@ -2,6 +2,7 @@
 
 var fs = require('fs');
 var path = require('path');
+var utils = require('../utils');
 
 var jsxgettext = require('../../lib/jsxgettext');
 var ejs = require('../../lib/parsers/ejs').ejs;
@@ -20,6 +21,11 @@ exports['test ejs'] = function (assert, cb) {
               'raw localizable strings are extracted');
     assert.ok(result.indexOf('this is a raw localizable plural string') !== -1,
               'raw localizable plural strings are extracted');
+              var singleLineResult = utils.getSingleLineString(result);
+    assert.ok(singleLineResult.indexOf('msgctxt "context_1"msgid "this is a localizable string in context"') !== -1, 'localizable string in context 1 are extracted');    
+    assert.ok(singleLineResult.indexOf('msgctxt "context_1"msgid "this is a localizable singular string in context"msgid_plural "this is a localizable plural string in context"') !== -1, 'localizable plural strings in context 1 are extracted');
+    assert.ok(singleLineResult.indexOf('msgctxt "context_2"msgid "this is a localizable string in context"') !== -1, 'localizable string in context 2 are extracted');
+    assert.ok(singleLineResult.indexOf('msgctxt "context_1"msgid "this is a localizable singular string in context"msgid_plural "this is a localizable plural string in context"') !== -1, 'localizable plural strings in context 2 are extracted');
     cb();
   });
 };