From 281fc442fcff9508010bf7b37f34423681dc8b08 Mon Sep 17 00:00:00 2001 From: John Hooks Date: Sun, 12 May 2024 08:11:10 -0700 Subject: [PATCH] feat: add support for compiling big-endian .mo files This commit adds support for compiling big-endian .mo files. This facilitates the use of .mo files in big-endian systems, the ability to parser big-endian .mo files was already present. This commit also does the following: - Adds a new mo parser options type, used to indicate `endian = 'le' | 'be'` - Adds tests and fixtures for big-endian .mo files. - Renames the existing .mo fixtures to indicate that they are little-endian. - Fixes direct usages of `writeUInt32LE`. Fixes #97 --- src/mocompiler.js | 25 ++++-- test/fixtures/latin13-be.mo | Bin 0 -> 678 bytes test/fixtures/{latin13.mo => latin13-le.mo} | Bin test/fixtures/obsolete-be.mo | Bin 0 -> 125 bytes test/fixtures/{obsolete.mo => obsolete-le.mo} | Bin test/fixtures/utf8-be.mo | Bin 0 -> 851 bytes test/fixtures/{utf8.mo => utf8-le.mo} | Bin test/mo-compiler-test.js | 73 +++++++++++++++--- test/mo-parser-test.js | 34 +++++++- test/po-obsolete-test.js | 2 +- 10 files changed, 111 insertions(+), 23 deletions(-) create mode 100644 test/fixtures/latin13-be.mo rename test/fixtures/{latin13.mo => latin13-le.mo} (100%) create mode 100644 test/fixtures/obsolete-be.mo rename test/fixtures/{obsolete.mo => obsolete-le.mo} (100%) create mode 100644 test/fixtures/utf8-be.mo rename test/fixtures/{utf8.mo => utf8-le.mo} (100%) diff --git a/src/mocompiler.js b/src/mocompiler.js index f538a6d..9747101 100644 --- a/src/mocompiler.js +++ b/src/mocompiler.js @@ -21,15 +21,22 @@ import contentType from 'content-type'; * @typedef {{ msgid: Buffer, msgstr: Buffer }} TranslationBuffers A translation object partially parsed. */ +/** + * + * @typedef {Object} CompilerOptions MO compiler options + * @property {'be'|'le'} [endian='le'] Endianness of the output buffer. Default is 'le' + */ + /** * Exposes general compiler function. Takes a translation * object as a parameter and returns binary MO object * * @param {GetTextTranslations} table Translation object + * @param {CompilerOptions} [options] MO compiler options * @return {Buffer} Compiled binary MO object */ -export default function (table) { - const compiler = new Compiler(table); +export default function (table, options = { endian: 'le' }) { + const compiler = new Compiler(table, options); return compiler.compile(); } @@ -91,8 +98,9 @@ function prepareTranslations (translations) { * @this {Compiler & Transform} * * @param {GetTextTranslations} [table] Translation table as defined in the README + * @param {CompilerOptions} [options] MO compiler options */ -function Compiler (table) { +function Compiler (table, options = { endian: 'le' }) { /** @type {GetTextTranslations} _table The translation table */ this._table = { charset: undefined, @@ -101,10 +109,11 @@ function Compiler (table) { }; this._translations = []; + /** * @type {WriteFunc} */ - this._writeFunc = 'writeUInt32LE'; + this._writeFunc = options?.endian === 'le' ? 'writeUInt32LE' : 'writeUInt32BE'; this._handleCharset(); @@ -258,8 +267,8 @@ Compiler.prototype._build = function (list, size) { for (i = 0, len = list.length; i < len; i++) { const msgidLength = /** @type {Buffer} */(/** @type {unknown} */(list[i].msgid)); msgidLength.copy(returnBuffer, curPosition); - returnBuffer.writeUInt32LE(list[i].msgid.length, 28 + i * 8); - returnBuffer.writeUInt32LE(curPosition, 28 + i * 8 + 4); + returnBuffer[this._writeFunc](list[i].msgid.length, 28 + i * 8); + returnBuffer[this._writeFunc](curPosition, 28 + i * 8 + 4); returnBuffer[curPosition + list[i].msgid.length] = 0x00; curPosition += list[i].msgid.length + 1; } @@ -268,8 +277,8 @@ Compiler.prototype._build = function (list, size) { for (i = 0, len = list.length; i < len; i++) { const msgstrLength = /** @type {Buffer} */(/** @type {unknown} */(list[i].msgstr)); msgstrLength.copy(returnBuffer, curPosition); - returnBuffer.writeUInt32LE(list[i].msgstr.length, 28 + (4 + 4) * list.length + i * 8); - returnBuffer.writeUInt32LE(curPosition, 28 + (4 + 4) * list.length + i * 8 + 4); + returnBuffer[this._writeFunc](list[i].msgstr.length, 28 + (4 + 4) * list.length + i * 8); + returnBuffer[this._writeFunc](curPosition, 28 + (4 + 4) * list.length + i * 8 + 4); returnBuffer[curPosition + list[i].msgstr.length] = 0x00; curPosition += list[i].msgstr.length + 1; } diff --git a/test/fixtures/latin13-be.mo b/test/fixtures/latin13-be.mo new file mode 100644 index 0000000000000000000000000000000000000000..aa12185eb81789ca4b9d3718126dc1ab0a5fb9d2 GIT binary patch literal 678 zcmZ8eJ&P1U5N+30aRdWngEmv7&Y9UgSbJUZ)DsTWTh_t6?J39BneL&x=eSD@j8zac zG7`bq&_wW;82JNCG!n7qL-%mt^{Y2kucm7H&G^h$);}kR&JwagKR?9BDSkmXO?XWh z5#AF{5(;0rvyc(s^{z3Tx{E-0_9Z z;d)be$gJSfM>!#*?X-PA|rjIF4>hA0Ttm z_)-Sz1mEiV@D*xPNyD!UeV;g#K~m?dWHCh}3xo`1_5XDL48tJna>1h;)&w;4y@Q$t zYInL+mr|QKSC87=!a{osv)SflNG9v*cDje?VB090Yf}h5d!R$KEE^|FxMf}C1vhmk z{UV+7K3tqP=N8F@c~t2NVejwXOg7?ZA8L!a4p0ue2)^xuHCmc=PT9r;cR;Co+3ww) kt#!uR#wi?T7X0wgOa;2+&|Nj1DI?jbUQFQk_x_^)00?ruxBvhE literal 0 HcmV?d00001 diff --git a/test/fixtures/latin13.mo b/test/fixtures/latin13-le.mo similarity index 100% rename from test/fixtures/latin13.mo rename to test/fixtures/latin13-le.mo diff --git a/test/fixtures/obsolete-be.mo b/test/fixtures/obsolete-be.mo new file mode 100644 index 0000000000000000000000000000000000000000..b70aacd9360d8d8e22d78f6ff9efa9348532aab1 GIT binary patch literal 125 zcmbQrB6N=d0+@i53=r!;`8G&27m#fW#2P^C0mR%uT*wfTT3n)#Tb!Pm!r+{rSCX1n sq8n0KkZPq+l3G!sUyzfSnP;t#oRL^moLXXAT9T$~!3Eb-TvEgU087&u{{R30 literal 0 HcmV?d00001 diff --git a/test/fixtures/obsolete.mo b/test/fixtures/obsolete-le.mo similarity index 100% rename from test/fixtures/obsolete.mo rename to test/fixtures/obsolete-le.mo diff --git a/test/fixtures/utf8-be.mo b/test/fixtures/utf8-be.mo new file mode 100644 index 0000000000000000000000000000000000000000..7c5023ec823999656d8bc4e6488ca9cbd52f115d GIT binary patch literal 851 zcmZ8f&1w`u5FR%{$p{|w>Y?vx`lXOPnLLw@H482O#|U})%k z1Rnx_20sLU1*6Z*T5$5tsktDHW>XUdX$*=57O-)Eow3(AAxyt9^MUD$(d*HZ(VNk` zz|04poewswxyPv&E19^>)+wVC%GrC~^`es2u?;u5GS-WmE?1ejTIWs-O+r#6!)T7^e%{Kfui}OfAMUu-|K~|(s?noQoBNWV-s2$_n}Qz zx|EtOP5R!qEWJeM%P7%#Eprw_mgWEI{uWGxOw$tzuNm!G<9~Y<1T|+mUlmg6MN02B zXY+&FZ;OR#UZZQ9tytEn$&}6$TG&!P+$d@*3vt8P(j}y;vFir$BDKrGX?cKd7?I}op~J=j+_Bd>eX;*x|8cNtESIDzSC^ET!dP<3lPuYvqts}J(X+2D$xKzsr8qKi xp%8SOp@3M`t~43$uEZlCrBjuu3~C{T3b}>$#J+9KRLWdSl7)Iu%WnI=egmOa^{@Z{ literal 0 HcmV?d00001 diff --git a/test/fixtures/utf8.mo b/test/fixtures/utf8-le.mo similarity index 100% rename from test/fixtures/utf8.mo rename to test/fixtures/utf8-le.mo diff --git a/test/mo-compiler-test.js b/test/mo-compiler-test.js index 3b0a8b6..5026234 100644 --- a/test/mo-compiler-test.js +++ b/test/mo-compiler-test.js @@ -11,32 +11,85 @@ const __dirname = path.dirname(__filename); const readFile = promisify(fsReadFile); const expect = chai.expect; + +const littleEndianMagic = [0xde, 0x12, 0x04, 0x95]; +const bigEndianMagic = [0x95, 0x04, 0x12, 0xde]; + chai.config.includeStack = true; describe('MO Compiler', () => { - describe('UTF-8', () => { + describe('UTF-8 LE', async () => { + const [json, moData] = await Promise.all([ + readFile(path.join(__dirname, 'fixtures/utf8-po.json'), 'utf8'), + readFile(path.join(__dirname, 'fixtures/utf8-le.mo')) + ]); + it('should compile', async () => { - const [json, moData] = await Promise.all([ - readFile(path.join(__dirname, 'fixtures/utf8-po.json'), 'utf8'), - readFile(path.join(__dirname, 'fixtures/utf8.mo')) - ]); + const compiled = mo.compile(JSON.parse(json)); + expect(compiled.toString('utf8')).to.deep.equal(moData.toString('utf8')); + }); + + it('should have the correct magic number', () => { const compiled = mo.compile(JSON.parse(json)); + expect(Array.from(compiled.subarray(0, 4))).to.eql(littleEndianMagic); + }); + }); + + describe('UTF-8 BE', async () => { + const [json, moData] = await Promise.all([ + readFile(path.join(__dirname, 'fixtures/utf8-po.json'), 'utf8'), + readFile(path.join(__dirname, 'fixtures/utf8-be.mo')) + ]); + + it('should compile', async () => { + const compiled = mo.compile(JSON.parse(json), { endian: 'be' }); + expect(compiled.toString('utf8')).to.deep.equal(moData.toString('utf8')); }); + + it('should have the correct magic number', () => { + const compiled = mo.compile(JSON.parse(json), { endian: 'be' }); + + expect(Array.from(compiled.subarray(0, 4))).to.eql(bigEndianMagic); + }); }); - describe('Latin-13', () => { + describe('Latin-13 LE', async () => { + const [json, moData] = await Promise.all([ + readFile(path.join(__dirname, 'fixtures/latin13-po.json'), 'utf8'), + readFile(path.join(__dirname, 'fixtures/latin13-le.mo')) + ]); + it('should compile', async () => { - const [json, moData] = await Promise.all([ - readFile(path.join(__dirname, 'fixtures/latin13-po.json'), 'utf8'), - readFile(path.join(__dirname, 'fixtures/latin13.mo')) - ]); + const compiled = mo.compile(JSON.parse(json)); + expect(compiled.toString('utf8')).to.equal(moData.toString('utf8')); + }); + + it('should have the correct magic number', () => { const compiled = mo.compile(JSON.parse(json)); + expect(Array.from(compiled.subarray(0, 4))).to.eql(littleEndianMagic); + }); + }); + + describe('Latin-13 BE', async () => { + const [json, moData] = await Promise.all([ + readFile(path.join(__dirname, 'fixtures/latin13-po.json'), 'utf8'), + readFile(path.join(__dirname, 'fixtures/latin13-be.mo')) + ]); + + it('should compile', async () => { + const compiled = mo.compile(JSON.parse(json), { endian: 'be' }); + expect(compiled.toString('utf8')).to.equal(moData.toString('utf8')); }); + + it('should have the correct magic number', () => { + const compiled = mo.compile(JSON.parse(json), { endian: 'be' }); + expect(Array.from(compiled.subarray(0, 4))).to.eql(bigEndianMagic); + }); }); }); diff --git a/test/mo-parser-test.js b/test/mo-parser-test.js index 4391471..2c463c6 100644 --- a/test/mo-parser-test.js +++ b/test/mo-parser-test.js @@ -14,10 +14,10 @@ const expect = chai.expect; chai.config.includeStack = true; describe('MO Parser', () => { - describe('UTF-8', () => { + describe('UTF-8 LE', () => { it('should parse', async () => { const [moData, json] = await Promise.all([ - readFile(path.join(__dirname, 'fixtures/utf8.mo')), + readFile(path.join(__dirname, 'fixtures/utf8-le.mo')), readFile(path.join(__dirname, 'fixtures/utf8-mo.json'), 'utf8') ]); @@ -27,10 +27,36 @@ describe('MO Parser', () => { }); }); - describe('Latin-13', () => { + describe('UTF-8 BE', () => { it('should parse', async () => { const [moData, json] = await Promise.all([ - readFile(path.join(__dirname, 'fixtures/latin13.mo')), + readFile(path.join(__dirname, 'fixtures/utf8-be.mo')), + readFile(path.join(__dirname, 'fixtures/utf8-mo.json'), 'utf8') + ]); + + const parsed = mo.parse(moData); + + expect(parsed).to.deep.equal(JSON.parse(json)); + }); + }); + + describe('Latin-13 LE', () => { + it('should parse', async () => { + const [moData, json] = await Promise.all([ + readFile(path.join(__dirname, 'fixtures/latin13-le.mo')), + readFile(path.join(__dirname, 'fixtures/latin13-mo.json'), 'utf8') + ]); + + const parsed = mo.parse(moData); + + expect(parsed).to.deep.equal(JSON.parse(json)); + }); + }); + + describe('Latin-13 BE', () => { + it('should parse', async () => { + const [moData, json] = await Promise.all([ + readFile(path.join(__dirname, 'fixtures/latin13-be.mo')), readFile(path.join(__dirname, 'fixtures/latin13-mo.json'), 'utf8') ]); diff --git a/test/po-obsolete-test.js b/test/po-obsolete-test.js index d8a4a18..aa7390d 100644 --- a/test/po-obsolete-test.js +++ b/test/po-obsolete-test.js @@ -17,7 +17,7 @@ chai.config.includeStack = true; describe('Obsolete', async () => { const [po, mo, jsonString] = await Promise.all([ readFile(path.join(__dirname, 'fixtures/obsolete.po')), - readFile(path.join(__dirname, 'fixtures/obsolete.mo')), + readFile(path.join(__dirname, 'fixtures/obsolete-le.mo')), readFile(path.join(__dirname, 'fixtures/obsolete.json'), 'utf8') ]);