Skip to content

Commit

Permalink
feat: add support for compiling big-endian .mo files
Browse files Browse the repository at this point in the history
This commit adds support for compiling big-endian .mo files. This facilitates
the use of .mo files in big-endian systems, the ability to parser big-endian
.mo files was already present.

This commit also does the following:

- Adds a new mo parser options type, used to indicate `endian = 'le' | 'be'`
- Adds tests and fixtures for big-endian .mo files.
- Renames the existing .mo fixtures to indicate that they are little-endian.
- Fixes direct usages of `writeUInt32LE`.

Fixes smhg#97
  • Loading branch information
johnhooks committed May 13, 2024
1 parent f360d08 commit 281fc44
Show file tree
Hide file tree
Showing 10 changed files with 111 additions and 23 deletions.
25 changes: 17 additions & 8 deletions src/mocompiler.js
Original file line number Diff line number Diff line change
Expand Up @@ -21,15 +21,22 @@ import contentType from 'content-type';
* @typedef {{ msgid: Buffer, msgstr: Buffer }} TranslationBuffers A translation object partially parsed.
*/

/**
*
* @typedef {Object} CompilerOptions MO compiler options
* @property {'be'|'le'} [endian='le'] Endianness of the output buffer. Default is 'le'
*/

/**
* Exposes general compiler function. Takes a translation
* object as a parameter and returns binary MO object
*
* @param {GetTextTranslations} table Translation object
* @param {CompilerOptions} [options] MO compiler options
* @return {Buffer} Compiled binary MO object
*/
export default function (table) {
const compiler = new Compiler(table);
export default function (table, options = { endian: 'le' }) {
const compiler = new Compiler(table, options);

return compiler.compile();
}
Expand Down Expand Up @@ -91,8 +98,9 @@ function prepareTranslations (translations) {
* @this {Compiler & Transform}
*
* @param {GetTextTranslations} [table] Translation table as defined in the README
* @param {CompilerOptions} [options] MO compiler options
*/
function Compiler (table) {
function Compiler (table, options = { endian: 'le' }) {
/** @type {GetTextTranslations} _table The translation table */
this._table = {
charset: undefined,
Expand All @@ -101,10 +109,11 @@ function Compiler (table) {
};

this._translations = [];

/**
* @type {WriteFunc}
*/
this._writeFunc = 'writeUInt32LE';
this._writeFunc = options?.endian === 'le' ? 'writeUInt32LE' : 'writeUInt32BE';

this._handleCharset();

Expand Down Expand Up @@ -258,8 +267,8 @@ Compiler.prototype._build = function (list, size) {
for (i = 0, len = list.length; i < len; i++) {
const msgidLength = /** @type {Buffer} */(/** @type {unknown} */(list[i].msgid));
msgidLength.copy(returnBuffer, curPosition);
returnBuffer.writeUInt32LE(list[i].msgid.length, 28 + i * 8);
returnBuffer.writeUInt32LE(curPosition, 28 + i * 8 + 4);
returnBuffer[this._writeFunc](list[i].msgid.length, 28 + i * 8);
returnBuffer[this._writeFunc](curPosition, 28 + i * 8 + 4);
returnBuffer[curPosition + list[i].msgid.length] = 0x00;
curPosition += list[i].msgid.length + 1;
}
Expand All @@ -268,8 +277,8 @@ Compiler.prototype._build = function (list, size) {
for (i = 0, len = list.length; i < len; i++) {
const msgstrLength = /** @type {Buffer} */(/** @type {unknown} */(list[i].msgstr));
msgstrLength.copy(returnBuffer, curPosition);
returnBuffer.writeUInt32LE(list[i].msgstr.length, 28 + (4 + 4) * list.length + i * 8);
returnBuffer.writeUInt32LE(curPosition, 28 + (4 + 4) * list.length + i * 8 + 4);
returnBuffer[this._writeFunc](list[i].msgstr.length, 28 + (4 + 4) * list.length + i * 8);
returnBuffer[this._writeFunc](curPosition, 28 + (4 + 4) * list.length + i * 8 + 4);
returnBuffer[curPosition + list[i].msgstr.length] = 0x00;
curPosition += list[i].msgstr.length + 1;
}
Expand Down
Binary file added test/fixtures/latin13-be.mo
Binary file not shown.
File renamed without changes.
Binary file added test/fixtures/obsolete-be.mo
Binary file not shown.
File renamed without changes.
Binary file added test/fixtures/utf8-be.mo
Binary file not shown.
File renamed without changes.
73 changes: 63 additions & 10 deletions test/mo-compiler-test.js
Original file line number Diff line number Diff line change
Expand Up @@ -11,32 +11,85 @@ const __dirname = path.dirname(__filename);
const readFile = promisify(fsReadFile);

const expect = chai.expect;

const littleEndianMagic = [0xde, 0x12, 0x04, 0x95];
const bigEndianMagic = [0x95, 0x04, 0x12, 0xde];

chai.config.includeStack = true;

describe('MO Compiler', () => {
describe('UTF-8', () => {
describe('UTF-8 LE', async () => {
const [json, moData] = await Promise.all([
readFile(path.join(__dirname, 'fixtures/utf8-po.json'), 'utf8'),
readFile(path.join(__dirname, 'fixtures/utf8-le.mo'))
]);

it('should compile', async () => {
const [json, moData] = await Promise.all([
readFile(path.join(__dirname, 'fixtures/utf8-po.json'), 'utf8'),
readFile(path.join(__dirname, 'fixtures/utf8.mo'))
]);
const compiled = mo.compile(JSON.parse(json));

expect(compiled.toString('utf8')).to.deep.equal(moData.toString('utf8'));
});

it('should have the correct magic number', () => {
const compiled = mo.compile(JSON.parse(json));

expect(Array.from(compiled.subarray(0, 4))).to.eql(littleEndianMagic);
});
});

describe('UTF-8 BE', async () => {
const [json, moData] = await Promise.all([
readFile(path.join(__dirname, 'fixtures/utf8-po.json'), 'utf8'),
readFile(path.join(__dirname, 'fixtures/utf8-be.mo'))
]);

it('should compile', async () => {
const compiled = mo.compile(JSON.parse(json), { endian: 'be' });

expect(compiled.toString('utf8')).to.deep.equal(moData.toString('utf8'));
});

it('should have the correct magic number', () => {
const compiled = mo.compile(JSON.parse(json), { endian: 'be' });

expect(Array.from(compiled.subarray(0, 4))).to.eql(bigEndianMagic);
});
});

describe('Latin-13', () => {
describe('Latin-13 LE', async () => {
const [json, moData] = await Promise.all([
readFile(path.join(__dirname, 'fixtures/latin13-po.json'), 'utf8'),
readFile(path.join(__dirname, 'fixtures/latin13-le.mo'))
]);

it('should compile', async () => {
const [json, moData] = await Promise.all([
readFile(path.join(__dirname, 'fixtures/latin13-po.json'), 'utf8'),
readFile(path.join(__dirname, 'fixtures/latin13.mo'))
]);
const compiled = mo.compile(JSON.parse(json));

expect(compiled.toString('utf8')).to.equal(moData.toString('utf8'));
});

it('should have the correct magic number', () => {
const compiled = mo.compile(JSON.parse(json));

expect(Array.from(compiled.subarray(0, 4))).to.eql(littleEndianMagic);
});
});

describe('Latin-13 BE', async () => {
const [json, moData] = await Promise.all([
readFile(path.join(__dirname, 'fixtures/latin13-po.json'), 'utf8'),
readFile(path.join(__dirname, 'fixtures/latin13-be.mo'))
]);

it('should compile', async () => {
const compiled = mo.compile(JSON.parse(json), { endian: 'be' });

expect(compiled.toString('utf8')).to.equal(moData.toString('utf8'));
});

it('should have the correct magic number', () => {
const compiled = mo.compile(JSON.parse(json), { endian: 'be' });
expect(Array.from(compiled.subarray(0, 4))).to.eql(bigEndianMagic);
});
});
});
34 changes: 30 additions & 4 deletions test/mo-parser-test.js
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,10 @@ const expect = chai.expect;
chai.config.includeStack = true;

describe('MO Parser', () => {
describe('UTF-8', () => {
describe('UTF-8 LE', () => {
it('should parse', async () => {
const [moData, json] = await Promise.all([
readFile(path.join(__dirname, 'fixtures/utf8.mo')),
readFile(path.join(__dirname, 'fixtures/utf8-le.mo')),
readFile(path.join(__dirname, 'fixtures/utf8-mo.json'), 'utf8')
]);

Expand All @@ -27,10 +27,36 @@ describe('MO Parser', () => {
});
});

describe('Latin-13', () => {
describe('UTF-8 BE', () => {
it('should parse', async () => {
const [moData, json] = await Promise.all([
readFile(path.join(__dirname, 'fixtures/latin13.mo')),
readFile(path.join(__dirname, 'fixtures/utf8-be.mo')),
readFile(path.join(__dirname, 'fixtures/utf8-mo.json'), 'utf8')
]);

const parsed = mo.parse(moData);

expect(parsed).to.deep.equal(JSON.parse(json));
});
});

describe('Latin-13 LE', () => {
it('should parse', async () => {
const [moData, json] = await Promise.all([
readFile(path.join(__dirname, 'fixtures/latin13-le.mo')),
readFile(path.join(__dirname, 'fixtures/latin13-mo.json'), 'utf8')
]);

const parsed = mo.parse(moData);

expect(parsed).to.deep.equal(JSON.parse(json));
});
});

describe('Latin-13 BE', () => {
it('should parse', async () => {
const [moData, json] = await Promise.all([
readFile(path.join(__dirname, 'fixtures/latin13-be.mo')),
readFile(path.join(__dirname, 'fixtures/latin13-mo.json'), 'utf8')
]);

Expand Down
2 changes: 1 addition & 1 deletion test/po-obsolete-test.js
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ chai.config.includeStack = true;
describe('Obsolete', async () => {
const [po, mo, jsonString] = await Promise.all([
readFile(path.join(__dirname, 'fixtures/obsolete.po')),
readFile(path.join(__dirname, 'fixtures/obsolete.mo')),
readFile(path.join(__dirname, 'fixtures/obsolete-le.mo')),
readFile(path.join(__dirname, 'fixtures/obsolete.json'), 'utf8')
]);

Expand Down

0 comments on commit 281fc44

Please sign in to comment.