uweb/en/searchurl/mdict/mdict-parser.js
2022-10-12 17:43:25 +08:00

938 lines
37 KiB
JavaScript

//define(['pako', 'lzo', 'ripemd128', 'bluebird', 'mdict-parseXml', 'mdict-MCommon']
//pako, lzo, ripemd128, Promise, parseXml, MCommon
var parseXml = function (str) {
return (new DOMParser()).parseFromString(str, 'text/xml');
};
var MParser = (function (){
// Value of undefined.
var UNDEFINED = void 0;
// A shared UTF-16LE text decorder used to read dictionary header string.
var UTF_16LE = new TextDecoder('utf-16le');
/**
* Return the first argument as result.
* This function is used to simulate consequence, i.e. read data and return it, then forward to a new position.
* @param any data or function call
* @return the first arugment
*/
function conseq(/* args... */) { return arguments[0]; }
/*
* Decrypt encrypted data block of keyword index (attrs.Encrypted = "2").
* @see https://github.com/zhansliu/writemdict/blob/master/fileformat.md#keyword-index-encryption
* @param buf an ArrayBuffer containing source data
* @param key an ArrayBuffer holding decryption key, which will be supplied to ripemd128() before decryption
* @return an ArrayBuffer carrying decrypted data, occupying the same memory space of source buffer
*/
function decrypt(buf, key) {
key = ripemd128(key);
var byte, keylen = key.length, prev = 0x36, i = 0, len = buf.length;
for (; i < len; i++) {
byte = buf[i];
byte = ((byte >> 4) | (byte << 4) ); // & 0xFF; <-- it's already a byte
byte = byte ^ prev ^ (i & 0xFF) ^ key[i % keylen];
prev = buf[i];
buf[i] = byte;
}
return buf;
}
/**
* For sliceThen(..).exec(proc, ..), mark what proc function returns is multiple values
* to be passed to further Promise#spread(..) call.
*/
function spreadus() {
var args = Array.prototype.slice.apply(arguments);
args._spreadus_ = true;
return args;
}
/**
* Slice part of a file/blob object, return a promise object which will resolve to an ArrayBuffer to feed subsequent process.
* The returned promise object is extened with an exec(proc, args...) method which can be chained with further process.
* @param file file or blob object
* @param offset start position to slice
* @param len length to slice
* @return a promise object which will resolve to an ArrayBuffer containing data been read
*/
function sliceThen(file, offset, len) {
var p = new Promise(function(_resolve) {
var reader = new FileReader();
reader.onload = function() { _resolve(reader.result); }
reader.readAsArrayBuffer(file.slice(offset, offset + len));
});
/**
* Call proc with specified arguments prepending with sliced file/blob data (ArrayBuffer) been read.
* @param the first argument is a function to be executed
* @param other optional arguments are passed to the function following auto supplied input ArrayBuffer
* @return a promise object which can be chained with further process through spread() method
*/
p.exec = function(proc /*, args... */) {
var args = Array.prototype.slice.call(arguments, 1);
return p.then(function(data) {
args.unshift(data);
var ret = proc.apply(null, args);
return resolve(ret !== UNDEFINED && ret._spreadus_ ? ret : [ret]);
});
};
return p;
}
/**
* Wrap value as a resolved promise.
*/
function resolve(value) { return Promise.resolve(value); }
/**
* Wrap value as a rejected promise.
*/
function reject(reason) { return Promise.reject(reason); }
/**
* Harvest any resolved promises, if all failed then return reasons.
*/
function harvest(outcomes) {
return Promise.settle(outcomes).then(function(results) {
if (results.length === 0) {
return reject("** NOT FOUND **");
}
var solved = [], failed = [];
for (var i = 0; i < results.length; i++) {
if (results[i].isResolved()) {
solved.push(results[i].value());
} else {
failed.push(results[i].reason());
}
}
return solved.length ? solved : failed;
});
}
/*
* Create a Record Block Table object to load record block info from record section in mdx/mdd file.
* Retrived data is stored in an Uint32Array which contains N pairs of (offset_comp, offset_decomp) value,
* where N is number of record blocks.
*
* When looking up a given key for its definition:
* 1. Search KEY_INDEX to locate keyword block containing the given key.
* 2. Scanning the found keyword block to get its record offset and size.
* 3. Search RECORD_BLOCK_TABLE to get record block containing the record.
* 4. Load the found record block, using its offset and size to retrieve record content.
*
* @see https://github.com/zhansliu/writemdict/blob/master/fileformat.md#record-section
*/
function createRecordBlockTable() {
var pos = 0, // current position
arr; // backed Uint32Array
return {
// Allocate required ArrayBuffer for storing record block table, where len is number of record blocks.
alloc: function(len) {
arr = new Uint32Array(len * 2);
},
// Store offset pair value (compressed & decompressed) for a record block
// NOTE: offset_comp is absolute offset counted from start of mdx/mdd file.
put: function(offset_comp, offset_decomp) {
arr[pos++] = offset_comp; arr[pos++] = offset_decomp;
},
// Given offset of a keyword after decompression, return a record block info containing it, else undefined if not found.
find: function(keyAt) {
var hi = (arr.length >> 1) - 1, lo = 0, i = (lo + hi) >> 1, val = arr[(i << 1) + 1];
if (keyAt > arr[(hi << 1) + 1] || keyAt < 0) {
return;
}
while (true) {
if (hi - lo <= 1) {
if (i < hi) {
return {
block_no: i,
comp_offset: arr[i <<= 1],
comp_size: arr[i + 2] - arr[i],
decomp_offset:arr[i + 1],
decomp_size: arr[i + 3] - arr[i + 1]
};
} else {
return;
}
}
(keyAt < val) ? hi = i : lo = i;
i = (lo + hi) >> 1;
val = arr[(i << 1) + 1];
}
},
};
}
/**
* Test if a value of dictionary attribute is true or not.
*/
function isTrue(v) {
v = ((v || false) + '').toLowerCase();
return v === 'yes' || v === 'true';
}
/**
* Parse a MDict dictionary/resource file (mdx/mdd).
* @param file a File/Blob object
* @param ext file extension, mdx/mdd
* @return a Promise object which will resolve to a lookup function.
*/
function parse_mdict(file, ext) {
var KEY_INDEX, // keyword index array
RECORD_BLOCK_TABLE = createRecordBlockTable(); // record block table
var attrs = {}, // storing dictionary attributes
_v2, // true if enginge version > 2
_bpu, // bytes per unit when converting text size to byte length for text data
_tail, // need to skip extra tail bytes after decoding text
_decoder, // text decorder
_decryptors = [false, false],
// [keyword_header_decryptor, keyword_index_decryptor], only keyword_index_decryptor is supported
_searchTextLen, // search NUL to get text length
_readShort = function(scanner) { return scanner.readUint8(); },
// read a "short" number representing kewword text size, 8-bit for version < 2, 16-bit for version >= 2
_readNum = function(scanner) { return scanner.readInt(); },
// Read a number representing offset or data block size, 16-bit for version < 2, 32-bit for version >= 2
_checksum_v2 = function() {},
// Version >= 2.0 only checksum
_adaptKey = function(key) { return key; },
// adapt key by converting to lower case or stripping punctuations according to dictionary attributes (KeyCaseSensitive, StripKey)
_slice = sliceThen.bind(null, file);
// bind sliceThen() with file argument
/**
* Config scanner according to dictionary attributes.
*/
function config() {
attrs.Encoding = attrs.Encoding || 'UTF-16';
_searchTextLen = (attrs.Encoding === 'UTF-16')
? function(dv, offset) {
offset = offset;
var mark = offset;
while (dv.getUint16(offset)) { offset+= _bpu /* scan for \u0000 */ };
return offset - mark;
} : function(dv, offset) {
offset = offset;
var mark = offset;
while (dv.getUint8(offset++)) { /* scan for NUL */ }
return offset - mark - 1;
};
_decoder = new TextDecoder(attrs.Encoding || 'UTF-16LE');
_bpu = (attrs.Encoding === 'UTF-16') ? 2 : 1;
if (parseInt(attrs.GeneratedByEngineVersion, 10) >= 2.0) {
_v2 = true;
_tail = _bpu;
// HUGE dictionary file (>4G) is not supported, take only lower 32-bit
_readNum = function(scanner) { return scanner.forward(4), scanner.readInt(); };
_readShort = function(scanner) { return scanner.readUint16(); };
_checksum_v2 = function(scanner) { return scanner.checksum(); };
} else {
_tail = 0;
}
// keyword index decrypted?
if (attrs.Encrypted & 0x02) {
_decryptors[1] = decrypt;
}
var regexp = MCommon.REGEXP_STRIPKEY[ext];
if (isTrue(attrs.KeyCaseSensitive)) {
_adaptKey = isTrue(attrs.StripKey)
? function(key) { return key.replace(regexp, '$1'); }
: function(key) { return key; };
} else {
_adaptKey = isTrue(attrs.StripKey || (_v2 ? '' : 'yes'))
? function(key) { return key.toLowerCase().replace(regexp, '$1'); }
: function(key) { return key.toLowerCase(); };
}
}
// Read data in current offset from target data ArrayBuffer
function Scanner(buf, len) {
var offset = 0, dv = new DataView(buf);
var methods = {
// target data size in bytes
size: function() { return len || buf.byteLength; },
// update offset to new position
forward: function(len) { return offset += len; },
// return current offset
offset: function() { return offset; },
// MDict file format uses big endian to store number
// 32-bit unsigned int
readInt: function() { return conseq(dv.getUint32(offset, false), this.forward(4)); },
readUint16: function() { return conseq(dv.getUint16(offset, false), this.forward(2)); },
readUint8: function() { return conseq(dv.getUint8(offset, false), this.forward(1)); },
// Read a "short" number representing keyword text size, 8-bit for version < 2, 16-bit for version >= 2
readShort: function() { return _readShort(this); },
// Read a number representing offset or data block size, 16-bit for version < 2, 32-bit for version >= 2
readNum: function() { return _readNum(this); },
readUTF16: function(len) { return conseq(UTF_16LE.decode(new Uint8Array(buf, offset, len)), this.forward(len)); },
// Read data to an Uint8Array and decode it to text with specified encoding.
// Text length in bytes is determined by searching terminated NUL.
// NOTE: After decoding the text, it is need to forward extra "tail" bytes according to specified encoding.
readText: function() {
var len = _searchTextLen(dv, offset);
return conseq(_decoder.decode(new Uint8Array(buf, offset, len)), this.forward(len + _bpu));
},
// Read data to an Uint8Array and decode it to text with specified encoding.
// @param len length in basic unit, need to multiply byte per unit to get length in bytes
// NOTE: After decoding the text, it is need to forward extra "tail" bytes according to specified encoding.
readTextSized: function(len) {
len *= _bpu;
return conseq(_decoder.decode(new Uint8Array(buf, offset, len)), this.forward(len + _tail));
},
// Skip checksum, just ignore it anyway.
checksum: function() { this.forward(4); },
// Version >= 2.0 only
checksum_v2: function() { return _checksum_v2(this); },
// Read data block of keyword index, key block or record content.
// These data block are maybe in compressed (gzip or lzo) format, while keyword index maybe be encrypted.
// @see https://github.com/zhansliu/writemdict/blob/master/fileformat.md#compression (with typo mistake)
readBlock: function(len, expectedBufSize, decryptor) {
var comp_type = dv.getUint8(offset, false); // compression type, 0 = non, 1 = lzo, 2 = gzip
if (comp_type === 0) {
if (_v2) {
this.forward(8); // for version >= 2, skip comp_type (4 bytes with tailing \x00) and checksum (4 bytes)
}
return this;
} else {
// skip comp_type (4 bytes with tailing \x00) and checksum (4 bytes)
offset += 8; len -= 8;
var tmp = new Uint8Array(buf, offset, len);
if (decryptor) {
var passkey = new Uint8Array(8);
passkey.set(new Uint8Array(buf, offset - 4, 4)); // key part 1: checksum
passkey.set([0x95, 0x36, 0x00, 0x00], 4); // key part 2: fixed data
tmp = decryptor(tmp, passkey);
}
tmp = comp_type === 2 ? pako.inflate(tmp) : lzo.decompress(tmp, expectedBufSize, 1308672);
this.forward(len);
return Scanner(tmp.buffer, tmp.length);
}
},
// Read raw data as Uint8Array from current offset with specified length in bytes
readRaw: function(len) {
return conseq(new Uint8Array(buf, offset, len), this.forward(len === UNDEFINED ? buf.length - offset : len));
},
};
return Object.create(methods);
}
/**
* Read the first 4 bytes of mdx/mdd file to get length of header_str.
* @see https://github.com/zhansliu/writemdict/blob/master/fileformat.md#file-structure
* @param input sliced file (start = 0, length = 4)
* @return length of header_str
*/
function read_file_head(input) {
return Scanner(input).readInt();
}
/**
* Read header section, parse dictionary attributes and config scanner according to engine version attribute.
* @see https://github.com/zhansliu/writemdict/blob/master/fileformat.md#header-section
* @param input sliced file (start = 4, length = len + 48), header string + header section (max length 48)
* @param len lenghth of header_str
* @return [remained length of header section (header_str and checksum, = len + 4), original input]
*/
function read_header_sect(input, len) {
var scanner = Scanner(input),
header_str = scanner.readUTF16(len).replace(/\0$/, ''); // need to remove tailing NUL
// parse dictionary attributes
var xml = parseXml(header_str).querySelector('Dictionary, Library_Data').attributes;
for (var i = 0, item; i < xml.length; i++) {
item = xml.item(i);
attrs[item.nodeName] = item.nodeValue;
}
attrs.Encrypted = parseInt(attrs.Encrypted, 10) || 0;
MCommon.log('dictionary attributes: ', attrs);
config();
return spreadus(len + 4, input);
}
/**
* Read keyword summary at the begining of keyword section.
* @see https://github.com/zhansliu/writemdict/blob/master/fileformat.md#keyword-section
* @param input sliced file, same as input passed to read_header_sect()
* @param offset start position of keyword section in sliced file, equals to length of header string plus checksum.\
* @return keyword_sect object
*/
function read_keyword_summary(input, offset) {
var scanner = Scanner(input);
scanner.forward(offset);
return {
num_blocks: scanner.readNum(),
num_entries: scanner.readNum(),
key_index_decomp_len: _v2 && scanner.readNum(), // Ver >= 2.0 only
key_index_comp_len: scanner.readNum(),
key_blocks_len: scanner.readNum(),
chksum: scanner.checksum_v2(),
// extra field
len: scanner.offset() - offset, // actual length of keyword section, varying with engine version attribute
};
}
/**
* Read keyword index part of keyword section.
* @see https://github.com/zhansliu/writemdict/blob/master/fileformat.md#keyword-header-encryption
* @see https://github.com/zhansliu/writemdict/blob/master/fileformat.md#keyword-index
* @param input sliced file, remained part of keyword section after keyword summary which can also be used to read following key blocks.
* @param keyword_summary
* @return [keyword_summary, array of keyword index]
*/
function read_keyword_index(input, keyword_summary) {
var scanner = Scanner(input).readBlock(keyword_summary.key_index_comp_len, keyword_summary.key_index_decomp_len, _decryptors[1]),
keyword_index = Array(keyword_summary.num_blocks),
offset = 0;
for (var i = 0, size; i < keyword_summary.num_blocks; i++) {
keyword_index[i] = {
num_entries: conseq(scanner.readNum(), size = scanner.readShort()),
// UNUSED, can be ignored
// first_size: size = scanner.readShort(),
first_word: conseq(scanner.readTextSized(size), size = scanner.readShort()),
// UNUSED, can be ignored
// last_size: size = scanner.readShort(),
last_word: scanner.readTextSized(size),
comp_size: size = scanner.readNum(),
decomp_size: scanner.readNum(),
// extra fields
offset: offset, // offset of the first byte for the target key block in mdx/mdd file
index: i // index of this key index, used to search previous/next block
};
offset += size;
}
return spreadus(keyword_summary, keyword_index);
}
/**
* Read keyword entries inside a keyword block and fill KEY_TABLE.
* @param scanner scanner object to read key entries, which starts at begining of target key block
* @param kdx corresponding keyword index object
* NOTE: no need to read keyword block anymore, for debug only.
*/
function read_key_block(scanner, kdx) {
var scanner = scanner.readBlock(kdx.comp_size, kdx.decomp_size);
for (var i = 0; i < kdx.num_entries; i++) {
// scanner.readNum(); scanner.readText();
var kk = [scanner.readNum(), scanner.readText()];
// console.log(scanner.readNum(), scanner.readText());
}
}
/**
* Delay to scan key table, for debug onyl.
* @param slicedKeyBlock a promise object which will resolve to an ArrayBuffer containing keyword blocks
* sliced from mdx/mdd file.
* @param num_entries number of keyword entries
* @param keyword_index array of keyword index
* @param delay time to delay for scanning key table
*/
function willScanKeyTable(slicedKeyBlock, num_entries, keyword_index, delay) {
slicedKeyBlock.delay(delay).then(function (input) {
MCommon.log('scan key table...');
var scanner = Scanner(input);
for (var i = 0, size = keyword_index.length; i < size; i++) {
read_key_block(scanner, keyword_index[i]);
}
MCommon.log('KEY_TABLE loaded.');
});
}
/**
* Read record summary at the begining of record section.
* @see https://github.com/zhansliu/writemdict/blob/master/fileformat.md#record-section
* @param input sliced file, start = begining of record section, length = 32 (max length of record summary)
* @param pos begining of record section
* @returj record summary object
*/
function read_record_summary(input, pos) {
var scanner = Scanner(input),
record_summary = {
num_blocks: scanner.readNum(),
num_entries: scanner.readNum(),
index_len: scanner.readNum(),
blocks_len: scanner.readNum(),
// extra field
len: scanner.offset(), // actual length of record section (excluding record block index), varying with engine version attribute
};
// start position of record block from head of mdx/mdd file
record_summary.block_pos = pos + record_summary.index_len + record_summary.len;
return record_summary;
}
/**
* Read record block index part in record section, and fill RECORD_BLOCK_TABLE
* @see https://github.com/zhansliu/writemdict/blob/master/fileformat.md#record-section
* @param input sliced file, start = begining of record block index, length = record_summary.index_len
* @param record_summary record summary object
*/
function read_record_block(input, record_summary) {
var scanner = Scanner(input),
size = record_summary.num_blocks,
record_index = Array(size),
p0 = record_summary.block_pos,
p1 = 0;
RECORD_BLOCK_TABLE.alloc(size + 1);
for (var i = 0, rdx; i < size; i++) {
record_index[i] = rdx = {
comp_size: scanner.readNum(),
decomp_size: scanner.readNum()
};
RECORD_BLOCK_TABLE.put(p0, p1);
p0 += rdx.comp_size;
p1 += rdx.decomp_size;
}
RECORD_BLOCK_TABLE.put(p0, p1);
}
/**
* Read definition in text for given keyinfo object.
* @param input record block sliced from the file
* @param block record block index
* @param keyinfo a object with property of record's offset and optional size for the given keyword
* @return definition in text
*/
function read_definition(input, block, keyinfo) {
var scanner = Scanner(input).readBlock(block.comp_size, block.decomp_size);
scanner.forward(keyinfo.offset - block.decomp_offset);
return scanner.readText();
}
/**
* Following link to find actual definition of keyword.
* @param definition maybe starts with "@@@LINK=" which links to another keyword
* @param lookup search function
* @return resolved actual definition
*/
function followLink(definition, lookup) {
return (definition.substring(0, 8) !== '@@@LINK=')
? definition
: lookup(definition.substring(8));
}
/**
* Read content in ArrayBuffer for give keyinfo object
* @param input record block sliced from the file
* @param block record block index
* @param keyinfo a object with property of record's offset and optional size for the given keyword
* @return an ArrayBuffer containing resource of image/audio/css/font etc.
*/
function read_object(input, block, keyinfo) {
if (input.byteLength > 0) {
var scanner = Scanner(input).readBlock(block.comp_size, block.decomp_size);
scanner.forward(keyinfo.offset - block.decomp_offset);
return scanner.readRaw(keyinfo.size);
} else {
throw '* OUT OF FILE RANGE * ' + keyinfo + ' @offset=' + block.comp_offset;
}
}
/**
* Find word definition for given keyinfo object.
* @param keyinfo a object with property of record's offset and optional size for the given keyword
* @return a promise object which will resolve to definition in text. Link to other keyword is followed to get actual definition.
*/
function findWord(keyinfo) {
var block = RECORD_BLOCK_TABLE.find(keyinfo.offset);
return _slice(block.comp_offset, block.comp_size)
.exec(read_definition, block, keyinfo)
.spread(function (definition) { return resolve(followLink(definition, LOOKUP.mdx)); });
}
/**
* Find resource (image, sound etc.) for given keyinfo object.
* @param keyinfo a object with property of record's offset and optional size for the given keyword
* @return a promise object which will resolve to an ArrayBuffer containing resource of image/audio/css/font etc.
* TODO: Follow link, maybe it's too expensive and a rarely used feature?
*/
function findResource(keyinfo) {
var block = RECORD_BLOCK_TABLE.find(keyinfo.offset);
return _slice(block.comp_offset, block.comp_size)
.exec(read_object, block, keyinfo)
.spread(function (blob) { return resolve(blob); });
}
//------------------------------------------------------------------------------------------------
// Implementation for look-up
//------------------------------------------------------------------------------------------------
var slicedKeyBlock,
_cached_keys, // cache latest keys
_trail, // store latest visited record block & position when search for candidate keys
mutual_ticket = 0; // a oneway increased ticket used to cancel unfinished pattern match
/**
* Reduce the key index array to an element which contains or is the nearest one matching a given phrase.
*/
function reduce(arr, phrase) {
var len = arr.length;
if (len > 1) {
len = len >> 1;
return phrase > _adaptKey(arr[len - 1].last_word)
? reduce(arr.slice(len), phrase)
: reduce(arr.slice(0, len), phrase);
} else {
return arr[0];
}
}
/**
* Reduce the array to index of an element which contains or is the nearest one matching a given phrase.
*/
function shrink(arr, phrase) {
var len = arr.length, sub;
if (len > 1) {
len = len >> 1;
var key = _adaptKey(arr[len]);
if (phrase < key) {
sub = arr.slice(0, len);
sub.pos = arr.pos;
} else {
sub = arr.slice(len);
sub.pos = (arr.pos || 0) + len;
}
return shrink(sub, phrase);
} else {
return (arr.pos || 0) + (phrase <= _adaptKey(arr[0]) ? 0 : 1);
}
}
/**
* Load keys for a keyword index object from mdx/mdd file.
* @param kdx keyword index object
*/
function loadKeys(kdx) {
if (_cached_keys && _cached_keys.pilot === kdx.first_word) {
return resolve(_cached_keys.list);
} else {
return slicedKeyBlock.then(function(input) {
var scanner = Scanner(input), list = Array(kdx.num_entries);
scanner.forward(kdx.offset);
scanner = scanner.readBlock(kdx.comp_size, kdx.decomp_size);
for (var i = 0; i < kdx.num_entries; i++) {
var offset = scanner.readNum();
list[i] = new Object(scanner.readText());
list[i].offset = offset;
if (i > 0) {
list[i - 1].size = offset - list[i - 1].offset;
}
}
_cached_keys = {list: list, pilot: kdx.first_word};
return list;
});
}
}
/**
* Search for the first keyword match given phrase.
*/
function seekVanguard(phrase) {
phrase = _adaptKey(phrase);
var kdx = reduce(KEY_INDEX, phrase);
// look back for the first record block containing keyword for the specified phrase
if (phrase <= _adaptKey(kdx.last_word)) {
var index = kdx.index - 1, prev;
while (prev = KEY_INDEX[index]) {
if (_adaptKey(prev.last_word) !== _adaptKey(kdx.last_word)) {
break;
}
kdx = prev;
index--;
}
}
return loadKeys(kdx).then(function (list) {
var idx = shrink(list, phrase);
// look back for the first matched keyword position
while (idx > 0) {
if (_adaptKey(list[--idx]) !== _adaptKey(phrase)) {
idx++;
break;
}
}
return [kdx, Math.min(idx, list.length - 1), list];
});
}
// TODO: have to restrict max count to improve response
/**
* Append more to word list according to a filter or expected size.
*/
function appendMore(word, list, nextKdx, expectedSize, filter, ticket) {
if (ticket !== mutual_ticket) {
throw 'force terminated';
}
if (filter) {
if (_trail.count < expectedSize && nextKdx && nextKdx.first_word.substr(0, word.length) === word) {
return loadKeys(nextKdx).delay(30).then(function(more) {
MCommon.log(nextKdx);
_trail.offset = 0;
_trail.block = nextKdx.index;
Array.prototype.push.apply(list, more.filter(filter, _trail));
return appendMore(word, list, KEY_INDEX[nextKdx.index + 1], expectedSize, filter, ticket);
});
} else {
if (list.length === 0) {
_trail.exhausted = true;
}
return resolve(list);
}
} else {
var shortage = expectedSize - list.length;
if (shortage > 0 && nextKdx) {
console.log('go next', nextKdx);
_trail.block = nextKdx.index;
return loadKeys(nextKdx).then(function(more) {
_trail.offset = 0;
_trail.pos = Math.min(shortage, more.length);
Array.prototype.push.apply(list, more.slice(0, shortage));
console.log('$$ ' + more[shortage - 1], shortage);
return appendMore(word, list, KEY_INDEX[nextKdx.index + 1], expectedSize, filter, ticket);
});
} else {
if (_trail.pos > expectedSize) {
_trail.pos = expectedSize;
}
list = list.slice(0, expectedSize);
_trail.count = list.length;
_trail.total += _trail.count;
return resolve(list);
}
}
}
function followUp() {
var kdx = KEY_INDEX[_trail.block];
return loadKeys(kdx).then(function (list) {
return [kdx, Math.min(_trail.offset + _trail.pos, list.length - 1), list];
});
}
function matchKeys(phrase, expectedSize, follow) {
expectedSize = Math.max(expectedSize || 0, 10);
var str = phrase.trim().toLowerCase(),
m = /([^?*]+)[?*]+/.exec(str),
word;
if (m) {
word = m[1];
var wildcard = new RegExp('^' + str.replace(/([\.\\\+\[\^\]\$\(\)])/g, '\\$1').replace(/\*+/g, '.*').replace(/\?/g, '.') + '$'),
tester = phrase[phrase.length - 1] === ' '
? function(s) { return wildcard.test(s); }
: function(s) { return wildcard.test(s) && !/ /.test(s); },
filter = function (s, i) {
if (_trail.count < expectedSize && tester(s)) {
_trail.count++;
_trail.total++;
_trail.pos = i + 1;
return true;
}
return false;
};
} else {
word = phrase.trim();
}
if (_trail && _trail.phrase !== phrase) {
follow = false;
}
if (follow && _trail && _trail.exhausted) {
return resolve([]);
}
var startFrom = follow && _trail ? followUp() : seekVanguard(word);
return startFrom.spread(function(kdx, idx, list) {
console.log('start ', kdx);
list = list.slice(idx);
_trail = {phrase: phrase,
block: kdx.index,
offset: idx,
pos: list.length,
count: 0,
total: follow ? _trail && _trail.total || 0 : 0
};
if (filter) {
list = list.filter(filter, _trail);
}
return appendMore(word, list, KEY_INDEX[kdx.index + 1], expectedSize, filter, ++mutual_ticket)
.then(function(result) {
if (_trail.block === KEY_INDEX.length - 1) {
if (_trail.offset + _trail.pos >= KEY_INDEX[_trail.block].num_entries) {
_trail.exhausted = true;
console.log('EXHAUSTED!!!!');
}
}
console.log('trail: ', _trail);
return result;
});
});
};
/**
* Match the first element in list with given offset.
*/
function matchOffset(list, offset) {
return list.some(function(el) { return el.offset === offset ? list = [el] : false; }) ? list : [];
}
// Lookup functions
var LOOKUP = {
/**
* @param query
* String
* {phrase: .., max: .., follow: true} object
*/
mdx: function(query) {
if (typeof query === 'string' || query instanceof String) {
_trail = null;
var word = query.trim().toLowerCase(), offset = query.offset;
return seekVanguard(word).spread(function(kdx, idx, list) {
list = list.slice(idx);
if (offset !== UNDEFINED) {
list = matchOffset(list, offset);
} else {
list = list.filter(function(el) { return el.toLowerCase() === word; });
}
return harvest(list.map(findWord));
});
} else {
return matchKeys(query.phrase, query.max, query.follow);
}
},
// TODO: chain multiple mdd file
mdd: function(phrase) {
var word = phrase.trim().toLowerCase();
word = '\\' + word.replace(/(^[/\\])|([/]$)/, '');
word = word.replace(/\//g, '\\');
return seekVanguard(word).spread(function(kdx, idx, list) {
return list.slice(idx).filter(function(one) {
return one.toLowerCase() === word;
});
}).then(function(candidates) {
if (candidates.length === 0) {
throw '*RESOURCE NOT FOUND* ' + phrase;
} else {
return findResource(candidates[0]);
}
});
}
};
// ------------------------------------------
// start to load mdx/mdd file
// ------------------------------------------
MCommon.log('start to load ' + file.name);
var pos = 0;
// read first 4 bytes to get header length
return _slice(pos, 4).exec(read_file_head).spread(function(len) {
pos += 4; // start of header string in header section
return _slice(pos, len + 48)
.exec(read_header_sect, len);
}).spread(function(header_remain_len, input) {
pos += header_remain_len; // start of keyword section
return read_keyword_summary(input, header_remain_len);
}).then(function(keyword_summary) { MCommon.log(keyword_summary);
pos += keyword_summary.len; // start of key index in keyword section
return _slice(pos, keyword_summary.key_index_comp_len)
.exec(read_keyword_index, keyword_summary);
}).spread(function (keyword_summary, keyword_index) {
pos += keyword_summary.key_index_comp_len; // start of keyword block in keyword section
slicedKeyBlock = _slice(pos, keyword_summary.key_blocks_len);
/*
// Now it's fast enough to look up word without key table, which scans keyword from the specified key blocks in an effcient way.
// No need to scan the whole key table in ahead.
willScanKeyTable(slicedKeyBlock, keyword_summary.num_entries, keyword_index, 00);
// */
pos += keyword_summary.key_blocks_len; // start of record section
KEY_INDEX = keyword_index;
}).then(function () {
return _slice(pos, 32)
.exec(read_record_summary, pos);
}).spread(function (record_summary) { MCommon.log(record_summary);
pos += record_summary.len; // start of record blocks in record section
return _slice(pos, record_summary.index_len)
.exec(read_record_block, record_summary);
}).spread(function() { MCommon.log('-- parse done --', file.name);
// resolve and return lookup() function according to file extension (mdx/mdd)
LOOKUP[ext].description = attrs.Description;
return resolve(LOOKUP[ext]);
});
};
// -------------------------
// END OF parse_mdict()
// -------------------------
/**
* Load a set of files which will be parsed as MDict dictionary & resource (mdx/mdd).
*/
return function load(files) {
var resources = [];
Array.prototype.forEach.call(files, function(f) {
var ext = MCommon.getExtension(f.name, 'mdx');
resources.push(resources[ext] = parse_mdict(f, ext));
});
return Promise.all(resources)
.then(function() { return resolve(resources); });
};
}());