peranger/qskinny
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
qskinny/inputcontext/3rdparty/pinyin/share/userdict.cpp
Uwe Rathmann 59b516a118 3rdparty moved to inputcontext/3rdparty. 
The 3rdparty files are now compiled as part of the corresponding input
method, so that the project files can be written without using platform
specific linker flags.
2018-03-30 18:31:13 +02:00

2291 lines
58 KiB
C++
Raw Blame History

/*
* Copyright (C) 2009 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "../include/userdict.h"
#include "../include/splparser.h"
#include "../include/ngram.h"
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#ifdef ___DEBUG_PERF___
#include <cutils/log.h>
#endif
#ifdef _WIN32
#include <io.h>
#else
#include <unistd.h>
#endif
#include <fcntl.h>
#include <sys/stat.h>
#include <assert.h>
#include <ctype.h>
#include <sys/types.h>
#ifndef _WIN32
#include <sys/time.h>
#endif
#include <time.h>
#ifdef _WIN32
#undef max
#undef min
#include <QDateTime>
#include <QMutex>
#else
#include <pthread.h>
#endif
#include <math.h>
namespace ime_pinyin {
#ifdef _WIN32
static int gettimeofday(struct timeval *tp, void *) {
const qint64 current_msecs_since_epoch = QDateTime::currentMSecsSinceEpoch();
tp->tv_sec = (long)(current_msecs_since_epoch / 1000);
tp->tv_usec = (long)((current_msecs_since_epoch % 1000) * 1000);
return 0;
}
#endif
#ifdef ___DEBUG_PERF___
static uint64 _ellapse_ = 0;
static struct timeval _tv_start_, _tv_end_;
#define DEBUG_PERF_BEGIN \
do { \
gettimeofday(&_tv_start_, NULL); \
} while (0)
#define DEBUG_PERF_END \
do { \
gettimeofday(&_tv_end_, NULL); \
_ellapse_ = (_tv_end_.tv_sec - _tv_start_.tv_sec) * 1000000 + \
(_tv_end_.tv_usec - _tv_start_.tv_usec); \
} while (0)
#define LOGD_PERF(message) \
ALOGD("PERFORMANCE[%s] %llu usec.", message, _ellapse_);
#else
#define DEBUG_PERF_BEGIN
#define DEBUG_PERF_END
#define LOGD_PERF(message)
#endif
// XXX File load and write are thread-safe by g_mutex_
#ifdef _WIN32
static QMutex g_mutex_;
#define pthread_mutex_lock(MUTEX) ((MUTEX)->lock())
#define pthread_mutex_unlock(MUTEX) ((MUTEX)->unlock())
#define pthread_mutex_trylock(MUTEX) (!(MUTEX)->tryLock(0))
#else
static pthread_mutex_t g_mutex_ = PTHREAD_MUTEX_INITIALIZER;
#endif
static struct timeval g_last_update_ = {0, 0};
inline uint32 UserDict::get_dict_file_size(UserDictInfo * info) {
return (4 + info->lemma_size + (info->lemma_count << 3)
#ifdef ___PREDICT_ENABLED___
+ (info->lemma_count << 2)
#endif
#ifdef ___SYNC_ENABLED___
+ (info->sync_count << 2)
#endif
+ sizeof(*info));
}
inline LmaScoreType UserDict::translate_score(int raw_score) {
// 1) ori_freq: original user frequency
uint32 ori_freq = extract_score_freq(raw_score);
// 2) lmt_off: lmt index (week offset for example)
uint64 lmt_off = ((raw_score & 0xffff0000) >> 16);
if (kUserDictLMTBitWidth < 16) {
uint64 mask = ~(1 << kUserDictLMTBitWidth);
lmt_off &= mask;
}
// 3) now_off: current time index (current week offset for example)
// assuming load_time_ is around current time
uint64 now_off = load_time_.tv_sec;
now_off = (now_off - kUserDictLMTSince) / kUserDictLMTGranularity;
now_off = (now_off << (64 - kUserDictLMTBitWidth));
now_off = (now_off >> (64 - kUserDictLMTBitWidth));
// 4) factor: decide expand-factor
int delta = now_off - lmt_off;
if (delta > 4)
delta = 4;
int factor = 80 - (delta << 4);
double tf = (double)(dict_info_.total_nfreq + total_other_nfreq_);
return (LmaScoreType)(log((double)factor * (double)ori_freq / tf)
* NGram::kLogValueAmplifier);
}
inline int UserDict::extract_score_freq(int raw_score) {
// Frequence stored in lowest 16 bits
int freq = (raw_score & 0x0000ffff);
return freq;
}
inline uint64 UserDict::extract_score_lmt(int raw_score) {
uint64 lmt = ((raw_score & 0xffff0000) >> 16);
if (kUserDictLMTBitWidth < 16) {
uint64 mask = ~(1 << kUserDictLMTBitWidth);
lmt &= mask;
}
lmt = lmt * kUserDictLMTGranularity + kUserDictLMTSince;
return lmt;
}
inline int UserDict::build_score(uint64 lmt, int freq) {
lmt = (lmt - kUserDictLMTSince) / kUserDictLMTGranularity;
lmt = (lmt << (64 - kUserDictLMTBitWidth));
lmt = (lmt >> (64 - kUserDictLMTBitWidth));
uint16 lmt16 = (uint16)lmt;
int s = freq;
s &= 0x0000ffff;
s = (lmt16 << 16) | s;
return s;
}
inline int64 UserDict::utf16le_atoll(uint16 *s, int len) {
int64 ret = 0;
if (len <= 0)
return ret;
int flag = 1;
const uint16 * endp = s + len;
if (*s == '-') {
flag = -1;
s++;
} else if (*s == '+') {
s++;
}
while (*s >= '0' && *s <= '9' && s < endp) {
ret += ret * 10 + (*s) - '0';
s++;
}
return ret * flag;
}
inline int UserDict::utf16le_lltoa(int64 v, uint16 *s, int size) {
if (!s || size <= 0)
return 0;
uint16 *endp = s + size;
int ret_len = 0;
if (v < 0) {
*(s++) = '-';
++ret_len;
v *= -1;
}
uint16 *b = s;
while (s < endp && v != 0) {
*(s++) = '0' + (v % 10);
v = v / 10;
++ret_len;
}
if (v != 0)
return 0;
--s;
while (b < s) {
*b = *s;
++b, --s;
}
return ret_len;
}
inline void UserDict::set_lemma_flag(uint32 offset, uint8 flag) {
offset &= kUserDictOffsetMask;
lemmas_[offset] |= flag;
}
inline char UserDict::get_lemma_flag(uint32 offset) {
offset &= kUserDictOffsetMask;
return (char)(lemmas_[offset]);
}
inline char UserDict::get_lemma_nchar(uint32 offset) {
offset &= kUserDictOffsetMask;
return (char)(lemmas_[offset + 1]);
}
inline uint16 * UserDict::get_lemma_spell_ids(uint32 offset) {
offset &= kUserDictOffsetMask;
return (uint16 *)(lemmas_ + offset + 2);
}
inline uint16 * UserDict::get_lemma_word(uint32 offset) {
offset &= kUserDictOffsetMask;
uint8 nchar = get_lemma_nchar(offset);
return (uint16 *)(lemmas_ + offset + 2 + (nchar << 1));
}
inline LemmaIdType UserDict::get_max_lemma_id() {
// When a lemma is deleted, we don't not claim its id back for
// simplicity and performance
return start_id_ + dict_info_.lemma_count - 1;
}
inline bool UserDict::is_valid_lemma_id(LemmaIdType id) {
if (id >= start_id_ && id <= get_max_lemma_id())
return true;
return false;
}
inline bool UserDict::is_valid_state() {
if (state_ == USER_DICT_NONE)
return false;
return true;
}
UserDict::UserDict()
: start_id_(0),
version_(0),
lemmas_(NULL),
offsets_(NULL),
scores_(NULL),
ids_(NULL),
#ifdef ___PREDICT_ENABLED___
predicts_(NULL),
#endif
#ifdef ___SYNC_ENABLED___
syncs_(NULL),
sync_count_size_(0),
#endif
offsets_by_id_(NULL),
lemma_count_left_(0),
lemma_size_left_(0),
dict_file_(NULL),
state_(USER_DICT_NONE) {
memset(&dict_info_, 0, sizeof(dict_info_));
memset(&load_time_, 0, sizeof(load_time_));
#ifdef ___CACHE_ENABLED___
cache_init();
#endif
}
UserDict::~UserDict() {
close_dict();
}
bool UserDict::load_dict(const char *file_name, LemmaIdType start_id,
LemmaIdType end_id) {
#ifdef ___DEBUG_PERF___
DEBUG_PERF_BEGIN;
#endif
dict_file_ = strdup(file_name);
if (!dict_file_)
return false;
start_id_ = start_id;
if (false == validate(file_name) && false == reset(file_name)) {
goto error;
}
if (false == load(file_name, start_id)) {
goto error;
}
state_ = USER_DICT_SYNC;
gettimeofday(&load_time_, NULL);
#ifdef ___DEBUG_PERF___
DEBUG_PERF_END;
LOGD_PERF("load_dict");
#endif
return true;
error:
free((void*)dict_file_);
dict_file_ = NULL;
start_id_ = 0;
return false;
}
bool UserDict::close_dict() {
if (state_ == USER_DICT_NONE)
return true;
if (state_ == USER_DICT_SYNC)
goto out;
// If dictionary is written back by others,
// we can not simply write back here
// To do a safe flush, we have to discard all newly added
// lemmas and try to reload dict file.
pthread_mutex_lock(&g_mutex_);
if (load_time_.tv_sec > g_last_update_.tv_sec ||
(load_time_.tv_sec == g_last_update_.tv_sec &&
load_time_.tv_usec > g_last_update_.tv_usec)) {
write_back();
gettimeofday(&g_last_update_, NULL);
}
pthread_mutex_unlock(&g_mutex_);
out:
free((void*)dict_file_);
free(lemmas_);
free(offsets_);
free(offsets_by_id_);
free(scores_);
free(ids_);
#ifdef ___PREDICT_ENABLED___
free(predicts_);
#endif
version_ = 0;
dict_file_ = NULL;
lemmas_ = NULL;
#ifdef ___SYNC_ENABLED___
syncs_ = NULL;
sync_count_size_ = 0;
#endif
offsets_ = NULL;
offsets_by_id_ = NULL;
scores_ = NULL;
ids_ = NULL;
#ifdef ___PREDICT_ENABLED___
predicts_ = NULL;
#endif
memset(&dict_info_, 0, sizeof(dict_info_));
lemma_count_left_ = 0;
lemma_size_left_ = 0;
state_ = USER_DICT_NONE;
return true;
}
size_t UserDict::number_of_lemmas() {
return dict_info_.lemma_count;
}
void UserDict::reset_milestones(uint16 from_step, MileStoneHandle from_handle) {
return;
}
MileStoneHandle UserDict::extend_dict(MileStoneHandle from_handle,
const DictExtPara *dep,
LmaPsbItem *lpi_items,
size_t lpi_max, size_t *lpi_num) {
if (is_valid_state() == false)
return 0;
bool need_extend = false;
#ifdef ___DEBUG_PERF___
DEBUG_PERF_BEGIN;
#endif
*lpi_num = _get_lpis(dep->splids, dep->splids_extended + 1,
lpi_items, lpi_max, &need_extend);
#ifdef ___DEBUG_PERF___
DEBUG_PERF_END;
LOGD_PERF("extend_dict");
#endif
return ((*lpi_num > 0 || need_extend) ? 1 : 0);
}
int UserDict::is_fuzzy_prefix_spell_id(
const uint16 * id1, uint16 len1, const UserDictSearchable *searchable) {
if (len1 < searchable->splids_len)
return 0;
SpellingTrie &spl_trie = SpellingTrie::get_instance();
uint32 i = 0;
for (i = 0; i < searchable->splids_len; i++) {
const char py1 = *spl_trie.get_spelling_str(id1[i]);
uint16 off = 8 * (i % 4);
const char py2 = ((searchable->signature[i/4] & (0xff << off)) >> off);
if (py1 == py2)
continue;
return 0;
}
return 1;
}
int UserDict::fuzzy_compare_spell_id(
const uint16 * id1, uint16 len1, const UserDictSearchable *searchable) {
if (len1 < searchable->splids_len)
return -1;
if (len1 > searchable->splids_len)
return 1;
SpellingTrie &spl_trie = SpellingTrie::get_instance();
uint32 i = 0;
for (i = 0; i < len1; i++) {
const char py1 = *spl_trie.get_spelling_str(id1[i]);
uint16 off = 8 * (i % 4);
const char py2 = ((searchable->signature[i/4] & (0xff << off)) >> off);
if (py1 == py2)
continue;
if (py1 > py2)
return 1;
return -1;
}
return 0;
}
bool UserDict::is_prefix_spell_id(
const uint16 * fullids, uint16 fulllen,
const UserDictSearchable *searchable) {
if (fulllen < searchable->splids_len)
return false;
uint32 i = 0;
for (; i < searchable->splids_len; i++) {
uint16 start_id = searchable->splid_start[i];
uint16 count = searchable->splid_count[i];
if (fullids[i] >= start_id && fullids[i] < start_id + count)
continue;
else
return false;
}
return true;
}
bool UserDict::equal_spell_id(
const uint16 * fullids, uint16 fulllen,
const UserDictSearchable *searchable) {
if (fulllen != searchable->splids_len)
return false;
uint32 i = 0;
for (; i < fulllen; i++) {
uint16 start_id = searchable->splid_start[i];
uint16 count = searchable->splid_count[i];
if (fullids[i] >= start_id && fullids[i] < start_id + count)
continue;
else
return false;
}
return true;
}
int32 UserDict::locate_first_in_offsets(const UserDictSearchable * searchable) {
int32 begin = 0;
int32 end = dict_info_.lemma_count - 1;
int32 middle = -1;
int32 first_prefix = middle;
int32 last_matched = middle;
while (begin <= end) {
middle = (begin + end) >> 1;
uint32 offset = offsets_[middle];
uint8 nchar = get_lemma_nchar(offset);
const uint16 * splids = get_lemma_spell_ids(offset);
int cmp = fuzzy_compare_spell_id(splids, nchar, searchable);
int pre = is_fuzzy_prefix_spell_id(splids, nchar, searchable);
if (pre)
first_prefix = middle;
if (cmp < 0) {
begin = middle + 1;
} else if (cmp > 0) {
end = middle - 1;
} else {
end = middle - 1;
last_matched = middle;
}
}
return first_prefix;
}
void UserDict::prepare_locate(UserDictSearchable *searchable,
const uint16 *splid_str,
uint16 splid_str_len) {
searchable->splids_len = splid_str_len;
memset(searchable->signature, 0, sizeof(searchable->signature));
SpellingTrie &spl_trie = SpellingTrie::get_instance();
uint32 i = 0;
for (; i < splid_str_len; i++) {
if (spl_trie.is_half_id(splid_str[i])) {
searchable->splid_count[i] =
spl_trie.half_to_full(splid_str[i],
&(searchable->splid_start[i]));
} else {
searchable->splid_count[i] = 1;
searchable->splid_start[i] = splid_str[i];
}
const unsigned char py = *spl_trie.get_spelling_str(splid_str[i]);
searchable->signature[i>>2] |= (py << (8 * (i % 4)));
}
}
size_t UserDict::get_lpis(const uint16 *splid_str, uint16 splid_str_len,
LmaPsbItem *lpi_items, size_t lpi_max) {
return _get_lpis(splid_str, splid_str_len, lpi_items, lpi_max, NULL);
}
size_t UserDict::_get_lpis(const uint16 *splid_str,
uint16 splid_str_len, LmaPsbItem *lpi_items,
size_t lpi_max, bool * need_extend) {
bool tmp_extend;
if (!need_extend)
need_extend = &tmp_extend;
*need_extend = false;
if (is_valid_state() == false)
return 0;
if (lpi_max <= 0)
return 0;
if (0 == pthread_mutex_trylock(&g_mutex_)) {
if (load_time_.tv_sec < g_last_update_.tv_sec ||
(load_time_.tv_sec == g_last_update_.tv_sec &&
load_time_.tv_usec < g_last_update_.tv_usec)) {
// Others updated disk file, have to reload
pthread_mutex_unlock(&g_mutex_);
flush_cache();
} else {
pthread_mutex_unlock(&g_mutex_);
}
} else {
}
UserDictSearchable searchable;
prepare_locate(&searchable, splid_str, splid_str_len);
uint32 max_off = dict_info_.lemma_count;
#ifdef ___CACHE_ENABLED___
int32 middle;
uint32 start, count;
bool cached = cache_hit(&searchable, &start, &count);
if (cached) {
middle = start;
max_off = start + count;
} else {
middle = locate_first_in_offsets(&searchable);
start = middle;
}
#else
int32 middle = locate_first_in_offsets(&searchable);
#endif
if (middle == -1) {
#ifdef ___CACHE_ENABLED___
if (!cached)
cache_push(USER_DICT_MISS_CACHE, &searchable, 0, 0);
#endif
return 0;
}
size_t lpi_current = 0;
bool fuzzy_break = false;
bool prefix_break = false;
while ((size_t)middle < max_off && !fuzzy_break && !prefix_break) {
if (lpi_current >= lpi_max)
break;
uint32 offset = offsets_[middle];
// Ignore deleted lemmas
if (offset & kUserDictOffsetFlagRemove) {
middle++;
continue;
}
uint8 nchar = get_lemma_nchar(offset);
uint16 * splids = get_lemma_spell_ids(offset);
#ifdef ___CACHE_ENABLED___
if (!cached && 0 != fuzzy_compare_spell_id(splids, nchar, &searchable)) {
#else
if (0 != fuzzy_compare_spell_id(splids, nchar, &searchable)) {
#endif
fuzzy_break = true;
}
if (prefix_break == false) {
if (is_fuzzy_prefix_spell_id(splids, nchar, &searchable)) {
if (*need_extend == false &&
is_prefix_spell_id(splids, nchar, &searchable)) {
*need_extend = true;
}
} else {
prefix_break = true;
}
}
if (equal_spell_id(splids, nchar, &searchable) == true) {
lpi_items[lpi_current].psb = translate_score(scores_[middle]);
lpi_items[lpi_current].id = ids_[middle];
lpi_items[lpi_current].lma_len = nchar;
lpi_current++;
}
middle++;
}
#ifdef ___CACHE_ENABLED___
if (!cached) {
count = middle - start;
cache_push(USER_DICT_CACHE, &searchable, start, count);
}
#endif
return lpi_current;
}
uint16 UserDict::get_lemma_str(LemmaIdType id_lemma, char16* str_buf,
uint16 str_max) {
if (is_valid_state() == false)
return 0;
if (is_valid_lemma_id(id_lemma) == false)
return 0;
uint32 offset = offsets_by_id_[id_lemma - start_id_];
uint8 nchar = get_lemma_nchar(offset);
char16 * str = get_lemma_word(offset);
uint16 m = nchar < str_max -1 ? nchar : str_max - 1;
int i = 0;
for (; i < m; i++) {
str_buf[i] = str[i];
}
str_buf[i] = 0;
return m;
}
uint16 UserDict::get_lemma_splids(LemmaIdType id_lemma, uint16 *splids,
uint16 splids_max, bool arg_valid) {
if (is_valid_lemma_id(id_lemma) == false)
return 0;
uint32 offset = offsets_by_id_[id_lemma - start_id_];
uint8 nchar = get_lemma_nchar(offset);
const uint16 * ids = get_lemma_spell_ids(offset);
int i = 0;
for (; i < nchar && i < splids_max; i++)
splids[i] = ids[i];
return i;
}
size_t UserDict::predict(const char16 last_hzs[], uint16 hzs_len,
NPredictItem *npre_items, size_t npre_max,
size_t b4_used) {
uint32 new_added = 0;
#ifdef ___PREDICT_ENABLED___
int32 end = dict_info_.lemma_count - 1;
int j = locate_first_in_predicts((const uint16*)last_hzs, hzs_len);
if (j == -1)
return 0;
while (j <= end) {
uint32 offset = predicts_[j];
// Ignore deleted lemmas
if (offset & kUserDictOffsetFlagRemove) {
j++;
continue;
}
uint32 nchar = get_lemma_nchar(offset);
uint16 * words = get_lemma_word(offset);
uint16 * splids = get_lemma_spell_ids(offset);
if (nchar <= hzs_len) {
j++;
continue;
}
if (memcmp(words, last_hzs, hzs_len << 1) == 0) {
if (new_added >= npre_max) {
return new_added;
}
uint32 cpy_len =
(nchar < kMaxPredictSize ? (nchar << 1) : (kMaxPredictSize << 1))
- (hzs_len << 1);
npre_items[new_added].his_len = hzs_len;
npre_items[new_added].psb = get_lemma_score(words, splids, nchar);
memcpy(npre_items[new_added].pre_hzs, words + hzs_len, cpy_len);
if ((cpy_len >> 1) < kMaxPredictSize) {
npre_items[new_added].pre_hzs[cpy_len >> 1] = 0;
}
new_added++;
} else {
break;
}
j++;
}
#endif
return new_added;
}
int32 UserDict::locate_in_offsets(char16 lemma_str[], uint16 splid_str[],
uint16 lemma_len) {
int32 max_off = dict_info_.lemma_count;
UserDictSearchable searchable;
prepare_locate(&searchable, splid_str, lemma_len);
#ifdef ___CACHE_ENABLED___
int32 off;
uint32 start, count;
bool cached = load_cache(&searchable, &start, &count);
if (cached) {
off = start;
max_off = start + count;
} else {
off = locate_first_in_offsets(&searchable);
start = off;
}
#else
int32 off = locate_first_in_offsets(&searchable);
#endif
if (off == -1) {
return off;
}
while (off < max_off) {
uint32 offset = offsets_[off];
if (offset & kUserDictOffsetFlagRemove) {
off++;
continue;
}
uint16 * splids = get_lemma_spell_ids(offset);
#ifdef ___CACHE_ENABLED___
if (!cached && 0 != fuzzy_compare_spell_id(splids, lemma_len, &searchable))
break;
#else
if (0 != fuzzy_compare_spell_id(splids, lemma_len, &searchable))
break;
#endif
if (equal_spell_id(splids, lemma_len, &searchable) == true) {
uint16 * str = get_lemma_word(offset);
uint32 i = 0;
for (i = 0; i < lemma_len; i++) {
if (str[i] == lemma_str[i])
continue;
break;
}
if (i < lemma_len) {
off++;
continue;
}
#ifdef ___CACHE_ENABLED___
// No need to save_cache here, since current function is invoked by
// put_lemma. It's rarely possible for a user input same lemma twice.
// That means first time user type a new lemma, it is newly added into
// user dictionary, then it's possible that user type the same lemma
// again.
// Another reason save_cache can not be invoked here is this function
// aborts when lemma is found, and it never knows the count.
#endif
return off;
}
off++;
}
return -1;
}
#ifdef ___PREDICT_ENABLED___
uint32 UserDict::locate_where_to_insert_in_predicts(
const uint16 * words, int lemma_len) {
int32 begin = 0;
int32 end = dict_info_.lemma_count - 1;
int32 middle = end;
uint32 last_matched = middle;
while (begin <= end) {
middle = (begin + end) >> 1;
uint32 offset = offsets_[middle];
uint8 nchar = get_lemma_nchar(offset);
const uint16 * ws = get_lemma_word(offset);
uint32 minl = nchar < lemma_len ? nchar : lemma_len;
uint32 k = 0;
int cmp = 0;
for (; k < minl; k++) {
if (ws[k] < words[k]) {
cmp = -1;
break;
} else if (ws[k] > words[k]) {
cmp = 1;
break;
}
}
if (cmp == 0) {
if (nchar < lemma_len)
cmp = -1;
else if (nchar > lemma_len)
cmp = 1;
}
if (cmp < 0) {
begin = middle + 1;
last_matched = middle;
} else if (cmp > 0) {
end = middle - 1;
} else {
end = middle - 1;
last_matched = middle;
}
}
return last_matched;
}
int32 UserDict::locate_first_in_predicts(const uint16 * words, int lemma_len) {
int32 begin = 0;
int32 end = dict_info_.lemma_count - 1;
int32 middle = -1;
int32 last_matched = middle;
while (begin <= end) {
middle = (begin + end) >> 1;
uint32 offset = offsets_[middle];
uint8 nchar = get_lemma_nchar(offset);
const uint16 * ws = get_lemma_word(offset);
uint32 minl = nchar < lemma_len ? nchar : lemma_len;
uint32 k = 0;
int cmp = 0;
for (; k < minl; k++) {
if (ws[k] < words[k]) {
cmp = -1;
break;
} else if (ws[k] > words[k]) {
cmp = 1;
break;
}
}
if (cmp == 0) {
if (nchar >= lemma_len)
last_matched = middle;
if (nchar < lemma_len)
cmp = -1;
else if (nchar > lemma_len)
cmp = 1;
}
if (cmp < 0) {
begin = middle + 1;
} else if (cmp > 0) {
end = middle - 1;
} else {
end = middle - 1;
}
}
return last_matched;
}
#endif
LemmaIdType UserDict::get_lemma_id(char16 lemma_str[], uint16 splids[],
uint16 lemma_len) {
int32 off = locate_in_offsets(lemma_str, splids, lemma_len);
if (off == -1) {
return 0;
}
return ids_[off];
}
LmaScoreType UserDict::get_lemma_score(LemmaIdType lemma_id) {
if (is_valid_state() == false)
return 0;
if (is_valid_lemma_id(lemma_id) == false)
return 0;
return translate_score(_get_lemma_score(lemma_id));
}
LmaScoreType UserDict::get_lemma_score(char16 lemma_str[], uint16 splids[],
uint16 lemma_len) {
if (is_valid_state() == false)
return 0;
return translate_score(_get_lemma_score(lemma_str, splids, lemma_len));
}
int UserDict::_get_lemma_score(LemmaIdType lemma_id) {
if (is_valid_state() == false)
return 0;
if (is_valid_lemma_id(lemma_id) == false)
return 0;
uint32 offset = offsets_by_id_[lemma_id - start_id_];
uint32 nchar = get_lemma_nchar(offset);
uint16 * spl = get_lemma_spell_ids(offset);
uint16 * wrd = get_lemma_word(offset);
int32 off = locate_in_offsets(wrd, spl, nchar);
if (off == -1) {
return 0;
}
return scores_[off];
}
int UserDict::_get_lemma_score(char16 lemma_str[], uint16 splids[],
uint16 lemma_len) {
if (is_valid_state() == false)
return 0;
int32 off = locate_in_offsets(lemma_str, splids, lemma_len);
if (off == -1) {
return 0;
}
return scores_[off];
}
#ifdef ___SYNC_ENABLED___
void UserDict::remove_lemma_from_sync_list(uint32 offset) {
offset &= kUserDictOffsetMask;
uint32 i = 0;
for (; i < dict_info_.sync_count; i++) {
unsigned int off = (syncs_[i] & kUserDictOffsetMask);
if (off == offset)
break;
}
if (i < dict_info_.sync_count) {
syncs_[i] = syncs_[dict_info_.sync_count - 1];
dict_info_.sync_count--;
}
}
#endif
#ifdef ___PREDICT_ENABLED___
void UserDict::remove_lemma_from_predict_list(uint32 offset) {
offset &= kUserDictOffsetMask;
uint32 i = 0;
for (; i < dict_info_.lemma_count; i++) {
unsigned int off = (predicts_[i] & kUserDictOffsetMask);
if (off == offset) {
predicts_[i] |= kUserDictOffsetFlagRemove;
break;
}
}
}
#endif
bool UserDict::remove_lemma_by_offset_index(int offset_index) {
if (is_valid_state() == false)
return 0;
int32 off = offset_index;
if (off == -1) {
return false;
}
uint32 offset = offsets_[off];
uint32 nchar = get_lemma_nchar(offset);
offsets_[off] |= kUserDictOffsetFlagRemove;
#ifdef ___SYNC_ENABLED___
// Remove corresponding sync item
remove_lemma_from_sync_list(offset);
#endif
#ifdef ___PREDICT_ENABLED___
remove_lemma_from_predict_list(offset);
#endif
dict_info_.free_count++;
dict_info_.free_size += (2 + (nchar << 2));
if (state_ < USER_DICT_OFFSET_DIRTY)
state_ = USER_DICT_OFFSET_DIRTY;
return true;
}
bool UserDict::remove_lemma(LemmaIdType lemma_id) {
if (is_valid_state() == false)
return 0;
if (is_valid_lemma_id(lemma_id) == false)
return false;
uint32 offset = offsets_by_id_[lemma_id - start_id_];
uint32 nchar = get_lemma_nchar(offset);
uint16 * spl = get_lemma_spell_ids(offset);
uint16 * wrd = get_lemma_word(offset);
int32 off = locate_in_offsets(wrd, spl, nchar);
return remove_lemma_by_offset_index(off);
}
void UserDict::flush_cache() {
LemmaIdType start_id = start_id_;
if (!dict_file_)
return;
const char * file = strdup(dict_file_);
if (!file)
return;
close_dict();
load_dict(file, start_id, kUserDictIdEnd);
free((void*)file);
#ifdef ___CACHE_ENABLED___
cache_init();
#endif
return;
}
bool UserDict::reset(const char *file) {
FILE *fp = fopen(file, "w+");
if (!fp) {
return false;
}
uint32 version = kUserDictVersion;
size_t wred = fwrite(&version, 1, 4, fp);
UserDictInfo info;
memset(&info, 0, sizeof(info));
// By default, no limitation for lemma count and size
// thereby, reclaim_ratio is never used
wred += fwrite(&info, 1, sizeof(info), fp);
if (wred != sizeof(info) + sizeof(version)) {
fclose(fp);
unlink(file);
return false;
}
fclose(fp);
return true;
}
bool UserDict::validate(const char *file) {
// b is ignored in POSIX compatible os including Linux
// while b is important flag for Windows to specify binary mode
FILE *fp = fopen(file, "rb");
if (!fp) {
return false;
}
size_t size;
size_t readed;
uint32 version;
UserDictInfo dict_info;
// validate
int err = fseek(fp, 0, SEEK_END);
if (err) {
goto error;
}
size = ftell(fp);
if (size < 4 + sizeof(dict_info)) {
goto error;
}
err = fseek(fp, 0, SEEK_SET);
if (err) {
goto error;
}
readed = fread(&version, 1, sizeof(version), fp);
if (readed < sizeof(version)) {
goto error;
}
if (version != kUserDictVersion) {
goto error;
}
err = fseek(fp, -1 * sizeof(dict_info), SEEK_END);
if (err) {
goto error;
}
readed = fread(&dict_info, 1, sizeof(dict_info), fp);
if (readed != sizeof(dict_info)) {
goto error;
}
if (size != get_dict_file_size(&dict_info)) {
goto error;
}
fclose(fp);
return true;
error:
fclose(fp);
return false;
}
bool UserDict::load(const char *file, LemmaIdType start_id) {
if (0 != pthread_mutex_trylock(&g_mutex_)) {
return false;
}
// b is ignored in POSIX compatible os including Linux
// while b is important flag for Windows to specify binary mode
FILE *fp = fopen(file, "rb");
if (!fp) {
pthread_mutex_unlock(&g_mutex_);
return false;
}
size_t readed, toread;
UserDictInfo dict_info;
uint8 *lemmas = NULL;
uint32 *offsets = NULL;
#ifdef ___SYNC_ENABLED___
uint32 *syncs = NULL;
#endif
uint32 *scores = NULL;
uint32 *ids = NULL;
uint32 *offsets_by_id = NULL;
#ifdef ___PREDICT_ENABLED___
uint32 *predicts = NULL;
#endif
size_t i;
int err;
err = fseek(fp, -1 * sizeof(dict_info), SEEK_END);
if (err) goto error;
readed = fread(&dict_info, 1, sizeof(dict_info), fp);
if (readed != sizeof(dict_info)) goto error;
lemmas = (uint8 *)malloc(
dict_info.lemma_size +
(kUserDictPreAlloc * (2 + (kUserDictAverageNchar << 2))));
if (!lemmas) goto error;
offsets = (uint32 *)malloc((dict_info.lemma_count + kUserDictPreAlloc) << 2);
if (!offsets) goto error;
#ifdef ___PREDICT_ENABLED___
predicts = (uint32 *)malloc((dict_info.lemma_count + kUserDictPreAlloc) << 2);
if (!predicts) goto error;
#endif
#ifdef ___SYNC_ENABLED___
syncs = (uint32 *)malloc((dict_info.sync_count + kUserDictPreAlloc) << 2);
if (!syncs) goto error;
#endif
scores = (uint32 *)malloc((dict_info.lemma_count + kUserDictPreAlloc) << 2);
if (!scores) goto error;
ids = (uint32 *)malloc((dict_info.lemma_count + kUserDictPreAlloc) << 2);
if (!ids) goto error;
offsets_by_id = (uint32 *)malloc(
(dict_info.lemma_count + kUserDictPreAlloc) << 2);
if (!offsets_by_id) goto error;
err = fseek(fp, 4, SEEK_SET);
if (err) goto error;
readed = 0;
while (readed < dict_info.lemma_size && !ferror(fp) && !feof(fp)) {
readed += fread(lemmas + readed, 1, dict_info.lemma_size - readed, fp);
}
if (readed < dict_info.lemma_size)
goto error;
toread = (dict_info.lemma_count << 2);
readed = 0;
while (readed < toread && !ferror(fp) && !feof(fp)) {
readed += fread((((uint8*)offsets) + readed), 1, toread - readed, fp);
}
if (readed < toread)
goto error;
#ifdef ___PREDICT_ENABLED___
toread = (dict_info.lemma_count << 2);
readed = 0;
while (readed < toread && !ferror(fp) && !feof(fp)) {
readed += fread((((uint8*)predicts) + readed), 1, toread - readed, fp);
}
if (readed < toread)
goto error;
#endif
readed = 0;
while (readed < toread && !ferror(fp) && !feof(fp)) {
readed += fread((((uint8*)scores) + readed), 1, toread - readed, fp);
}
if (readed < toread)
goto error;
#ifdef ___SYNC_ENABLED___
toread = (dict_info.sync_count << 2);
readed = 0;
while (readed < toread && !ferror(fp) && !feof(fp)) {
readed += fread((((uint8*)syncs) + readed), 1, toread - readed, fp);
}
if (readed < toread)
goto error;
#endif
for (i = 0; i < dict_info.lemma_count; i++) {
ids[i] = start_id + i;
offsets_by_id[i] = offsets[i];
}
lemmas_ = lemmas;
offsets_ = offsets;
#ifdef ___SYNC_ENABLED___
syncs_ = syncs;
sync_count_size_ = dict_info.sync_count + kUserDictPreAlloc;
#endif
offsets_by_id_ = offsets_by_id;
scores_ = scores;
ids_ = ids;
#ifdef ___PREDICT_ENABLED___
predicts_ = predicts;
#endif
lemma_count_left_ = kUserDictPreAlloc;
lemma_size_left_ = kUserDictPreAlloc * (2 + (kUserDictAverageNchar << 2));
memcpy(&dict_info_, &dict_info, sizeof(dict_info));
state_ = USER_DICT_SYNC;
fclose(fp);
pthread_mutex_unlock(&g_mutex_);
return true;
error:
if (lemmas) free(lemmas);
if (offsets) free(offsets);
#ifdef ___SYNC_ENABLED___
if (syncs) free(syncs);
#endif
if (scores) free(scores);
if (ids) free(ids);
if (offsets_by_id) free(offsets_by_id);
#ifdef ___PREDICT_ENABLED___
if (predicts) free(predicts);
#endif
fclose(fp);
pthread_mutex_unlock(&g_mutex_);
return false;
}
void UserDict::write_back() {
// XXX write back is only allowed from close_dict due to thread-safe sake
if (state_ == USER_DICT_NONE || state_ == USER_DICT_SYNC)
return;
int fd = open(dict_file_, O_WRONLY);
if (fd == -1)
return;
switch (state_) {
case USER_DICT_DEFRAGMENTED:
write_back_all(fd);
break;
case USER_DICT_LEMMA_DIRTY:
write_back_lemma(fd);
break;
case USER_DICT_OFFSET_DIRTY:
write_back_offset(fd);
break;
case USER_DICT_SCORE_DIRTY:
write_back_score(fd);
break;
#ifdef ___SYNC_ENABLED___
case USER_DICT_SYNC_DIRTY:
write_back_sync(fd);
break;
#endif
default:
break;
}
// It seems truncate is not need on Linux, Windows except Mac
// I am doing it here anyway for safety.
off_t cur = lseek(fd, 0, SEEK_CUR);
#ifndef _WIN32
ftruncate(fd, cur);
#endif
close(fd);
state_ = USER_DICT_SYNC;
}
#ifdef ___SYNC_ENABLED___
void UserDict::write_back_sync(int fd) {
int err = lseek(fd, 4 + dict_info_.lemma_size
+ (dict_info_.lemma_count << 3)
#ifdef ___PREDICT_ENABLED___
+ (dict_info_.lemma_count << 2)
#endif
, SEEK_SET);
if (err == -1)
return;
write(fd, syncs_, dict_info_.sync_count << 2);
write(fd, &dict_info_, sizeof(dict_info_));
}
#endif
void UserDict::write_back_offset(int fd) {
int err = lseek(fd, 4 + dict_info_.lemma_size, SEEK_SET);
if (err == -1)
return;
write(fd, offsets_, dict_info_.lemma_count << 2);
#ifdef ___PREDICT_ENABLED___
write(fd, predicts_, dict_info_.lemma_count << 2);
#endif
write(fd, scores_, dict_info_.lemma_count << 2);
#ifdef ___SYNC_ENABLED___
write(fd, syncs_, dict_info_.sync_count << 2);
#endif
write(fd, &dict_info_, sizeof(dict_info_));
}
void UserDict::write_back_score(int fd) {
int err = lseek(fd, 4 + dict_info_.lemma_size
+ (dict_info_.lemma_count << 2)
#ifdef ___PREDICT_ENABLED___
+ (dict_info_.lemma_count << 2)
#endif
, SEEK_SET);
if (err == -1)
return;
write(fd, scores_, dict_info_.lemma_count << 2);
#ifdef ___SYNC_ENABLED___
write(fd, syncs_, dict_info_.sync_count << 2);
#endif
write(fd, &dict_info_, sizeof(dict_info_));
}
void UserDict::write_back_lemma(int fd) {
int err = lseek(fd, 4, SEEK_SET);
if (err == -1)
return;
// New lemmas are always appended, no need to write whole lemma block
size_t need_write = kUserDictPreAlloc *
(2 + (kUserDictAverageNchar << 2)) - lemma_size_left_;
err = lseek(fd, dict_info_.lemma_size - need_write, SEEK_CUR);
if (err == -1)
return;
write(fd, lemmas_ + dict_info_.lemma_size - need_write, need_write);
write(fd, offsets_, dict_info_.lemma_count << 2);
#ifdef ___PREDICT_ENABLED___
write(fd, predicts_, dict_info_.lemma_count << 2);
#endif
write(fd, scores_, dict_info_.lemma_count << 2);
#ifdef ___SYNC_ENABLED___
write(fd, syncs_, dict_info_.sync_count << 2);
#endif
write(fd, &dict_info_, sizeof(dict_info_));
}
void UserDict::write_back_all(int fd) {
// XXX lemma_size is handled differently in writeall
// and writelemma. I update lemma_size and lemma_count in different
// places for these two cases. Should fix it to make it consistent.
int err = lseek(fd, 4, SEEK_SET);
if (err == -1)
return;
write(fd, lemmas_, dict_info_.lemma_size);
write(fd, offsets_, dict_info_.lemma_count << 2);
#ifdef ___PREDICT_ENABLED___
write(fd, predicts_, dict_info_.lemma_count << 2);
#endif
write(fd, scores_, dict_info_.lemma_count << 2);
#ifdef ___SYNC_ENABLED___
write(fd, syncs_, dict_info_.sync_count << 2);
#endif
write(fd, &dict_info_, sizeof(dict_info_));
}
#ifdef ___CACHE_ENABLED___
bool UserDict::load_cache(UserDictSearchable *searchable,
uint32 *offset, uint32 *length) {
UserDictCache *cache = &caches_[searchable->splids_len - 1];
if (cache->head == cache->tail)
return false;
uint16 j, sig_len = kMaxLemmaSize / 4;
uint16 i = cache->head;
while (1) {
j = 0;
for (; j < sig_len; j++) {
if (cache->signatures[i][j] != searchable->signature[j])
break;
}
if (j < sig_len) {
i++;
if (i >= kUserDictCacheSize)
i -= kUserDictCacheSize;
if (i == cache->tail)
break;
continue;
}
*offset = cache->offsets[i];
*length = cache->lengths[i];
return true;
}
return false;
}
void UserDict::save_cache(UserDictSearchable *searchable,
uint32 offset, uint32 length) {
UserDictCache *cache = &caches_[searchable->splids_len - 1];
uint16 next = cache->tail;
cache->offsets[next] = offset;
cache->lengths[next] = length;
uint16 sig_len = kMaxLemmaSize / 4;
uint16 j = 0;
for (; j < sig_len; j++) {
cache->signatures[next][j] = searchable->signature[j];
}
if (++next >= kUserDictCacheSize) {
next -= kUserDictCacheSize;
}
if (next == cache->head) {
cache->head++;
if (cache->head >= kUserDictCacheSize) {
cache->head -= kUserDictCacheSize;
}
}
cache->tail = next;
}
void UserDict::reset_cache() {
memset(caches_, 0, sizeof(caches_));
}
bool UserDict::load_miss_cache(UserDictSearchable *searchable) {
UserDictMissCache *cache = &miss_caches_[searchable->splids_len - 1];
if (cache->head == cache->tail)
return false;
uint16 j, sig_len = kMaxLemmaSize / 4;
uint16 i = cache->head;
while (1) {
j = 0;
for (; j < sig_len; j++) {
if (cache->signatures[i][j] != searchable->signature[j])
break;
}
if (j < sig_len) {
i++;
if (i >= kUserDictMissCacheSize)
i -= kUserDictMissCacheSize;
if (i == cache->tail)
break;
continue;
}
return true;
}
return false;
}
void UserDict::save_miss_cache(UserDictSearchable *searchable) {
UserDictMissCache *cache = &miss_caches_[searchable->splids_len - 1];
uint16 next = cache->tail;
uint16 sig_len = kMaxLemmaSize / 4;
uint16 j = 0;
for (; j < sig_len; j++) {
cache->signatures[next][j] = searchable->signature[j];
}
if (++next >= kUserDictMissCacheSize) {
next -= kUserDictMissCacheSize;
}
if (next == cache->head) {
cache->head++;
if (cache->head >= kUserDictMissCacheSize) {
cache->head -= kUserDictMissCacheSize;
}
}
cache->tail = next;
}
void UserDict::reset_miss_cache() {
memset(miss_caches_, 0, sizeof(miss_caches_));
}
void UserDict::cache_init() {
reset_cache();
reset_miss_cache();
}
bool UserDict::cache_hit(UserDictSearchable *searchable,
uint32 *offset, uint32 *length) {
bool hit = load_miss_cache(searchable);
if (hit) {
*offset = 0;
*length = 0;
return true;
}
hit = load_cache(searchable, offset, length);
if (hit) {
return true;
}
return false;
}
void UserDict::cache_push(UserDictCacheType type,
UserDictSearchable *searchable,
uint32 offset, uint32 length) {
switch (type) {
case USER_DICT_MISS_CACHE:
save_miss_cache(searchable);
break;
case USER_DICT_CACHE:
save_cache(searchable, offset, length);
break;
default:
break;
}
}
#endif
void UserDict::defragment(void) {
#ifdef ___DEBUG_PERF___
DEBUG_PERF_BEGIN;
#endif
if (is_valid_state() == false)
return;
// Fixup offsets_, set REMOVE flag to lemma's flag if needed
size_t first_freed = 0;
size_t first_inuse = 0;
while (first_freed < dict_info_.lemma_count) {
// Find first freed offset
while ((offsets_[first_freed] & kUserDictOffsetFlagRemove) == 0 &&
first_freed < dict_info_.lemma_count) {
first_freed++;
}
if (first_freed < dict_info_.lemma_count) {
// Save REMOVE flag to lemma flag
int off = offsets_[first_freed];
set_lemma_flag(off, kUserDictLemmaFlagRemove);
} else {
break;
}
// Find first inuse offse after first_freed
first_inuse = first_freed + 1;
while ((offsets_[first_inuse] & kUserDictOffsetFlagRemove) &&
(first_inuse < dict_info_.lemma_count)) {
// Save REMOVE flag to lemma flag
int off = offsets_[first_inuse];
set_lemma_flag(off, kUserDictLemmaFlagRemove);
first_inuse++;
}
if (first_inuse >= dict_info_.lemma_count) {
break;
}
// Swap offsets_
int tmp = offsets_[first_inuse];
offsets_[first_inuse] = offsets_[first_freed];
offsets_[first_freed] = tmp;
// Move scores_, no need to swap
tmp = scores_[first_inuse];
scores_[first_inuse] = scores_[first_freed];
scores_[first_freed] = tmp;
// Swap ids_
LemmaIdType tmpid = ids_[first_inuse];
ids_[first_inuse] = ids_[first_freed];
ids_[first_freed] = tmpid;
// Go on
first_freed++;
}
#ifdef ___PREDICT_ENABLED___
// Fixup predicts_
first_freed = 0;
first_inuse = 0;
while (first_freed < dict_info_.lemma_count) {
// Find first freed offset
while ((predicts_[first_freed] & kUserDictOffsetFlagRemove) == 0 &&
first_freed < dict_info_.lemma_count) {
first_freed++;
}
if (first_freed >= dict_info_.lemma_count)
break;
// Find first inuse offse after first_freed
first_inuse = first_freed + 1;
while ((predicts_[first_inuse] & kUserDictOffsetFlagRemove)
&& (first_inuse < dict_info_.lemma_count)) {
first_inuse++;
}
if (first_inuse >= dict_info_.lemma_count) {
break;
}
// Swap offsets_
int tmp = predicts_[first_inuse];
predicts_[first_inuse] = predicts_[first_freed];
predicts_[first_freed] = tmp;
// Go on
first_freed++;
}
#endif
dict_info_.lemma_count = first_freed;
// Fixup lemmas_
size_t begin = 0;
size_t end = 0;
size_t dst = 0;
int total_size = dict_info_.lemma_size + lemma_size_left_;
int total_count = dict_info_.lemma_count + lemma_count_left_;
size_t real_size = total_size - lemma_size_left_;
while (dst < real_size) {
unsigned char flag = get_lemma_flag(dst);
unsigned char nchr = get_lemma_nchar(dst);
if ((flag & kUserDictLemmaFlagRemove) == 0) {
dst += nchr * 4 + 2;
continue;
}
break;
}
if (dst >= real_size)
return;
end = dst;
while (end < real_size) {
begin = end + get_lemma_nchar(end) * 4 + 2;
repeat:
// not used any more
if (begin >= real_size)
break;
unsigned char flag = get_lemma_flag(begin);
unsigned char nchr = get_lemma_nchar(begin);
if (flag & kUserDictLemmaFlagRemove) {
begin += nchr * 4 + 2;
goto repeat;
}
end = begin + nchr * 4 + 2;
while (end < real_size) {
unsigned char eflag = get_lemma_flag(end);
unsigned char enchr = get_lemma_nchar(end);
if ((eflag & kUserDictLemmaFlagRemove) == 0) {
end += enchr * 4 + 2;
continue;
}
break;
}
memmove(lemmas_ + dst, lemmas_ + begin, end - begin);
for (size_t j = 0; j < dict_info_.lemma_count; j++) {
if (offsets_[j] >= begin && offsets_[j] < end) {
offsets_[j] -= (begin - dst);
offsets_by_id_[ids_[j] - start_id_] = offsets_[j];
}
#ifdef ___PREDICT_ENABLED___
if (predicts_[j] >= begin && predicts_[j] < end) {
predicts_[j] -= (begin - dst);
}
#endif
}
#ifdef ___SYNC_ENABLED___
for (size_t j = 0; j < dict_info_.sync_count; j++) {
if (syncs_[j] >= begin && syncs_[j] < end) {
syncs_[j] -= (begin - dst);
}
}
#endif
dst += (end - begin);
}
dict_info_.free_count = 0;
dict_info_.free_size = 0;
dict_info_.lemma_size = dst;
lemma_size_left_ = total_size - dict_info_.lemma_size;
lemma_count_left_ = total_count - dict_info_.lemma_count;
// XXX Without following code,
// offsets_by_id_ is not reordered.
// That's to say, all removed lemmas' ids are not collected back.
// There may not be room for addition of new lemmas due to
// offsests_by_id_ reason, although lemma_size_left_ is fixed.
// By default, we do want defrag as fast as possible, because
// during defrag procedure, other peers can not write new lemmas
// to user dictionary file.
// XXX If write-back is invoked immediately after
// this defragment, no need to fix up following in-mem data.
for (uint32 i = 0; i < dict_info_.lemma_count; i++) {
ids_[i] = start_id_ + i;
offsets_by_id_[i] = offsets_[i];
}
state_ = USER_DICT_DEFRAGMENTED;
#ifdef ___DEBUG_PERF___
DEBUG_PERF_END;
LOGD_PERF("defragment");
#endif
}
#ifdef ___SYNC_ENABLED___
void UserDict::clear_sync_lemmas(unsigned int start, unsigned int end) {
if (is_valid_state() == false)
return;
if (end > dict_info_.sync_count)
end = dict_info_.sync_count;
memmove(syncs_ + start, syncs_ + end, (dict_info_.sync_count - end) << 2);
dict_info_.sync_count -= (end - start);
if (state_ < USER_DICT_SYNC_DIRTY)
state_ = USER_DICT_SYNC_DIRTY;
}
int UserDict::get_sync_count() {
if (is_valid_state() == false)
return 0;
return dict_info_.sync_count;
}
LemmaIdType UserDict::put_lemma_no_sync(char16 lemma_str[], uint16 splids[],
uint16 lemma_len, uint16 count, uint64 lmt) {
int again = 0;
begin:
LemmaIdType id;
uint32 * syncs_bak = syncs_;
syncs_ = NULL;
id = _put_lemma(lemma_str, splids, lemma_len, count, lmt);
syncs_ = syncs_bak;
if (id == 0 && again == 0) {
if ((dict_info_.limit_lemma_count > 0 &&
dict_info_.lemma_count >= dict_info_.limit_lemma_count)
|| (dict_info_.limit_lemma_size > 0 &&
dict_info_.lemma_size + (2 + (lemma_len << 2))
> dict_info_.limit_lemma_size)) {
// XXX Always reclaim and defrag in sync code path
// sync thread is background thread and ok with heavy work
reclaim();
defragment();
flush_cache();
again = 1;
goto begin;
}
}
return id;
}
int UserDict::put_lemmas_no_sync_from_utf16le_string(char16 * lemmas, int len) {
int newly_added = 0;
SpellingParser * spl_parser = new SpellingParser();
if (!spl_parser) {
return 0;
}
#ifdef ___DEBUG_PERF___
DEBUG_PERF_BEGIN;
#endif
char16 *ptr = lemmas;
// Extract pinyin,words,frequence,last_mod_time
char16 * p = ptr, * py16 = ptr;
char16 * hz16 = NULL;
int py16_len = 0;
uint16 splid[kMaxLemmaSize];
int splid_len = 0;
int hz16_len = 0;
char16 * fr16 = NULL;
int fr16_len = 0;
while (p - ptr < len) {
// Pinyin
py16 = p;
splid_len = 0;
while (*p != 0x2c && (p - ptr) < len) {
if (*p == 0x20)
splid_len++;
p++;
}
splid_len++;
if (p - ptr == len)
break;
py16_len = p - py16;
if (kMaxLemmaSize < splid_len) {
break;
}
bool is_pre;
int splidl = spl_parser->splstr16_to_idxs_f(
py16, py16_len, splid, NULL, kMaxLemmaSize, is_pre);
if (splidl != splid_len)
break;
// Phrase
hz16 = ++p;
while (*p != 0x2c && (p - ptr) < len) {
p++;
}
hz16_len = p - hz16;
if (hz16_len != splid_len)
break;
// Frequency
fr16 = ++p;
fr16_len = 0;
while (*p != 0x2c && (p - ptr) < len) {
p++;
}
fr16_len = p - fr16;
uint32 intf = (uint32)utf16le_atoll(fr16, fr16_len);
// Last modified time
fr16 = ++p;
fr16_len = 0;
while (*p != 0x3b && (p - ptr) < len) {
p++;
}
fr16_len = p - fr16;
uint64 last_mod = utf16le_atoll(fr16, fr16_len);
put_lemma_no_sync(hz16, splid, splid_len, intf, last_mod);
newly_added++;
p++;
}
#ifdef ___DEBUG_PERF___
DEBUG_PERF_END;
LOGD_PERF("put_lemmas_no_sync_from_utf16le_string");
#endif
return newly_added;
}
int UserDict::get_sync_lemmas_in_utf16le_string_from_beginning(
char16 * str, int size, int * count) {
int len = 0;
*count = 0;
int left_len = size;
if (is_valid_state() == false)
return len;
SpellingTrie * spl_trie = &SpellingTrie::get_instance();
if (!spl_trie) {
return 0;
}
uint32 i;
for (i = 0; i < dict_info_.sync_count; i++) {
int offset = syncs_[i];
uint32 nchar = get_lemma_nchar(offset);
uint16 *spl = get_lemma_spell_ids(offset);
uint16 *wrd = get_lemma_word(offset);
int score = _get_lemma_score(wrd, spl, nchar);
static char score_temp[32], *pscore_temp = score_temp;
static char16 temp[256], *ptemp = temp;
pscore_temp = score_temp;
ptemp = temp;
uint32 j;
// Add pinyin
for (j = 0; j < nchar; j++) {
int ret_len = spl_trie->get_spelling_str16(
spl[j], ptemp, temp + sizeof(temp) - ptemp);
if (ret_len <= 0)
break;
ptemp += ret_len;
if (ptemp < temp + sizeof(temp) - 1) {
*(ptemp++) = ' ';
} else {
j = 0;
break;
}
}
if (j < nchar) {
continue;
}
ptemp--;
if (ptemp < temp + sizeof(temp) - 1) {
*(ptemp++) = ',';
} else {
continue;
}
// Add phrase
for (j = 0; j < nchar; j++) {
if (ptemp < temp + sizeof(temp) - 1) {
*(ptemp++) = wrd[j];
} else {
break;
}
}
if (j < nchar) {
continue;
}
if (ptemp < temp + sizeof(temp) - 1) {
*(ptemp++) = ',';
} else {
continue;
}
// Add frequency
uint32 intf = extract_score_freq(score);
int ret_len = utf16le_lltoa(intf, ptemp, temp + sizeof(temp) - ptemp);
if (ret_len <= 0)
continue;
ptemp += ret_len;
if (ptemp < temp + sizeof(temp) - 1) {
*(ptemp++) = ',';
} else {
continue;
}
// Add last modified time
uint64 last_mod = extract_score_lmt(score);
ret_len = utf16le_lltoa(last_mod, ptemp, temp + sizeof(temp) - ptemp);
if (ret_len <= 0)
continue;
ptemp += ret_len;
if (ptemp < temp + sizeof(temp) - 1) {
*(ptemp++) = ';';
} else {
continue;
}
// Write to string
int need_len = ptemp - temp;
if (need_len > left_len)
break;
memcpy(str + len, temp, need_len * 2);
left_len -= need_len;
len += need_len;
(*count)++;
}
if (len > 0) {
if (state_ < USER_DICT_SYNC_DIRTY)
state_ = USER_DICT_SYNC_DIRTY;
}
return len;
}
#endif
bool UserDict::state(UserDictStat * stat) {
if (is_valid_state() == false)
return false;
if (!stat)
return false;
stat->version = version_;
stat->file_name = dict_file_;
stat->load_time.tv_sec = load_time_.tv_sec;
stat->load_time.tv_usec = load_time_.tv_usec;
pthread_mutex_lock(&g_mutex_);
stat->last_update.tv_sec = g_last_update_.tv_sec;
stat->last_update.tv_usec = g_last_update_.tv_usec;
pthread_mutex_unlock(&g_mutex_);
stat->disk_size = get_dict_file_size(&dict_info_);
stat->lemma_count = dict_info_.lemma_count;
stat->lemma_size = dict_info_.lemma_size;
stat->delete_count = dict_info_.free_count;
stat->delete_size = dict_info_.free_size;
#ifdef ___SYNC_ENABLED___
stat->sync_count = dict_info_.sync_count;
#endif
stat->limit_lemma_count = dict_info_.limit_lemma_count;
stat->limit_lemma_size = dict_info_.limit_lemma_size;
stat->reclaim_ratio = dict_info_.reclaim_ratio;
return true;
}
void UserDict::set_limit(uint32 max_lemma_count,
uint32 max_lemma_size, uint32 reclaim_ratio) {
dict_info_.limit_lemma_count = max_lemma_count;
dict_info_.limit_lemma_size = max_lemma_size;
if (reclaim_ratio > 100)
reclaim_ratio = 100;
dict_info_.reclaim_ratio = reclaim_ratio;
}
void UserDict::reclaim() {
if (is_valid_state() == false)
return;
switch (dict_info_.reclaim_ratio) {
case 0:
return;
case 100:
// TODO: CLEAR to be implemented
assert(false);
return;
default:
break;
}
// XXX Reclaim is only based on count, not size
uint32 count = dict_info_.lemma_count;
int rc = count * dict_info_.reclaim_ratio / 100;
UserDictScoreOffsetPair * score_offset_pairs = NULL;
score_offset_pairs = (UserDictScoreOffsetPair *)malloc(
sizeof(UserDictScoreOffsetPair) * rc);
if (score_offset_pairs == NULL) {
return;
}
for (int i = 0; i < rc; i++) {
int s = scores_[i];
score_offset_pairs[i].score = s;
score_offset_pairs[i].offset_index = i;
}
for (int i = (rc + 1) / 2; i >= 0; i--)
shift_down(score_offset_pairs, i, rc);
for (uint32 i = rc; i < dict_info_.lemma_count; i++) {
int s = scores_[i];
if (s < score_offset_pairs[0].score) {
score_offset_pairs[0].score = s;
score_offset_pairs[0].offset_index = i;
shift_down(score_offset_pairs, 0, rc);
}
}
for (int i = 0; i < rc; i++) {
int off = score_offset_pairs[i].offset_index;
remove_lemma_by_offset_index(off);
}
if (rc > 0) {
if (state_ < USER_DICT_OFFSET_DIRTY)
state_ = USER_DICT_OFFSET_DIRTY;
}
free(score_offset_pairs);
}
inline void UserDict::swap(UserDictScoreOffsetPair * sop, int i, int j) {
int s = sop[i].score;
int p = sop[i].offset_index;
sop[i].score = sop[j].score;
sop[i].offset_index = sop[j].offset_index;
sop[j].score = s;
sop[j].offset_index = p;
}
void UserDict::shift_down(UserDictScoreOffsetPair * sop, int i, int n) {
int par = i;
while (par < n) {
int left = par * 2 + 1;
int right = left + 1;
if (left >= n && right >= n)
break;
if (right >= n) {
if (sop[left].score > sop[par].score) {
swap(sop, left, par);
par = left;
continue;
}
} else if (sop[left].score > sop[right].score &&
sop[left].score > sop[par].score) {
swap(sop, left, par);
par = left;
continue;
} else if (sop[right].score > sop[left].score &&
sop[right].score > sop[par].score) {
swap(sop, right, par);
par = right;
continue;
}
break;
}
}
LemmaIdType UserDict::put_lemma(char16 lemma_str[], uint16 splids[],
uint16 lemma_len, uint16 count) {
return _put_lemma(lemma_str, splids, lemma_len, count, time(NULL));
}
LemmaIdType UserDict::_put_lemma(char16 lemma_str[], uint16 splids[],
uint16 lemma_len, uint16 count, uint64 lmt) {
#ifdef ___DEBUG_PERF___
DEBUG_PERF_BEGIN;
#endif
if (is_valid_state() == false)
return 0;
int32 off = locate_in_offsets(lemma_str, splids, lemma_len);
if (off != -1) {
int delta_score = count - scores_[off];
dict_info_.total_nfreq += delta_score;
scores_[off] = build_score(lmt, count);
if (state_ < USER_DICT_SCORE_DIRTY)
state_ = USER_DICT_SCORE_DIRTY;
#ifdef ___DEBUG_PERF___
DEBUG_PERF_END;
LOGD_PERF("_put_lemma(update)");
#endif
return ids_[off];
} else {
if ((dict_info_.limit_lemma_count > 0 &&
dict_info_.lemma_count >= dict_info_.limit_lemma_count)
|| (dict_info_.limit_lemma_size > 0 &&
dict_info_.lemma_size + (2 + (lemma_len << 2))
> dict_info_.limit_lemma_size)) {
// XXX Don't defragment here, it's too time-consuming.
return 0;
}
int flushed = 0;
if (lemma_count_left_ == 0 ||
lemma_size_left_ < (size_t)(2 + (lemma_len << 2))) {
// XXX When there is no space for new lemma, we flush to disk
// flush_cache() may be called by upper user
// and better place shoule be found instead of here
flush_cache();
flushed = 1;
// Or simply return and do nothing
// return 0;
}
#ifdef ___DEBUG_PERF___
DEBUG_PERF_END;
LOGD_PERF(flushed ? "_put_lemma(flush+add)" : "_put_lemma(add)");
#endif
LemmaIdType id = append_a_lemma(lemma_str, splids, lemma_len, count, lmt);
#ifdef ___SYNC_ENABLED___
if (syncs_ && id != 0) {
queue_lemma_for_sync(id);
}
#endif
return id;
}
return 0;
}
#ifdef ___SYNC_ENABLED___
void UserDict::queue_lemma_for_sync(LemmaIdType id) {
if (dict_info_.sync_count < sync_count_size_) {
syncs_[dict_info_.sync_count++] = offsets_by_id_[id - start_id_];
} else {
uint32 * syncs = (uint32*)realloc(
syncs_, (sync_count_size_ + kUserDictPreAlloc) << 2);
if (syncs) {
sync_count_size_ += kUserDictPreAlloc;
syncs_ = syncs;
syncs_[dict_info_.sync_count++] = offsets_by_id_[id - start_id_];
}
}
}
#endif
LemmaIdType UserDict::update_lemma(LemmaIdType lemma_id, int16 delta_count,
bool selected) {
#ifdef ___DEBUG_PERF___
DEBUG_PERF_BEGIN;
#endif
if (is_valid_state() == false)
return 0;
if (is_valid_lemma_id(lemma_id) == false)
return 0;
uint32 offset = offsets_by_id_[lemma_id - start_id_];
uint8 lemma_len = get_lemma_nchar(offset);
char16 * lemma_str = get_lemma_word(offset);
uint16 * splids = get_lemma_spell_ids(offset);
int32 off = locate_in_offsets(lemma_str, splids, lemma_len);
if (off != -1) {
int score = scores_[off];
int count = extract_score_freq(score);
uint64 lmt = extract_score_lmt(score);
if (count + delta_count > kUserDictMaxFrequency ||
count + delta_count < count) {
delta_count = kUserDictMaxFrequency - count;
}
count += delta_count;
dict_info_.total_nfreq += delta_count;
if (selected) {
lmt = time(NULL);
}
scores_[off] = build_score(lmt, count);
if (state_ < USER_DICT_SCORE_DIRTY)
state_ = USER_DICT_SCORE_DIRTY;
#ifdef ___DEBUG_PERF___
DEBUG_PERF_END;
LOGD_PERF("update_lemma");
#endif
#ifdef ___SYNC_ENABLED___
queue_lemma_for_sync(ids_[off]);
#endif
return ids_[off];
}
return 0;
}
size_t UserDict::get_total_lemma_count() {
return dict_info_.total_nfreq;
}
void UserDict::set_total_lemma_count_of_others(size_t count) {
total_other_nfreq_ = count;
}
LemmaIdType UserDict::append_a_lemma(char16 lemma_str[], uint16 splids[],
uint16 lemma_len, uint16 count, uint64 lmt) {
LemmaIdType id = get_max_lemma_id() + 1;
size_t offset = dict_info_.lemma_size;
if (offset > kUserDictOffsetMask)
return 0;
lemmas_[offset] = 0;
lemmas_[offset + 1] = (uint8)lemma_len;
for (size_t i = 0; i < lemma_len; i++) {
*((uint16*)&lemmas_[offset + 2 + (i << 1)]) = splids[i];
*((char16*)&lemmas_[offset + 2 + (lemma_len << 1) + (i << 1)])
= lemma_str[i];
}
uint32 off = dict_info_.lemma_count;
offsets_[off] = offset;
scores_[off] = build_score(lmt, count);
ids_[off] = id;
#ifdef ___PREDICT_ENABLED___
predicts_[off] = offset;
#endif
offsets_by_id_[id - start_id_] = offset;
dict_info_.lemma_count++;
dict_info_.lemma_size += (2 + (lemma_len << 2));
lemma_count_left_--;
lemma_size_left_ -= (2 + (lemma_len << 2));
// Sort
UserDictSearchable searchable;
prepare_locate(&searchable, splids, lemma_len);
size_t i = 0;
while (i < off) {
offset = offsets_[i];
uint32 nchar = get_lemma_nchar(offset);
uint16 * spl = get_lemma_spell_ids(offset);
if (0 <= fuzzy_compare_spell_id(spl, nchar, &searchable))
break;
i++;
}
if (i != off) {
uint32 temp = offsets_[off];
memmove(offsets_ + i + 1, offsets_ + i, (off - i) << 2);
offsets_[i] = temp;
temp = scores_[off];
memmove(scores_ + i + 1, scores_ + i, (off - i) << 2);
scores_[i] = temp;
temp = ids_[off];
memmove(ids_ + i + 1, ids_ + i, (off - i) << 2);
ids_[i] = temp;
}
#ifdef ___PREDICT_ENABLED___
uint32 j = 0;
uint16 * words_new = get_lemma_word(predicts_[off]);
j = locate_where_to_insert_in_predicts(words_new, lemma_len);
if (j != off) {
uint32 temp = predicts_[off];
memmove(predicts_ + j + 1, predicts_ + j, (off - j) << 2);
predicts_[j] = temp;
}
#endif
if (state_ < USER_DICT_LEMMA_DIRTY)
state_ = USER_DICT_LEMMA_DIRTY;
#ifdef ___CACHE_ENABLED___
cache_init();
#endif
dict_info_.total_nfreq += count;
return id;
}
}
Reference in New Issue View Git Blame Copy Permalink