1 /*
2  * Copyright (C) 2013 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 /*
18  * !!!!! DO NOT EDIT THIS FILE !!!!!
19  *
20  * This file was generated from
21  *   dictionary/structure/v4/content/bigram_dict_content.cpp
22  */
23 
24 #include "dictionary/structure/backward/v402/content/bigram_dict_content.h"
25 
26 #include "dictionary/utils/buffer_with_extendable_buffer.h"
27 
28 namespace latinime {
29 namespace backward {
30 namespace v402 {
31 
getBigramEntryAndAdvancePosition(int * const bigramEntryPos) const32 const BigramEntry BigramDictContent::getBigramEntryAndAdvancePosition(
33         int *const bigramEntryPos) const {
34     const BufferWithExtendableBuffer *const bigramListBuffer = getContentBuffer();
35     const int bigramEntryTailPos = (*bigramEntryPos) + getBigramEntrySize();
36     if (*bigramEntryPos < 0 || bigramEntryTailPos > bigramListBuffer->getTailPosition()) {
37         AKLOGE("Invalid bigram entry position. bigramEntryPos: %d, bigramEntryTailPos: %d, "
38                 "bufSize: %d", *bigramEntryPos, bigramEntryTailPos,
39                         bigramListBuffer->getTailPosition());
40         ASSERT(false);
41         return BigramEntry(false /* hasNext */, NOT_A_PROBABILITY,
42                 Ver4DictConstants::NOT_A_TERMINAL_ID);
43     }
44     const int bigramFlags = bigramListBuffer->readUintAndAdvancePosition(
45             Ver4DictConstants::BIGRAM_FLAGS_FIELD_SIZE, bigramEntryPos);
46     const bool hasNext = (bigramFlags & Ver4DictConstants::BIGRAM_HAS_NEXT_MASK) != 0;
47     int probability = NOT_A_PROBABILITY;
48     int timestamp = NOT_A_TIMESTAMP;
49     int level = 0;
50     int count = 0;
51     if (mHasHistoricalInfo) {
52         timestamp = bigramListBuffer->readUintAndAdvancePosition(
53                 Ver4DictConstants::TIME_STAMP_FIELD_SIZE, bigramEntryPos);
54         level = bigramListBuffer->readUintAndAdvancePosition(
55                 Ver4DictConstants::WORD_LEVEL_FIELD_SIZE, bigramEntryPos);
56         count = bigramListBuffer->readUintAndAdvancePosition(
57                 Ver4DictConstants::WORD_COUNT_FIELD_SIZE, bigramEntryPos);
58     } else {
59         probability = bigramListBuffer->readUintAndAdvancePosition(
60                 Ver4DictConstants::PROBABILITY_SIZE, bigramEntryPos);
61     }
62     const int encodedTargetTerminalId = bigramListBuffer->readUintAndAdvancePosition(
63             Ver4DictConstants::BIGRAM_TARGET_TERMINAL_ID_FIELD_SIZE, bigramEntryPos);
64     const int targetTerminalId =
65             (encodedTargetTerminalId == Ver4DictConstants::INVALID_BIGRAM_TARGET_TERMINAL_ID) ?
66                     Ver4DictConstants::NOT_A_TERMINAL_ID : encodedTargetTerminalId;
67     if (mHasHistoricalInfo) {
68         // Hack for better migration.
69         count += level;
70         const HistoricalInfo historicalInfo(timestamp, level, count);
71         return BigramEntry(hasNext, probability, &historicalInfo, targetTerminalId);
72     } else {
73         return BigramEntry(hasNext, probability, targetTerminalId);
74     }
75 }
76 
writeBigramEntryAndAdvancePosition(const BigramEntry * const bigramEntryToWrite,int * const entryWritingPos)77 bool BigramDictContent::writeBigramEntryAndAdvancePosition(
78         const BigramEntry *const bigramEntryToWrite, int *const entryWritingPos) {
79     BufferWithExtendableBuffer *const bigramListBuffer = getWritableContentBuffer();
80     const int bigramFlags = createAndGetBigramFlags(bigramEntryToWrite->hasNext());
81     if (!bigramListBuffer->writeUintAndAdvancePosition(bigramFlags,
82             Ver4DictConstants::BIGRAM_FLAGS_FIELD_SIZE, entryWritingPos)) {
83         AKLOGE("Cannot write bigram flags. pos: %d, flags: %x", *entryWritingPos, bigramFlags);
84         return false;
85     }
86     if (mHasHistoricalInfo) {
87         const HistoricalInfo *const historicalInfo = bigramEntryToWrite->getHistoricalInfo();
88         if (!bigramListBuffer->writeUintAndAdvancePosition(historicalInfo->getTimestamp(),
89                 Ver4DictConstants::TIME_STAMP_FIELD_SIZE, entryWritingPos)) {
90             AKLOGE("Cannot write bigram timestamps. pos: %d, timestamp: %d", *entryWritingPos,
91                     historicalInfo->getTimestamp());
92             return false;
93         }
94         if (!bigramListBuffer->writeUintAndAdvancePosition(historicalInfo->getLevel(),
95                 Ver4DictConstants::WORD_LEVEL_FIELD_SIZE, entryWritingPos)) {
96             AKLOGE("Cannot write bigram level. pos: %d, level: %d", *entryWritingPos,
97                     historicalInfo->getLevel());
98             return false;
99         }
100         if (!bigramListBuffer->writeUintAndAdvancePosition(historicalInfo->getCount(),
101                 Ver4DictConstants::WORD_COUNT_FIELD_SIZE, entryWritingPos)) {
102             AKLOGE("Cannot write bigram count. pos: %d, count: %d", *entryWritingPos,
103                     historicalInfo->getCount());
104             return false;
105         }
106     } else {
107         if (!bigramListBuffer->writeUintAndAdvancePosition(bigramEntryToWrite->getProbability(),
108                 Ver4DictConstants::PROBABILITY_SIZE, entryWritingPos)) {
109             AKLOGE("Cannot write bigram probability. pos: %d, probability: %d", *entryWritingPos,
110                     bigramEntryToWrite->getProbability());
111             return false;
112         }
113     }
114     const int targetTerminalIdToWrite =
115             (bigramEntryToWrite->getTargetTerminalId() == Ver4DictConstants::NOT_A_TERMINAL_ID) ?
116                     Ver4DictConstants::INVALID_BIGRAM_TARGET_TERMINAL_ID :
117                             bigramEntryToWrite->getTargetTerminalId();
118     if (!bigramListBuffer->writeUintAndAdvancePosition(targetTerminalIdToWrite,
119             Ver4DictConstants::BIGRAM_TARGET_TERMINAL_ID_FIELD_SIZE, entryWritingPos)) {
120         AKLOGE("Cannot write bigram target terminal id. pos: %d, target terminal id: %d",
121                 *entryWritingPos, bigramEntryToWrite->getTargetTerminalId());
122         return false;
123     }
124     return true;
125 }
126 
copyBigramList(const int bigramListPos,const int toPos,int * const outTailEntryPos)127 bool BigramDictContent::copyBigramList(const int bigramListPos, const int toPos,
128         int *const outTailEntryPos) {
129     int readingPos = bigramListPos;
130     int writingPos = toPos;
131     bool hasNext = true;
132     while (hasNext) {
133         const BigramEntry bigramEntry = getBigramEntryAndAdvancePosition(&readingPos);
134         hasNext = bigramEntry.hasNext();
135         if (!hasNext) {
136             *outTailEntryPos = writingPos;
137         }
138         if (!writeBigramEntryAndAdvancePosition(&bigramEntry, &writingPos)) {
139             AKLOGE("Cannot write bigram entry to copy. pos: %d", writingPos);
140             return false;
141         }
142     }
143     return true;
144 }
145 
runGC(const TerminalPositionLookupTable::TerminalIdMap * const terminalIdMap,const BigramDictContent * const originalBigramDictContent,int * const outBigramEntryCount)146 bool BigramDictContent::runGC(const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap,
147         const BigramDictContent *const originalBigramDictContent,
148         int *const outBigramEntryCount) {
149     for (TerminalPositionLookupTable::TerminalIdMap::const_iterator it = terminalIdMap->begin();
150             it != terminalIdMap->end(); ++it) {
151         const int originalBigramListPos =
152                 originalBigramDictContent->getBigramListHeadPos(it->first);
153         if (originalBigramListPos == NOT_A_DICT_POS) {
154             // This terminal does not have a bigram list.
155             continue;
156         }
157         const int bigramListPos = getContentBuffer()->getTailPosition();
158         int bigramEntryCount = 0;
159         // Copy bigram list with GC from original content.
160         if (!runGCBigramList(originalBigramListPos, originalBigramDictContent, bigramListPos,
161                 terminalIdMap, &bigramEntryCount)) {
162             AKLOGE("Cannot complete GC for the bigram list. original pos: %d, pos: %d",
163                     originalBigramListPos, bigramListPos);
164             return false;
165         }
166         if (bigramEntryCount == 0) {
167             // All bigram entries are useless. This terminal does not have a bigram list.
168             continue;
169         }
170         *outBigramEntryCount += bigramEntryCount;
171         // Set bigram list position to the lookup table.
172         if (!getUpdatableAddressLookupTable()->set(it->second, bigramListPos)) {
173             AKLOGE("Cannot set bigram list position. terminal id: %d, pos: %d",
174                     it->second, bigramListPos);
175             return false;
176         }
177     }
178     return true;
179 }
180 
181 // Returns whether GC for the bigram list was succeeded or not.
runGCBigramList(const int bigramListPos,const BigramDictContent * const sourceBigramDictContent,const int toPos,const TerminalPositionLookupTable::TerminalIdMap * const terminalIdMap,int * const outEntrycount)182 bool BigramDictContent::runGCBigramList(const int bigramListPos,
183         const BigramDictContent *const sourceBigramDictContent, const int toPos,
184         const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap,
185         int *const outEntrycount) {
186     bool hasNext = true;
187     int readingPos = bigramListPos;
188     int writingPos = toPos;
189     int lastEntryPos = NOT_A_DICT_POS;
190     while (hasNext) {
191         const BigramEntry originalBigramEntry =
192                 sourceBigramDictContent->getBigramEntryAndAdvancePosition(&readingPos);
193         hasNext = originalBigramEntry.hasNext();
194         if (originalBigramEntry.getTargetTerminalId() == Ver4DictConstants::NOT_A_TERMINAL_ID) {
195             continue;
196         }
197         TerminalPositionLookupTable::TerminalIdMap::const_iterator it =
198                 terminalIdMap->find(originalBigramEntry.getTargetTerminalId());
199         if (it == terminalIdMap->end()) {
200             // Target word has been removed.
201             continue;
202         }
203         lastEntryPos = hasNext ? writingPos : NOT_A_DICT_POS;
204         const BigramEntry updatedBigramEntry =
205                 originalBigramEntry.updateTargetTerminalIdAndGetEntry(it->second);
206         if (!writeBigramEntryAndAdvancePosition(&updatedBigramEntry, &writingPos)) {
207             AKLOGE("Cannot write bigram entry to run GC. pos: %d", writingPos);
208             return false;
209         }
210         *outEntrycount += 1;
211     }
212     if (lastEntryPos != NOT_A_DICT_POS) {
213         // Update has next flag in the last written entry.
214         const BigramEntry bigramEntry = getBigramEntry(lastEntryPos).updateHasNextAndGetEntry(
215                 false /* hasNext */);
216         if (!writeBigramEntry(&bigramEntry, lastEntryPos)) {
217             AKLOGE("Cannot write bigram entry to set hasNext flag after GC. pos: %d", writingPos);
218             return false;
219         }
220     }
221     return true;
222 }
223 
224 } // namespace v402
225 } // namespace backward
226 } // namespace latinime
227