1 // Tencent is pleased to support the open source community by making RapidJSON available.
2 //
3 // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved.
4 //
5 // Licensed under the MIT License (the "License"); you may not use this file except
6 // in compliance with the License. You may obtain a copy of the License at
7 //
8 // http://opensource.org/licenses/MIT
9 //
10 // Unless required by applicable law or agreed to in writing, software distributed
11 // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
12 // CONDITIONS OF ANY KIND, either express or implied. See the License for the
13 // specific language governing permissions and limitations under the License.
14
15 #include "unittest.h"
16 #include "rapidjson/filereadstream.h"
17 #include "rapidjson/filewritestream.h"
18 #include "rapidjson/encodedstream.h"
19 #include "rapidjson/stringbuffer.h"
20 #include "rapidjson/memorystream.h"
21 #include "rapidjson/memorybuffer.h"
22
23 using namespace rapidjson;
24
25 class EncodedStreamTest : public ::testing::Test {
26 public:
EncodedStreamTest()27 EncodedStreamTest() : json_(), length_() {}
28
SetUp()29 virtual void SetUp() {
30 json_ = ReadFile("utf8.json", true, &length_);
31 }
32
TearDown()33 virtual void TearDown() {
34 free(json_);
35 json_ = 0;
36 }
37
38 private:
39 EncodedStreamTest(const EncodedStreamTest&);
40 EncodedStreamTest& operator=(const EncodedStreamTest&);
41
42 protected:
Open(const char * filename)43 static FILE* Open(const char* filename) {
44 const char *paths[] = {
45 "encodings/%s",
46 "bin/encodings/%s",
47 "../bin/encodings/%s",
48 "../../bin/encodings/%s",
49 "../../../bin/encodings/%s"
50 };
51 char buffer[1024];
52 for (size_t i = 0; i < sizeof(paths) / sizeof(paths[0]); i++) {
53 sprintf(buffer, paths[i], filename);
54 FILE *fp = fopen(buffer, "rb");
55 if (fp)
56 return fp;
57 }
58 return 0;
59 }
60
ReadFile(const char * filename,bool appendPath,size_t * outLength)61 static char *ReadFile(const char* filename, bool appendPath, size_t* outLength) {
62 FILE *fp = appendPath ? Open(filename) : fopen(filename, "rb");
63
64 if (!fp) {
65 *outLength = 0;
66 return 0;
67 }
68
69 fseek(fp, 0, SEEK_END);
70 *outLength = (size_t)ftell(fp);
71 fseek(fp, 0, SEEK_SET);
72 char* buffer = (char*)malloc(*outLength + 1);
73 size_t readLength = fread(buffer, 1, *outLength, fp);
74 buffer[readLength] = '\0';
75 fclose(fp);
76 return buffer;
77 }
78
79 template <typename FileEncoding, typename MemoryEncoding>
TestEncodedInputStream(const char * filename)80 void TestEncodedInputStream(const char* filename) {
81 // Test FileReadStream
82 {
83 char buffer[16];
84 FILE *fp = Open(filename);
85 ASSERT_TRUE(fp != 0);
86 FileReadStream fs(fp, buffer, sizeof(buffer));
87 EncodedInputStream<FileEncoding, FileReadStream> eis(fs);
88 StringStream s(json_);
89
90 while (eis.Peek() != '\0') {
91 unsigned expected, actual;
92 EXPECT_TRUE(UTF8<>::Decode(s, &expected));
93 EXPECT_TRUE(MemoryEncoding::Decode(eis, &actual));
94 EXPECT_EQ(expected, actual);
95 }
96 EXPECT_EQ('\0', s.Peek());
97 fclose(fp);
98 }
99
100 // Test MemoryStream
101 {
102 size_t size;
103 char* data = ReadFile(filename, true, &size);
104 MemoryStream ms(data, size);
105 EncodedInputStream<FileEncoding, MemoryStream> eis(ms);
106 StringStream s(json_);
107
108 while (eis.Peek() != '\0') {
109 unsigned expected, actual;
110 EXPECT_TRUE(UTF8<>::Decode(s, &expected));
111 EXPECT_TRUE(MemoryEncoding::Decode(eis, &actual));
112 EXPECT_EQ(expected, actual);
113 }
114 EXPECT_EQ('\0', s.Peek());
115 free(data);
116 EXPECT_EQ(size, eis.Tell());
117 }
118 }
119
TestAutoUTFInputStream(const char * filename,bool expectHasBOM)120 void TestAutoUTFInputStream(const char *filename, bool expectHasBOM) {
121 // Test FileReadStream
122 {
123 char buffer[16];
124 FILE *fp = Open(filename);
125 ASSERT_TRUE(fp != 0);
126 FileReadStream fs(fp, buffer, sizeof(buffer));
127 AutoUTFInputStream<unsigned, FileReadStream> eis(fs);
128 EXPECT_EQ(expectHasBOM, eis.HasBOM());
129 StringStream s(json_);
130 while (eis.Peek() != '\0') {
131 unsigned expected, actual;
132 EXPECT_TRUE(UTF8<>::Decode(s, &expected));
133 EXPECT_TRUE(AutoUTF<unsigned>::Decode(eis, &actual));
134 EXPECT_EQ(expected, actual);
135 }
136 EXPECT_EQ('\0', s.Peek());
137 fclose(fp);
138 }
139
140 // Test MemoryStream
141 {
142 size_t size;
143 char* data = ReadFile(filename, true, &size);
144 MemoryStream ms(data, size);
145 AutoUTFInputStream<unsigned, MemoryStream> eis(ms);
146 EXPECT_EQ(expectHasBOM, eis.HasBOM());
147 StringStream s(json_);
148
149 while (eis.Peek() != '\0') {
150 unsigned expected, actual;
151 EXPECT_TRUE(UTF8<>::Decode(s, &expected));
152 EXPECT_TRUE(AutoUTF<unsigned>::Decode(eis, &actual));
153 EXPECT_EQ(expected, actual);
154 }
155 EXPECT_EQ('\0', s.Peek());
156 free(data);
157 EXPECT_EQ(size, eis.Tell());
158 }
159 }
160
161 template <typename FileEncoding, typename MemoryEncoding>
TestEncodedOutputStream(const char * expectedFilename,bool putBOM)162 void TestEncodedOutputStream(const char* expectedFilename, bool putBOM) {
163 // Test FileWriteStream
164 {
165 char filename[L_tmpnam];
166 FILE* fp = TempFile(filename);
167 char buffer[16];
168 FileWriteStream os(fp, buffer, sizeof(buffer));
169 EncodedOutputStream<FileEncoding, FileWriteStream> eos(os, putBOM);
170 StringStream s(json_);
171 while (s.Peek() != '\0') {
172 bool success = Transcoder<UTF8<>, MemoryEncoding>::Transcode(s, eos);
173 EXPECT_TRUE(success);
174 }
175 eos.Flush();
176 fclose(fp);
177 EXPECT_TRUE(CompareFile(filename, expectedFilename));
178 remove(filename);
179 }
180
181 // Test MemoryBuffer
182 {
183 MemoryBuffer mb;
184 EncodedOutputStream<FileEncoding, MemoryBuffer> eos(mb, putBOM);
185 StringStream s(json_);
186 while (s.Peek() != '\0') {
187 bool success = Transcoder<UTF8<>, MemoryEncoding>::Transcode(s, eos);
188 EXPECT_TRUE(success);
189 }
190 eos.Flush();
191 EXPECT_TRUE(CompareBufferFile(mb.GetBuffer(), mb.GetSize(), expectedFilename));
192 }
193 }
194
TestAutoUTFOutputStream(UTFType type,bool putBOM,const char * expectedFilename)195 void TestAutoUTFOutputStream(UTFType type, bool putBOM, const char *expectedFilename) {
196 // Test FileWriteStream
197 {
198 char filename[L_tmpnam];
199 FILE* fp = TempFile(filename);
200
201 char buffer[16];
202 FileWriteStream os(fp, buffer, sizeof(buffer));
203 AutoUTFOutputStream<unsigned, FileWriteStream> eos(os, type, putBOM);
204 StringStream s(json_);
205 while (s.Peek() != '\0') {
206 bool success = Transcoder<UTF8<>, AutoUTF<unsigned> >::Transcode(s, eos);
207 EXPECT_TRUE(success);
208 }
209 eos.Flush();
210 fclose(fp);
211 EXPECT_TRUE(CompareFile(filename, expectedFilename));
212 remove(filename);
213 }
214
215 // Test MemoryBuffer
216 {
217 MemoryBuffer mb;
218 AutoUTFOutputStream<unsigned, MemoryBuffer> eos(mb, type, putBOM);
219 StringStream s(json_);
220 while (s.Peek() != '\0') {
221 bool success = Transcoder<UTF8<>, AutoUTF<unsigned> >::Transcode(s, eos);
222 EXPECT_TRUE(success);
223 }
224 eos.Flush();
225 EXPECT_TRUE(CompareBufferFile(mb.GetBuffer(), mb.GetSize(), expectedFilename));
226 }
227 }
228
CompareFile(const char * filename,const char * expectedFilename)229 bool CompareFile(const char* filename, const char* expectedFilename) {
230 size_t actualLength, expectedLength;
231 char* actualBuffer = ReadFile(filename, false, &actualLength);
232 char* expectedBuffer = ReadFile(expectedFilename, true, &expectedLength);
233 bool ret = (expectedLength == actualLength) && memcmp(expectedBuffer, actualBuffer, actualLength) == 0;
234 free(actualBuffer);
235 free(expectedBuffer);
236 return ret;
237 }
238
CompareBufferFile(const char * actualBuffer,size_t actualLength,const char * expectedFilename)239 bool CompareBufferFile(const char* actualBuffer, size_t actualLength, const char* expectedFilename) {
240 size_t expectedLength;
241 char* expectedBuffer = ReadFile(expectedFilename, true, &expectedLength);
242 bool ret = (expectedLength == actualLength) && memcmp(expectedBuffer, actualBuffer, actualLength) == 0;
243 free(expectedBuffer);
244 return ret;
245 }
246
247 char *json_;
248 size_t length_;
249 };
250
TEST_F(EncodedStreamTest,EncodedInputStream)251 TEST_F(EncodedStreamTest, EncodedInputStream) {
252 TestEncodedInputStream<UTF8<>, UTF8<> >("utf8.json");
253 TestEncodedInputStream<UTF8<>, UTF8<> >("utf8bom.json");
254 TestEncodedInputStream<UTF16LE<>, UTF16<> >("utf16le.json");
255 TestEncodedInputStream<UTF16LE<>, UTF16<> >("utf16lebom.json");
256 TestEncodedInputStream<UTF16BE<>, UTF16<> >("utf16be.json");
257 TestEncodedInputStream<UTF16BE<>, UTF16<> >("utf16bebom.json");
258 TestEncodedInputStream<UTF32LE<>, UTF32<> >("utf32le.json");
259 TestEncodedInputStream<UTF32LE<>, UTF32<> >("utf32lebom.json");
260 TestEncodedInputStream<UTF32BE<>, UTF32<> >("utf32be.json");
261 TestEncodedInputStream<UTF32BE<>, UTF32<> >("utf32bebom.json");
262 }
263
TEST_F(EncodedStreamTest,AutoUTFInputStream)264 TEST_F(EncodedStreamTest, AutoUTFInputStream) {
265 TestAutoUTFInputStream("utf8.json", false);
266 TestAutoUTFInputStream("utf8bom.json", true);
267 TestAutoUTFInputStream("utf16le.json", false);
268 TestAutoUTFInputStream("utf16lebom.json",true);
269 TestAutoUTFInputStream("utf16be.json", false);
270 TestAutoUTFInputStream("utf16bebom.json",true);
271 TestAutoUTFInputStream("utf32le.json", false);
272 TestAutoUTFInputStream("utf32lebom.json",true);
273 TestAutoUTFInputStream("utf32be.json", false);
274 TestAutoUTFInputStream("utf32bebom.json", true);
275
276 {
277 // Auto detection fail, use user defined UTF type
278 const char json[] = "{ }";
279 MemoryStream ms(json, sizeof(json));
280 AutoUTFInputStream<unsigned, MemoryStream> eis(ms, kUTF8);
281 EXPECT_FALSE(eis.HasBOM());
282 EXPECT_EQ(kUTF8, eis.GetType());
283 }
284 }
285
TEST_F(EncodedStreamTest,EncodedOutputStream)286 TEST_F(EncodedStreamTest, EncodedOutputStream) {
287 TestEncodedOutputStream<UTF8<>, UTF8<> >("utf8.json", false);
288 TestEncodedOutputStream<UTF8<>, UTF8<> >("utf8bom.json", true);
289 TestEncodedOutputStream<UTF16LE<>, UTF16<> >("utf16le.json", false);
290 TestEncodedOutputStream<UTF16LE<>, UTF16<> >("utf16lebom.json",true);
291 TestEncodedOutputStream<UTF16BE<>, UTF16<> >("utf16be.json", false);
292 TestEncodedOutputStream<UTF16BE<>, UTF16<> >("utf16bebom.json",true);
293 TestEncodedOutputStream<UTF32LE<>, UTF32<> >("utf32le.json", false);
294 TestEncodedOutputStream<UTF32LE<>, UTF32<> >("utf32lebom.json",true);
295 TestEncodedOutputStream<UTF32BE<>, UTF32<> >("utf32be.json", false);
296 TestEncodedOutputStream<UTF32BE<>, UTF32<> >("utf32bebom.json",true);
297 }
298
TEST_F(EncodedStreamTest,AutoUTFOutputStream)299 TEST_F(EncodedStreamTest, AutoUTFOutputStream) {
300 TestAutoUTFOutputStream(kUTF8, false, "utf8.json");
301 TestAutoUTFOutputStream(kUTF8, true, "utf8bom.json");
302 TestAutoUTFOutputStream(kUTF16LE, false, "utf16le.json");
303 TestAutoUTFOutputStream(kUTF16LE, true, "utf16lebom.json");
304 TestAutoUTFOutputStream(kUTF16BE, false, "utf16be.json");
305 TestAutoUTFOutputStream(kUTF16BE, true, "utf16bebom.json");
306 TestAutoUTFOutputStream(kUTF32LE, false, "utf32le.json");
307 TestAutoUTFOutputStream(kUTF32LE, true, "utf32lebom.json");
308 TestAutoUTFOutputStream(kUTF32BE, false, "utf32be.json");
309 TestAutoUTFOutputStream(kUTF32BE, true, "utf32bebom.json");
310 }
311