1 /*
2  * Copyright (C) 2010 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #ifndef _UTILS_TOKENIZER_H
18 #define _UTILS_TOKENIZER_H
19 
20 #include <assert.h>
21 #include <utils/Errors.h>
22 #include <utils/FileMap.h>
23 #include <utils/String8.h>
24 
25 namespace android {
26 
27 /**
28  * A simple tokenizer for loading and parsing ASCII text files line by line.
29  */
30 class Tokenizer {
31     Tokenizer(const String8& filename, FileMap* fileMap, char* buffer,
32             bool ownBuffer, size_t length);
33 
34 public:
35     ~Tokenizer();
36 
37     /**
38      * Opens a file and maps it into memory.
39      *
40      * Returns OK and a tokenizer for the file, if successful.
41      * Otherwise returns an error and sets outTokenizer to NULL.
42      */
43     static status_t open(const String8& filename, Tokenizer** outTokenizer);
44 
45     /**
46      * Prepares to tokenize the contents of a string.
47      *
48      * Returns OK and a tokenizer for the string, if successful.
49      * Otherwise returns an error and sets outTokenizer to NULL.
50      */
51     static status_t fromContents(const String8& filename,
52             const char* contents, Tokenizer** outTokenizer);
53 
54     /**
55      * Returns true if at the end of the file.
56      */
isEof()57     inline bool isEof() const { return mCurrent == getEnd(); }
58 
59     /**
60      * Returns true if at the end of the line or end of the file.
61      */
isEol()62     inline bool isEol() const { return isEof() || *mCurrent == '\n'; }
63 
64     /**
65      * Gets the name of the file.
66      */
getFilename()67     inline String8 getFilename() const { return mFilename; }
68 
69     /**
70      * Gets a 1-based line number index for the current position.
71      */
getLineNumber()72     inline int32_t getLineNumber() const { return mLineNumber; }
73 
74     /**
75      * Formats a location string consisting of the filename and current line number.
76      * Returns a string like "MyFile.txt:33".
77      */
78     String8 getLocation() const;
79 
80     /**
81      * Gets the character at the current position.
82      * Returns null at end of file.
83      */
peekChar()84     inline char peekChar() const { return isEof() ? '\0' : *mCurrent; }
85 
86     /**
87      * Gets the remainder of the current line as a string, excluding the newline character.
88      */
89     String8 peekRemainderOfLine() const;
90 
91     /**
92      * Gets the character at the current position and advances past it.
93      * Returns null at end of file.
94      */
nextChar()95     inline char nextChar() { return isEof() ? '\0' : *(mCurrent++); }
96 
97     /**
98      * Gets the next token on this line stopping at the specified delimiters
99      * or the end of the line whichever comes first and advances past it.
100      * Also stops at embedded nulls.
101      * Returns the token or an empty string if the current character is a delimiter
102      * or is at the end of the line.
103      */
104     String8 nextToken(const char* delimiters);
105 
106     /**
107      * Advances to the next line.
108      * Does nothing if already at the end of the file.
109      */
110     void nextLine();
111 
112     /**
113      * Skips over the specified delimiters in the line.
114      * Also skips embedded nulls.
115      */
116     void skipDelimiters(const char* delimiters);
117 
118 private:
119     Tokenizer(const Tokenizer& other); // not copyable
120 
121     String8 mFilename;
122     FileMap* mFileMap;
123     char* mBuffer;
124     bool mOwnBuffer;
125     size_t mLength;
126 
127     const char* mCurrent;
128     int32_t mLineNumber;
129 
getEnd()130     inline const char* getEnd() const { return mBuffer + mLength; }
131 
132 };
133 
134 } // namespace android
135 
136 #endif // _UTILS_TOKENIZER_H
137