1#!/usr/bin/env python3
2#
3#   Copyright 2019 - The Android Open Source Project
4#
5#   Licensed under the Apache License, Version 2.0 (the "License");
6#   you may not use this file except in compliance with the License.
7#   You may obtain a copy of the License at
8#
9#       http://www.apache.org/licenses/LICENSE-2.0
10#
11#   Unless required by applicable law or agreed to in writing, software
12#   distributed under the License is distributed on an "AS IS" BASIS,
13#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14#   See the License for the specific language governing permissions and
15#   limitations under the License.
16'''Python Module for GNSS test log utilities.'''
17
18import re as regex
19import datetime
20import functools as fts
21import numpy as npy
22import pandas as pds
23from acts import logger
24
25# GPS API Log Reading Config
26CONFIG_GPSAPILOG = {
27    'phone_time':
28    r'^(?P<date>\d+\/\d+\/\d+)\s+(?P<time>\d+:\d+:\d+)\s+'
29    r'Read:\s+(?P<logsize>\d+)\s+bytes',
30    'SpaceVehicle':
31    r'^Fix:\s+(?P<Fix>\w+)\s+Type:\s+(?P<Type>\w+)\s+'
32    r'SV:\s+(?P<SV>\d+)\s+C\/No:\s+(?P<CNo>\d+\.\d+)\s+'
33    r'Elevation:\s+(?P<Elevation>\d+\.\d+)\s+'
34    r'Azimuth:\s+(?P<Azimuth>\d+\.\d+)\s+'
35    r'Signal:\s+(?P<Signal>\w+)\s+'
36    r'Frequency:\s+(?P<Frequency>\d+\.\d+)\s+'
37    r'EPH:\s+(?P<EPH>\w+)\s+ALM:\s+(?P<ALM>\w+)',
38    'SpaceVehicle_wBB':
39    r'^Fix:\s+(?P<Fix>\w+)\s+Type:\s+(?P<Type>\w+)\s+'
40    r'SV:\s+(?P<SV>\d+)\s+C\/No:\s+(?P<AntCNo>\d+\.\d+),\s+'
41    r'(?P<BbCNo>\d+\.\d+)\s+'
42    r'Elevation:\s+(?P<Elevation>\d+\.\d+)\s+'
43    r'Azimuth:\s+(?P<Azimuth>\d+\.\d+)\s+'
44    r'Signal:\s+(?P<Signal>\w+)\s+'
45    r'Frequency:\s+(?P<Frequency>\d+\.\d+)\s+'
46    r'EPH:\s+(?P<EPH>\w+)\s+ALM:\s+(?P<ALM>\w+)',
47    'HistoryAvgTop4CNo':
48    r'^History\s+Avg\s+Top4\s+:\s+(?P<HistoryAvgTop4CNo>\d+\.\d+)',
49    'CurrentAvgTop4CNo':
50    r'^Current\s+Avg\s+Top4\s+:\s+(?P<CurrentAvgTop4CNo>\d+\.\d+)',
51    'HistoryAvgCNo':
52    r'^History\s+Avg\s+:\s+(?P<HistoryAvgCNo>\d+\.\d+)',
53    'CurrentAvgCNo':
54    r'^Current\s+Avg\s+:\s+(?P<CurrentAvgCNo>\d+\.\d+)',
55    'AntennaHistoryAvgTop4CNo':
56    r'^Antenna_History\s+Avg\s+Top4\s+:\s+(?P<AntennaHistoryAvgTop4CNo>\d+\.\d+)',
57    'AntennaCurrentAvgTop4CNo':
58    r'^Antenna_Current\s+Avg\s+Top4\s+:\s+(?P<AntennaCurrentAvgTop4CNo>\d+\.\d+)',
59    'AntennaHistoryAvgCNo':
60    r'^Antenna_History\s+Avg\s+:\s+(?P<AntennaHistoryAvgCNo>\d+\.\d+)',
61    'AntennaCurrentAvgCNo':
62    r'^Antenna_Current\s+Avg\s+:\s+(?P<AntennaCurrentAvgCNo>\d+\.\d+)',
63    'BasebandHistoryAvgTop4CNo':
64    r'^Baseband_History\s+Avg\s+Top4\s+:\s+(?P<BasebandHistoryAvgTop4CNo>\d+\.\d+)',
65    'BasebandCurrentAvgTop4CNo':
66    r'^Baseband_Current\s+Avg\s+Top4\s+:\s+(?P<BasebandCurrentAvgTop4CNo>\d+\.\d+)',
67    'BasebandHistoryAvgCNo':
68    r'^Baseband_History\s+Avg\s+:\s+(?P<BasebandHistoryAvgCNo>\d+\.\d+)',
69    'BasebandCurrentAvgCNo':
70    r'^Baseband_Current\s+Avg\s+:\s+(?P<BasebandCurrentAvgCNo>\d+\.\d+)',
71    'L5inFix':
72    r'^L5\s+used\s+in\s+fix:\s+(?P<L5inFix>\w+)',
73    'L5EngagingRate':
74    r'^L5\s+engaging\s+rate:\s+(?P<L5EngagingRate>\d+.\d+)%',
75    'Provider':
76    r'^Provider:\s+(?P<Provider>\w+)',
77    'Latitude':
78    r'^Latitude:\s+(?P<Latitude>-?\d+.\d+)',
79    'Longitude':
80    r'^Longitude:\s+(?P<Longitude>-?\d+.\d+)',
81    'Altitude':
82    r'^Altitude:\s+(?P<Altitude>-?\d+.\d+)',
83    'GNSSTime':
84    r'^Time:\s+(?P<Date>\d+\/\d+\/\d+)\s+'
85    r'(?P<Time>\d+:\d+:\d+)',
86    'Speed':
87    r'^Speed:\s+(?P<Speed>\d+.\d+)',
88    'Bearing':
89    r'^Bearing:\s+(?P<Bearing>\d+.\d+)',
90}
91
92# Space Vehicle Statistics Dataframe List
93# Handle the pre GPSTool 2.12.24 case
94LIST_SVSTAT = [
95    'HistoryAvgTop4CNo', 'CurrentAvgTop4CNo', 'HistoryAvgCNo', 'CurrentAvgCNo',
96    'L5inFix', 'L5EngagingRate'
97]
98# Handle the post GPSTool 2.12.24 case with baseband CNo
99LIST_SVSTAT_WBB = [
100    'AntennaHistoryAvgTop4CNo', 'AntennaCurrentAvgTop4CNo',
101    'AntennaHistoryAvgCNo', 'AntennaCurrentAvgCNo',
102    'BasebandHistoryAvgTop4CNo', 'BasebandCurrentAvgTop4CNo',
103    'BasebandHistoryAvgCNo', 'BasebandCurrentAvgCNo', 'L5inFix',
104    'L5EngagingRate'
105]
106
107# Location Fix Info Dataframe List
108LIST_LOCINFO = [
109    'Provider', 'Latitude', 'Longitude', 'Altitude', 'GNSSTime', 'Speed',
110    'Bearing'
111]
112
113# GPS TTFF Log Reading Config
114CONFIG_GPSTTFFLOG = {
115    'ttff_info':
116    r'Loop:(?P<loop>\d+)\s+'
117    r'(?P<start_datetime>\d+\/\d+\/\d+-\d+:\d+:\d+.\d+)\s+'
118    r'(?P<stop_datetime>\d+\/\d+\/\d+-\d+:\d+:\d+.\d+)\s+'
119    r'(?P<ttff>\d+.\d+)\s+'
120    r'\[Avg Top4 : (?P<avg_top4_cn0>\d+.\d+)\]\s'
121    r'\[Avg : (?P<avg_cn0>\d+.\d+)\]\s+\[(?P<fix_type>\d+\w+ fix)\]\s+'
122    r'\[Satellites used for fix : (?P<satnum_for_fix>\d+)\]'
123}
124
125LOGPARSE_UTIL_LOGGER = logger.create_logger()
126
127
128def parse_log_to_df(filename, configs, index_rownum=True):
129    r"""Parse log to a dictionary of Pandas dataframes.
130
131    Args:
132      filename: log file name.
133        Type String.
134      configs: configs dictionary of parsed Pandas dataframes.
135        Type dictionary.
136        dict key, the parsed pattern name, such as 'Speed',
137        dict value, regex of the config pattern,
138          Type Raw String.
139      index_rownum: index row number from raw data.
140        Type Boolean.
141        Default, True.
142
143    Returns:
144      parsed_data: dictionary of parsed data.
145        Type dictionary.
146        dict key, the parsed pattern name, such as 'Speed',
147        dict value, the corresponding parsed dataframe.
148
149    Examples:
150      configs = {
151          'GNSSTime':
152          r'Time:\s+(?P<Date>\d+\/\d+\/\d+)\s+
153          r(?P<Time>\d+:\d+:\d+)')},
154          'Speed': r'Speed:\s+(?P<Speed>\d+.\d+)',
155      }
156    """
157    # Init a local config dictionary to hold compiled regex and match dict.
158    configs_local = {}
159    # Construct parsed data dictionary
160    parsed_data = {}
161
162    # Loop the config dictionary to compile regex and init data list
163    for key, regex_string in configs.items():
164        configs_local[key] = {
165            'cregex': regex.compile(regex_string),
166            'datalist': [],
167        }
168
169    # Open the file, loop and parse
170    with open(filename, 'r') as fid:
171
172        for idx_line, current_line in enumerate(fid):
173            for _, config in configs_local.items():
174                matched_log_object = config['cregex'].search(current_line)
175
176                if matched_log_object:
177                    matched_data = matched_log_object.groupdict()
178                    matched_data['rownumber'] = idx_line + 1
179                    config['datalist'].append(matched_data)
180
181    # Loop to generate parsed data from configs list
182    for key, config in configs_local.items():
183        parsed_data[key] = pds.DataFrame(config['datalist'])
184        if index_rownum and not parsed_data[key].empty:
185            parsed_data[key].set_index('rownumber', inplace=True)
186        elif parsed_data[key].empty:
187            LOGPARSE_UTIL_LOGGER.debug(
188                'The parsed dataframe of "%s" is empty.', key)
189
190    # Return parsed data list
191    return parsed_data
192
193
194def parse_gpstool_ttfflog_to_df(filename):
195    """Parse GPSTool ttff log to Pandas dataframes.
196
197    Args:
198      filename: full log file name.
199        Type, String.
200
201    Returns:
202      ttff_df: TTFF Data Frame.
203        Type, Pandas DataFrame.
204    """
205    # Get parsed dataframe list
206    parsed_data = parse_log_to_df(
207        filename=filename,
208        configs=CONFIG_GPSTTFFLOG,
209    )
210    ttff_df = parsed_data['ttff_info']
211
212    # Data Conversion
213    ttff_df['loop'] = ttff_df['loop'].astype(int)
214    ttff_df['start_datetime'] = pds.to_datetime(ttff_df['start_datetime'])
215    ttff_df['stop_datetime'] = pds.to_datetime(ttff_df['stop_datetime'])
216    ttff_df['ttff'] = ttff_df['ttff'].astype(float)
217    ttff_df['avg_top4_cn0'] = ttff_df['avg_top4_cn0'].astype(float)
218    ttff_df['avg_cn0'] = ttff_df['avg_cn0'].astype(float)
219    ttff_df['satnum_for_fix'] = ttff_df['satnum_for_fix'].astype(int)
220
221    # return ttff dataframe
222    return ttff_df
223
224
225def parse_gpsapilog_to_df(filename):
226    """Parse GPS API log to Pandas dataframes.
227
228    Args:
229      filename: full log file name.
230        Type, String.
231
232    Returns:
233      timestamp_df: Timestamp Data Frame.
234        Type, Pandas DataFrame.
235      sv_info_df: GNSS SV info Data Frame.
236        Type, Pandas DataFrame.
237      sv_stat_df: GNSS SV statistic Data Frame.
238        Type, Pandas DataFrame
239      loc_info_df: Location Information Data Frame.
240        Type, Pandas DataFrame.
241        include Provider, Latitude, Longitude, Altitude, GNSSTime, Speed, Bearing
242    """
243    def get_phone_time(target_df_row, timestamp_df):
244        """subfunction to get the phone_time."""
245
246        try:
247            row_num = timestamp_df[
248                timestamp_df.index < target_df_row.name].iloc[-1].name
249            phone_time = timestamp_df.loc[row_num]['phone_time']
250        except IndexError:
251            row_num = npy.NaN
252            phone_time = npy.NaN
253
254        return phone_time, row_num
255
256    # Get parsed dataframe list
257    parsed_data = parse_log_to_df(
258        filename=filename,
259        configs=CONFIG_GPSAPILOG,
260    )
261
262    # get DUT Timestamp
263    timestamp_df = parsed_data['phone_time']
264    timestamp_df['phone_time'] = timestamp_df.apply(
265        lambda row: datetime.datetime.strptime(row.date + '-' + row.time,
266                                               '%Y/%m/%d-%H:%M:%S'),
267        axis=1)
268
269    # Add phone_time from timestamp_df dataframe by row number
270    for key in parsed_data:
271        if key != 'phone_time':
272            current_df = parsed_data[key]
273            time_n_row_num = current_df.apply(get_phone_time,
274                                              axis=1,
275                                              timestamp_df=timestamp_df)
276            current_df[['phone_time', 'time_row_num'
277                        ]] = pds.DataFrame(time_n_row_num.apply(pds.Series))
278
279    # Get space vehicle info dataframe
280    sv_info_df = parsed_data['SpaceVehicle']
281
282    # Get space vehicle statistics dataframe
283    # First merge all dataframe from LIST_SVSTAT[1:],
284    # Drop duplicated 'phone_time', based on time_row_num
285    sv_stat_df = fts.reduce(
286        lambda item1, item2: pds.merge(item1, item2, on='time_row_num'), [
287            parsed_data[key].drop(['phone_time'], axis=1)
288            for key in LIST_SVSTAT[1:]
289        ])
290    # Then merge with LIST_SVSTAT[0]
291    sv_stat_df = pds.merge(sv_stat_df,
292                           parsed_data[LIST_SVSTAT[0]],
293                           on='time_row_num')
294
295    # Get location fix information dataframe
296    # First merge all dataframe from LIST_LOCINFO[1:],
297    # Drop duplicated 'phone_time', based on time_row_num
298    loc_info_df = fts.reduce(
299        lambda item1, item2: pds.merge(item1, item2, on='time_row_num'), [
300            parsed_data[key].drop(['phone_time'], axis=1)
301            for key in LIST_LOCINFO[1:]
302        ])
303    # Then merge with LIST_LOCINFO[8]
304    loc_info_df = pds.merge(loc_info_df,
305                            parsed_data[LIST_LOCINFO[0]],
306                            on='time_row_num')
307    # Convert GNSS Time
308    loc_info_df['gnsstime'] = loc_info_df.apply(
309        lambda row: datetime.datetime.strptime(row.Date + '-' + row.Time,
310                                               '%Y/%m/%d-%H:%M:%S'),
311        axis=1)
312
313    return timestamp_df, sv_info_df, sv_stat_df, loc_info_df
314
315
316def parse_gpsapilog_to_df_v2(filename):
317    """Parse GPS API log to Pandas dataframes, by using merge_asof.
318
319    Args:
320      filename: full log file name.
321        Type, String.
322
323    Returns:
324      timestamp_df: Timestamp Data Frame.
325        Type, Pandas DataFrame.
326      sv_info_df: GNSS SV info Data Frame.
327        Type, Pandas DataFrame.
328      sv_stat_df: GNSS SV statistic Data Frame.
329        Type, Pandas DataFrame
330      loc_info_df: Location Information Data Frame.
331        Type, Pandas DataFrame.
332        include Provider, Latitude, Longitude, Altitude, GNSSTime, Speed, Bearing
333    """
334    # Get parsed dataframe list
335    parsed_data = parse_log_to_df(
336        filename=filename,
337        configs=CONFIG_GPSAPILOG,
338    )
339
340    # get DUT Timestamp
341    timestamp_df = parsed_data['phone_time']
342    timestamp_df['phone_time'] = timestamp_df.apply(
343        lambda row: datetime.datetime.strptime(row.date + '-' + row.time,
344                                               '%Y/%m/%d-%H:%M:%S'),
345        axis=1)
346    # drop logsize, date, time
347    parsed_data['phone_time'] = timestamp_df.drop(['logsize', 'date', 'time'],
348                                                  axis=1)
349
350    # Add phone_time from timestamp dataframe by row number
351    for key in parsed_data:
352        if (key != 'phone_time') and (not parsed_data[key].empty):
353            parsed_data[key] = pds.merge_asof(parsed_data[key],
354                                              parsed_data['phone_time'],
355                                              left_index=True,
356                                              right_index=True)
357
358    # Get space vehicle info dataframe
359    # Handle the pre GPSTool 2.12.24 case
360    if not parsed_data['SpaceVehicle'].empty:
361        sv_info_df = parsed_data['SpaceVehicle']
362
363    # Handle the post GPSTool 2.12.24 case with baseband CNo
364    elif not parsed_data['SpaceVehicle_wBB'].empty:
365        sv_info_df = parsed_data['SpaceVehicle_wBB']
366
367    # Get space vehicle statistics dataframe
368    # Handle the pre GPSTool 2.12.24 case
369    if not parsed_data['HistoryAvgTop4CNo'].empty:
370        # First merge all dataframe from LIST_SVSTAT[1:],
371        sv_stat_df = fts.reduce(
372            lambda item1, item2: pds.merge(item1, item2, on='phone_time'),
373            [parsed_data[key] for key in LIST_SVSTAT[1:]])
374        # Then merge with LIST_SVSTAT[0]
375        sv_stat_df = pds.merge(sv_stat_df,
376                               parsed_data[LIST_SVSTAT[0]],
377                               on='phone_time')
378
379    # Handle the post GPSTool 2.12.24 case with baseband CNo
380    elif not parsed_data['AntennaHistoryAvgTop4CNo'].empty:
381        # First merge all dataframe from LIST_SVSTAT[1:],
382        sv_stat_df = fts.reduce(
383            lambda item1, item2: pds.merge(item1, item2, on='phone_time'),
384            [parsed_data[key] for key in LIST_SVSTAT_WBB[1:]])
385        # Then merge with LIST_SVSTAT[0]
386        sv_stat_df = pds.merge(sv_stat_df,
387                               parsed_data[LIST_SVSTAT_WBB[0]],
388                               on='phone_time')
389
390    # Get location fix information dataframe
391    # First merge all dataframe from LIST_LOCINFO[1:],
392    loc_info_df = fts.reduce(
393        lambda item1, item2: pds.merge(item1, item2, on='phone_time'),
394        [parsed_data[key] for key in LIST_LOCINFO[1:]])
395    # Then merge with LIST_LOCINFO[8]
396    loc_info_df = pds.merge(loc_info_df,
397                            parsed_data[LIST_LOCINFO[0]],
398                            on='phone_time')
399    # Convert GNSS Time
400    loc_info_df['gnsstime'] = loc_info_df.apply(
401        lambda row: datetime.datetime.strptime(row.Date + '-' + row.Time,
402                                               '%Y/%m/%d-%H:%M:%S'),
403        axis=1)
404
405    # Data Conversion
406    timestamp_df['logsize'] = timestamp_df['logsize'].astype(int)
407
408    sv_info_df['SV'] = sv_info_df['SV'].astype(int)
409    sv_info_df['Elevation'] = sv_info_df['Elevation'].astype(float)
410    sv_info_df['Azimuth'] = sv_info_df['Azimuth'].astype(float)
411    sv_info_df['Frequency'] = sv_info_df['Frequency'].astype(float)
412
413    if 'CNo' in list(sv_info_df.columns):
414        sv_info_df['CNo'] = sv_info_df['CNo'].astype(float)
415        sv_info_df['AntCNo'] = sv_info_df['CNo']
416    elif 'AntCNo' in list(sv_info_df.columns):
417        sv_info_df['AntCNo'] = sv_info_df['AntCNo'].astype(float)
418        sv_info_df['BbCNo'] = sv_info_df['BbCNo'].astype(float)
419
420    if 'CurrentAvgTop4CNo' in list(sv_stat_df.columns):
421        sv_stat_df['CurrentAvgTop4CNo'] = sv_stat_df[
422            'CurrentAvgTop4CNo'].astype(float)
423        sv_stat_df['CurrentAvgCNo'] = sv_stat_df['CurrentAvgCNo'].astype(float)
424        sv_stat_df['HistoryAvgTop4CNo'] = sv_stat_df[
425            'HistoryAvgTop4CNo'].astype(float)
426        sv_stat_df['HistoryAvgCNo'] = sv_stat_df['HistoryAvgCNo'].astype(float)
427        sv_stat_df['AntennaCurrentAvgTop4CNo'] = sv_stat_df[
428            'CurrentAvgTop4CNo']
429        sv_stat_df['AntennaCurrentAvgCNo'] = sv_stat_df['CurrentAvgCNo']
430        sv_stat_df['AntennaHistoryAvgTop4CNo'] = sv_stat_df[
431            'HistoryAvgTop4CNo']
432        sv_stat_df['AntennaHistoryAvgCNo'] = sv_stat_df['HistoryAvgCNo']
433        sv_stat_df['BasebandCurrentAvgTop4CNo'] = npy.nan
434        sv_stat_df['BasebandCurrentAvgCNo'] = npy.nan
435        sv_stat_df['BasebandHistoryAvgTop4CNo'] = npy.nan
436        sv_stat_df['BasebandHistoryAvgCNo'] = npy.nan
437
438    elif 'AntennaCurrentAvgTop4CNo' in list(sv_stat_df.columns):
439        sv_stat_df['AntennaCurrentAvgTop4CNo'] = sv_stat_df[
440            'AntennaCurrentAvgTop4CNo'].astype(float)
441        sv_stat_df['AntennaCurrentAvgCNo'] = sv_stat_df[
442            'AntennaCurrentAvgCNo'].astype(float)
443        sv_stat_df['AntennaHistoryAvgTop4CNo'] = sv_stat_df[
444            'AntennaHistoryAvgTop4CNo'].astype(float)
445        sv_stat_df['AntennaHistoryAvgCNo'] = sv_stat_df[
446            'AntennaHistoryAvgCNo'].astype(float)
447        sv_stat_df['BasebandCurrentAvgTop4CNo'] = sv_stat_df[
448            'BasebandCurrentAvgTop4CNo'].astype(float)
449        sv_stat_df['BasebandCurrentAvgCNo'] = sv_stat_df[
450            'BasebandCurrentAvgCNo'].astype(float)
451        sv_stat_df['BasebandHistoryAvgTop4CNo'] = sv_stat_df[
452            'BasebandHistoryAvgTop4CNo'].astype(float)
453        sv_stat_df['BasebandHistoryAvgCNo'] = sv_stat_df[
454            'BasebandHistoryAvgCNo'].astype(float)
455
456    sv_stat_df['L5EngagingRate'] = sv_stat_df['L5EngagingRate'].astype(float)
457
458    loc_info_df['Latitude'] = loc_info_df['Latitude'].astype(float)
459    loc_info_df['Longitude'] = loc_info_df['Longitude'].astype(float)
460    loc_info_df['Altitude'] = loc_info_df['Altitude'].astype(float)
461    loc_info_df['Speed'] = loc_info_df['Speed'].astype(float)
462    loc_info_df['Bearing'] = loc_info_df['Bearing'].astype(float)
463
464    return timestamp_df, sv_info_df, sv_stat_df, loc_info_df
465