diff --git a/src/modules/qt/filter_gpsgraphic.cpp b/src/modules/qt/filter_gpsgraphic.cpp index b19f57aca..76d18a280 100644 --- a/src/modules/qt/filter_gpsgraphic.cpp +++ b/src/modules/qt/filter_gpsgraphic.cpp @@ -633,26 +633,44 @@ static void process_filter_properties(mlt_filter filter, mlt_frame frame) / 1000); } -//checks if a new file is selected and if so, does private_data cleanup and calls qxml_parse_file +//checks if a new file is selected and if so, does private_data cleanup and calls q_parse_file //+ computes some one time pdata values static void process_file(mlt_filter filter, mlt_frame frame) { private_data *pdata = (private_data *) filter->child; mlt_properties properties = MLT_FILTER_PROPERTIES(filter); char *filename = mlt_properties_get(properties, "resource"); - - //if there's no file selected just return - if (!filename || !strcmp(filename, "")) - return; + char auto_filename[PATH_MAX] = {0}; + bool try_auto_filename = false; + + //if there's no file selected, try to find one, and just return if failure or already tried + if (!filename || !strcmp(filename, "")) { + if (pdata->checked_gps_companion) + return; + + //get the parent clip's path + mlt_producer producer = mlt_producer_cut_parent(mlt_frame_get_original_producer(frame)); + char *clip_path = mlt_properties_get(MLT_PRODUCER_PROPERTIES(producer), "resource"); + bool ret = q_auto_gps_file(clip_path, auto_filename); + pdata->checked_gps_companion = true; + + if (ret) { + filename = auto_filename; + try_auto_filename = true; + } else + return; + } //check if the file has been changed, if not, current data is ok, do nothing if (strcmp(pdata->last_filename, filename)) { - // mlt_log_info(filter, "Reading new file: last_filename (%s) != entered_filename (%s), swap_180 = %d\n", pdata->last_filename, filename, swap); default_priv_data(pdata); + strncpy(pdata->last_filename, filename, sizeof(pdata->last_filename) - 1); pdata->last_filename[sizeof(pdata->last_filename) - 1] = '\0'; - if (qxml_parse_file(filter_to_gps_data(filter)) == 1) { + if (q_parse_file(filter_to_gps_data(filter)) == 1) { + if (try_auto_filename) + mlt_properties_set_string(properties, "resource", filename); get_first_gps_time(filter_to_gps_data(filter)); get_last_gps_time(filter_to_gps_data(filter)); @@ -665,6 +683,8 @@ static void process_file(mlt_filter filter, mlt_frame frame) //store name in pdata or it'll retry reading every frame if bad file strncpy(pdata->last_filename, filename, sizeof(pdata->last_filename) - 1); pdata->last_filename[sizeof(pdata->last_filename) - 1] = '\0'; + if (try_auto_filename) + pdata->checked_gps_companion = true; } } } @@ -740,7 +760,7 @@ mlt_filter filter_gpsgraphic_init(mlt_profile profile, mlt_properties_set_int(properties, "show_now_text", 1); mlt_properties_set_double(properties, "angle", 0); mlt_properties_set_int(properties, "thickness", 5); - mlt_properties_set(properties, "rect", "10% 10% 30% 30%"); + mlt_properties_set(properties, "rect", "75% 75% 20% 20%"); mlt_properties_set_int(properties, "show_grid", 0); mlt_properties_set(properties, "legend_unit", ""); mlt_properties_set_int(properties, "draw_individual_dots", 0); diff --git a/src/modules/qt/filter_gpsgraphic.h b/src/modules/qt/filter_gpsgraphic.h index 9129432cc..7769aedeb 100644 --- a/src/modules/qt/filter_gpsgraphic.h +++ b/src/modules/qt/filter_gpsgraphic.h @@ -33,7 +33,7 @@ struct s_gps_data_bounds namespace { -typedef struct +struct private_data { gps_point_raw *gps_points_r; //raw gps data from file gps_point_proc *gps_points_p; //processed gps data @@ -55,31 +55,27 @@ typedef struct double bg_img_scaled_width, bg_img_scaled_height; QRectF bg_img_matched_rect; int swap_180; -} private_data; + bool checked_gps_companion; +}; } // namespace -typedef enum { - gspg_location_src = 0, - gpsg_altitude_src, - gpsg_hr_src, - gpsg_speed_src -} gpsg_data_sources; +enum gpsg_data_sources { gspg_location_src = 0, gpsg_altitude_src, gpsg_hr_src, gpsg_speed_src }; -typedef enum { +enum gpsg_data_subids { gpsg_latitude_id = gspg_location_src, gpsg_longitude_id = gpsg_latitude_id + 100 -} gpsg_data_subids; +}; -typedef enum { +enum gpsg_graph_types { gpsg_2d_map_graph = 0, gpsg_crop_center_graph, gpsg_speedometer_graph //gpsg_ladder_graph, //was thinking a vertical ruler/gauge thing for altitude, couldn't design something that actually looks good //gpsg_compass_graph //probably easy to adapt from speedometer but not really that important to be worth the extra code -} gpsg_graph_types; +}; -typedef enum { +enum gpsg_color_styles { gpsg_color_by_solid = 0, gpsg_color_by_solid_past_future, gpsg_color_by_solid_past, @@ -93,7 +89,7 @@ typedef enum { gpsg_color_by_speed_max100, gpsg_color_by_grade_max90, gpsg_color_by_grade_max20 -} gpsg_color_styles; +}; struct s_base_crops { diff --git a/src/modules/qt/filter_gpsgraphic.yml b/src/modules/qt/filter_gpsgraphic.yml index d405818e0..b4bc77b7c 100644 --- a/src/modules/qt/filter_gpsgraphic.yml +++ b/src/modules/qt/filter_gpsgraphic.yml @@ -2,7 +2,7 @@ schema_version: 7.2 type: filter identifier: gpsgraphic title: GPS Graphic -version: 4 +version: 5 copyright: Meltytech, LLC creator: Daniel F license: LGPLv2.1 @@ -17,7 +17,8 @@ parameters: - identifier: resource title: GPS File/URL description: > - The fullpath of file containing location (GPS) data. Supported extensions: .gpx, .tcx. + The fullpath of the file containing location (GPS) data. Supported extensions: .gpx, .tcx .txt + (see the gpstext yml for .txt support details) type: string required: yes readonly: no diff --git a/src/modules/qt/filter_gpstext.cpp b/src/modules/qt/filter_gpstext.cpp index 0a281d57f..d421041b0 100644 --- a/src/modules/qt/filter_gpstext.cpp +++ b/src/modules/qt/filter_gpstext.cpp @@ -24,7 +24,7 @@ namespace { -typedef struct +struct private_data { gps_point_raw *gps_points_r; //raw gps data from file gps_point_proc *gps_points_p; //processed gps data @@ -41,7 +41,8 @@ typedef struct char last_filename[PATH_MAX]; //gps file fullpath char interpolated; int swap_180; -} private_data; + bool checked_gps_companion; +}; } // namespace @@ -129,7 +130,6 @@ static int get_next_token(char *str, int *pos, char *token, int *is_keyword) */ static int64_t extract_offset_time_ms_keyword(char *keyword) { - //mlt_log_info(NULL, "extract_offset_time_keyword: in keyword: %s\n", keyword); char *end = NULL; if (keyword == NULL) return 0; @@ -601,22 +601,41 @@ static void process_file(mlt_filter filter, mlt_frame frame) private_data *pdata = (private_data *) filter->child; mlt_properties properties = MLT_FILTER_PROPERTIES(filter); char *filename = mlt_properties_get(properties, "resource"); + char auto_filename[PATH_MAX] = {0}; + bool try_auto_filename = false; + if (filename == NULL) filename = mlt_properties_get(properties, "gps.file"); /* for backwards compatibility with v1 */ - //if there's no file selected just return - if (!filename || !strcmp(filename, "")) - return; + //if there's no file selected, try to find one, and just return if failure or already tried + if (!filename || !strcmp(filename, "")) { + if (pdata->checked_gps_companion) + return; + + //get the parent clip's path + mlt_producer producer = mlt_producer_cut_parent(mlt_frame_get_original_producer(frame)); + char *clip_path = mlt_properties_get(MLT_PRODUCER_PROPERTIES(producer), "resource"); + bool ret = q_auto_gps_file(clip_path, auto_filename); + pdata->checked_gps_companion = true; + + if (ret) { + filename = auto_filename; + try_auto_filename = true; + } else + return; + } //check if the file has been changed, if not, current data is ok, do nothing if (strcmp(pdata->last_filename, filename)) { - // mlt_log_info(filter, "Reading new file: last_filename (%s) != entered_filename (%s), swap_180 = %d\n", pdata->last_filename, filename, swap); default_priv_data(pdata); + strncpy(pdata->last_filename, filename, sizeof(pdata->last_filename) - 1); pdata->last_filename[sizeof(pdata->last_filename) - 1] = '\0'; - if (qxml_parse_file(filter_to_gps_data(filter)) == 1) { + if (q_parse_file(filter_to_gps_data(filter)) == 1) { + if (try_auto_filename) + mlt_properties_set_string(properties, "resource", filename); get_first_gps_time(filter_to_gps_data(filter)); get_last_gps_time(filter_to_gps_data(filter)); @@ -628,6 +647,8 @@ static void process_file(mlt_filter filter, mlt_frame frame) //store name in pdata or it'll retry reading every frame if bad file strncpy(pdata->last_filename, filename, sizeof(pdata->last_filename) - 1); pdata->last_filename[sizeof(pdata->last_filename) - 1] = '\0'; + if (try_auto_filename) + pdata->checked_gps_companion = true; } } } @@ -718,7 +739,7 @@ mlt_filter filter_gpstext_init(mlt_profile profile, mlt_service_type type, const "Altitude: #gps_elev#m\n\n" "GPS time: #gps_datetime_now# UTC\n" "GPS location: #gps_lat#, #gps_lon#"); - mlt_properties_set_string(my_properties, "geometry", "10%/10%:80%x80%:100%"); + mlt_properties_set_string(my_properties, "geometry", "5% 50% 45% 45% 100%"); mlt_properties_set_string(my_properties, "family", "Sans"); mlt_properties_set_string(my_properties, "size", "26"); mlt_properties_set_string(my_properties, "weight", "400"); diff --git a/src/modules/qt/filter_gpstext.yml b/src/modules/qt/filter_gpstext.yml index eeacdb096..85898e331 100644 --- a/src/modules/qt/filter_gpstext.yml +++ b/src/modules/qt/filter_gpstext.yml @@ -2,7 +2,7 @@ schema_version: 7.2 type: filter identifier: gpstext title: GPS Text -version: 9 +version: 10 copyright: Meltytech, LLC creator: Daniel F license: LGPLv2.1 @@ -81,8 +81,44 @@ parameters: - identifier: resource title: GPS File/URL - description: > - "The fullpath of file containing location (GPS) data. Supported extensions: .gpx, .tcx." + description: | + The fullpath of the file containing location (GPS) data. Supported extensions: .gpx, .tcx. .txt. + Note: while .gpx and .tcx are international standards, .txt files do not generally contain info + on how and what GPS details are storing, yet many devices (mainly dashcams) use them anyway and + record positions in a human-readable format. + Also note that while text files usually have the .txt extension, any input file that is not + ending in .gpx or .tcx will be passed to the generic text_parser, so as long as it is text (i.e. + not binary) and it has one entry per line, there is a good chance it will work. + At this point there are 2 modes for parsing text files: + 1) Automatic regex: + * This was introduced as a no hassle alternative for just loading a reasonable formatted + plaintext file. It only looks for datetime and a latitude/longitude pair. + * It searches for this pattern: yyyy*MM*dd*HH*mm*ss[.sss] (optional 1-3 digit miliseconds) and + if found, it parses it as the date-time; any non-digit separators can exist between these fields, + so both 2026/10/23_12:23:34 and 20261023122334 are found correctly. + * It then searches for latitude (and after it, a longitude) with a regex matching 0.1234 to + 123.45678910 (so 1-3 digits followed by a dot then 4+ digits). Optionally starts with NSEW or +-. + * See the q_parse_txt() function in gps_parser.cpp for the exact QT regexes used). + 2) Manual format line: + * Regex is limited and can fail, so if you want extra fields or if your date is non standard + (like month before day) there is another option: you need to manually edit the .txt file and + insert a format string like the one below as the first line: + #delimiter=" " datetime="yyyy/MM/dd" datetime="hh:mm:ss" latitude longitude speed skip heading altitude + which corresponds to: + 2025/06/12 02:45:01 N:45.123456 E:26.123456 0.0 km/h 0.00 72.50 10 x:-4.13 y:0.27 z:0.49 + * The "#delimiter=" keyword is mandatory at the start, and it will treat everything between the "" + as the delimiter. Common values are space (" ") and comma (","). + * datetime= is a special keyword that takes an argument between "" which will be passed to + QDateTime::fromString, so use the patterns QT accepts. + * For date and time, there are also the "date" and "time" keywords that will attempt to separately + match yyyy*MM*dd and hh*mm*ss*[.ms], "datetime" which will do them together, and datetimeiso which + will match ISO 8601 extended format ("2026-10-23T18:25:20.739" with optional timezone). + * Supported keywords: datetime= datetimeiso datetime date time lat lon speed alt ele head bear hr + heart cad temp pow. + * Note: any format/keyword name not recognized will be skipped along with it's corresponding column + value. In my example above, the "skip" keyword simply skips over the unit which follows the speed. + * Also note that everything after the last keyword is skipped so you don't need to bother with the + entire line in the file. type: string required: yes readonly: no diff --git a/src/modules/qt/gps_parser.cpp b/src/modules/qt/gps_parser.cpp index 58f6610c5..1eb9e4ed0 100644 --- a/src/modules/qt/gps_parser.cpp +++ b/src/modules/qt/gps_parser.cpp @@ -23,11 +23,49 @@ #endif #include "gps_parser.h" #include +#include +#include +#include #include #define _x (const xmlChar *) #define _s (const char *) -#define _qxml_getthestring(s) qUtf8Printable((s).toString()) + +QStringList accepted_distance_formats + = {"km", "kilometer", "mi", "mile", "nm", "nautical", "ft", "feet"}; + +QStringList accepted_speed_formats = {"km", + "km/h", + "kilo", + "ms", + "m/s", + "meter", + "mmin", + "m/min", + "ftmin", + "ft/min", + "mi", + "mi/h", + "mile", + "kn", + "nm/h", + "knots", + "ft", + "ft/s", + "feet"}; + +bool qstring_contains_any_qstringlist(const QString &main_string, const QStringList &list) +{ + for (const auto &s : list) { + if (main_string.contains(s, Qt::CaseInsensitive)) + return true; + } + return false; +} + +const QString sample_format_line + = "#delimiter=\" \" datetime=\"yyyy/MM/dd\" datetime=\"hh:mm:ss\" latitude longitude " + "speed skip heading altitude"; // Maps are usually in ESPG3857 format, not in GPS format EPSG4326 / WG84. We have to project // the latitude, otherwise big maps have wrong aspect ratio @@ -49,6 +87,30 @@ double get_180_swapped(double lon) return lon + (lon > 0 ? -180 : 180); } +inline void _mlt_log_gps_point(const int i, const gps_point_raw &crt_point) +{ + char str_time[25] = {0}; + mseconds_to_timestring(crt_point.time, NULL, str_time); + mlt_log_info(NULL, + "gps_point[%d]: time:%" PRId64 + " (%s), lat:%f, lon:%f, lat_projected:%f, speed:%f, " + "distance:%f, ele:%f, hr:%f, bearing:%f, cadence:%f, temp:%f, power:%f\n", + i, + crt_point.time, + str_time, + crt_point.lat, + crt_point.lon, + crt_point.lat_projected, + crt_point.speed, + crt_point.total_dist, + crt_point.ele, + crt_point.hr, + crt_point.bearing, + crt_point.cad, + crt_point.atemp, + crt_point.power); +} + //testing max valid time between 2 points = 10 * avg_gps_time, better option would be a user entered value int get_max_gps_diff_ms(gps_private_data gdata) { @@ -81,11 +143,15 @@ int64_t datetimeXMLstring_to_mseconds(const char *text, const char *format /* = datetime = QDateTime::fromString(QString(text), Qt::ISODateWithMs); if (!datetime.isValid()) { - mlt_log_warning(NULL, "[filter gpstext] %s failed on string: %.25s\n", __FUNCTION__, text); + mlt_log_warning(NULL, + "datetimeXMLstring_to_mseconds - QDateTime::fromString failed on string: " + "'%.25s' with format: '%s'; returning 0.", + text, + (format ? format : "NULL")); return 0; } -#if QT_VERSION > QT_VERSION_CHECK(6, 5, 0) +#if QT_VERSION >= QT_VERSION_CHECK(6, 5, 0) datetime.setTimeZone(QTimeZone::UTC); #else datetime.setTimeZone(QTimeZone(0)); @@ -378,8 +444,7 @@ double convert_speed_to_format(double x, const char *format) if (x == GPS_UNINIT) return x; //order is important as short keywords will match anywhere (ms in kms and mi in mi[nutes]) - if (format == NULL || strstr(format, "kms") || strstr(format, "km/s") - || strstr(format, "kilometer")) + if (format == NULL || strstr(format, "km") || strstr(format, "km/h") || strstr(format, "kilo")) return x * 3.6; //default km/h if (strstr(format, "ms") || strstr(format, "m/s") || strstr(format, "meter")) return x; @@ -1041,12 +1106,10 @@ void process_gps_smoothing(gps_private_data gdata, char do_processing) //File parsing with QT's XML stream reader // Parses a .gpx file into a gps_point_raw linked list -void qxml_parse_gpx(QXmlStreamReader &reader, gps_point_ll **gps_list, int *count_pts) +void qxml_parse_gpx(QXmlStreamReader &reader, + QVector &gps_points_file, + int &no_time_count) { - int64_t last_time = -1; - //support no time in file because many services (Strava included) do this for exported routes - gps_point_ll *no_time_head = NULL, *no_time_prev = NULL; - int no_time_count = 0; /* // sample point from .GPX file (version 1.0) + TrackPointExtension @@ -1064,128 +1127,109 @@ void qxml_parse_gpx(QXmlStreamReader &reader, gps_point_ll **gps_list, int *coun */ + + int nr_pt = 0, print5 = 0; while (!reader.atEnd() && !reader.hasError()) { reader.readNext(); - // mlt_log_info(NULL, " readNext tag: %s type = %d\n", _qxml_getthestring(reader.name()), reader.tokenType()); + // mlt_log_info(NULL, " readNext tag: %s type = %d\n", qUtf8Printable(reader.name()), reader.tokenType()); if (reader.isStartElement() && reader.name() == QStringLiteral("trkpt")) { gps_point_raw crt_point = uninit_gps_raw_point; + bool valid_datetime = false, got_other = false; + int got_latlon = 0; + nr_pt++; QXmlStreamAttributes attributes = reader.attributes(); if (attributes.hasAttribute("lat")) { crt_point.lat = attributes.value("lat").toDouble(); crt_point.lat_projected = project_latitude(crt_point.lat); + got_latlon++; } - if (attributes.hasAttribute("lon")) + if (attributes.hasAttribute("lon")) { crt_point.lon = attributes.value("lon").toDouble(); + got_latlon++; + } while (reader.readNext() && !(reader.name() == QStringLiteral("trkpt") && reader.tokenType() == QXmlStreamReader::EndElement)) //until closing trkpt { - // mlt_log_info(NULL, "[trkpt->1] readNext tag: %s, type=%d\n", _qxml_getthestring(reader.name()), reader.tokenType()); + // mlt_log_info(NULL, "[trkpt->1] readNext tag: %s, type=%d\n", qUtf8Printable(reader.name()), reader.tokenType()); if (!reader.isStartElement()) continue; - if (reader.name() == QStringLiteral("ele")) + if (reader.name() == QStringLiteral("ele")) { crt_point.ele = reader.readElementText().toDouble(); - else if (reader.name() == QStringLiteral("time")) + got_other = true; + } else if (reader.name() == QStringLiteral("time")) { crt_point.time = datetimeXMLstring_to_mseconds( qUtf8Printable(reader.readElementText())); - else if (reader.name() == QStringLiteral("speed")) + valid_datetime = (crt_point.time != 0); + } else if (reader.name() == QStringLiteral("speed")) { crt_point.speed = reader.readElementText().toDouble(); - else if (reader.name() == QStringLiteral("course")) + got_other = true; + } else if (reader.name() == QStringLiteral("course")) { crt_point.bearing = reader.readElementText().toDouble(); - else if (reader.name() == QStringLiteral("extensions")) { + got_other = true; + } else if (reader.name() == QStringLiteral("extensions")) { while (reader.readNext() && !(reader.name() == QStringLiteral("extensions") && reader.tokenType() == QXmlStreamReader::EndElement)) { - if (reader.name() == QStringLiteral("power")) + if (reader.name() == QStringLiteral("power")) { crt_point.power = reader.readElementText().toDouble(); - else if (reader.name() == QStringLiteral("TrackPointExtension")) { + got_other = true; + } else if (reader.name() == QStringLiteral("TrackPointExtension")) { while (reader.readNext() && !(reader.name() == QStringLiteral("TrackPointExtension") && reader.tokenType() == QXmlStreamReader::EndElement)) { - if (reader.name() == QStringLiteral("hr")) + if (reader.name() == QStringLiteral("hr")) { crt_point.hr = reader.readElementText().toDouble(); - else if (reader.name() == QStringLiteral("cad")) + got_other = true; + } else if (reader.name() == QStringLiteral("cad")) { crt_point.cad = reader.readElementText().toDouble(); - else if (reader.name() == QStringLiteral("atemp")) + got_other = true; + } else if (reader.name() == QStringLiteral("atemp")) { crt_point.atemp = reader.readElementText().toDouble(); + got_other = true; + } } } } } } - //now add the point to linked list (but only if increasing time) - if (crt_point.time != GPS_UNINIT && crt_point.time > last_time) { - *gps_list = (gps_point_ll *) calloc(1, sizeof(gps_point_ll)); - if (*gps_list == NULL) - return; - - *count_pts += 1; - (*gps_list)->gp = crt_point; - (*gps_list)->next = NULL; - gps_list = &(*gps_list)->next; - last_time = crt_point.time; - } else if (crt_point.time == GPS_UNINIT) { - //if no time is stored, we make it up starting from 1 - gps_point_ll *tmp = (gps_point_ll *) calloc(1, sizeof(gps_point_ll)); - if (tmp == NULL) - return; - if (no_time_head == NULL) { - no_time_head = tmp; - no_time_prev = tmp; - } else { - no_time_prev->next = tmp; - no_time_prev = tmp; + + bool valid_pt = (valid_datetime && got_other) || (got_latlon == 2); + if (!valid_pt) { + if (print5++ < 5) { + mlt_log_info( + NULL, + "qxlm_parse_gpx: skipping read point[%d], minimum condition: (datetime + " + "something) or (location)", + nr_pt); + _mlt_log_gps_point(nr_pt, crt_point); + if (print5 == 5) + mlt_log_info(NULL, "(will stop printing skipped points)"); } + continue; + } + + //store the point in the array, set synthetic time, we'll deal with invalid times after sorting + if (crt_point.time == GPS_UNINIT || crt_point.time == 0) { + //if no time is read (or bad format), we make it up starting from 1 no_time_count += 1; crt_point.time = no_time_count * 1000; - tmp->gp = crt_point; - tmp->next = NULL; - } else { - mlt_log_info(NULL, - "qxml_parse_gpx: skipping point due to time [%d] %f,%f - crt:%u.%u, " - "last:%u.%u\n", - *count_pts, - crt_point.lat, - crt_point.lon, - (unsigned) (crt_point.time / 1000), - (unsigned) (crt_point.time % 1000), - (unsigned) (last_time / 1000), - (unsigned) (last_time % 1000)); } + gps_points_file.append(crt_point); } } - //if no points had time, we replace the real list with the no_time one - if (*count_pts == 0 && no_time_count > 0) { - mlt_log_info(NULL, - "qxml_parse_gpx: all GPS points are missing time values, inserting each " - "point at 1 second interval!\n"); - *gps_list = no_time_head; - *count_pts = no_time_count; - } else if (*count_pts > 0 && no_time_count > 0) { - mlt_log_info(NULL, - "qxml_parse_gpx: %d GPS points have no time in original file and have been " - "discarded (total valid points kept: %d)!\n", - no_time_count, - *count_pts); - //cleanup the no_time list as we're not using it - while (no_time_head) { - gps_point_ll *tmp = no_time_head; - no_time_head = no_time_head->next; - free(tmp); - } - } + if (print5 > 5) + mlt_log_info(NULL, "qxml_parse_tcx: skipped %d points (out of %d)", print5, nr_pt); } // Parses a .tcx file into a gps_point_raw linked list -void qxml_parse_tcx(QXmlStreamReader &reader, gps_point_ll **gps_list, int *count_pts) +void qxml_parse_tcx(QXmlStreamReader &reader, + QVector &gps_points_file, + int &no_time_count) { - int64_t last_time = -1; - //support no time in file because many services (Strava included) do this for exported routes - gps_point_ll *no_time_head = NULL, *no_time_prev = NULL; - int no_time_count = 0; /* //sample point from .TCX file @@ -1209,117 +1253,434 @@ void qxml_parse_tcx(QXmlStreamReader &reader, gps_point_ll **gps_list, int *coun */ + + int nr_pt = 0, print5 = 0; while (!reader.atEnd() && !reader.hasError()) { reader.readNext(); - // mlt_log_info(NULL, " readNextStartElement tag: %s\n", _qxml_getthestring(reader.name())); + // mlt_log_info(NULL, " readNextStartElement tag: %s\n", qUtf8Printable(reader.name())); if (reader.isStartElement() && reader.name() == QStringLiteral("Trackpoint")) { gps_point_raw crt_point = uninit_gps_raw_point; + bool valid_datetime = false, got_other = false; + int got_latlon = 0; + nr_pt++; + while ( reader.readNext() && !(reader.name() == QStringLiteral("Trackpoint") && reader.tokenType() == QXmlStreamReader::EndElement)) //loop until we hit the closing { - // mlt_log_info(NULL, "[Trackpoint->1] readNext tag: %s, type=%d\n", _qxml_getthestring(reader.name()), reader.tokenType()); + // mlt_log_info(NULL, "[Trackpoint->1] readNext tag: %s, type=%d\n", qUtf8Printable(reader.name()), reader.tokenType()); if (!reader.isStartElement()) continue; - if (reader.name() == QStringLiteral("Time")) + if (reader.name() == QStringLiteral("Time")) { crt_point.time = datetimeXMLstring_to_mseconds( qUtf8Printable(reader.readElementText())); - else if (reader.name() == QStringLiteral("Position")) { + valid_datetime = (crt_point.time != 0); + } else if (reader.name() == QStringLiteral("Position")) { reader.readNextStartElement(); if (reader.name() == QStringLiteral("LatitudeDegrees")) { crt_point.lat = reader.readElementText().toDouble(); crt_point.lat_projected = project_latitude(crt_point.lat); + got_latlon++; } reader.readNextStartElement(); - if (reader.name() == QStringLiteral("LongitudeDegrees")) + if (reader.name() == QStringLiteral("LongitudeDegrees")) { crt_point.lon = reader.readElementText().toDouble(); + got_latlon++; + } } else if (reader.name() == QStringLiteral("AltitudeMeters")) { crt_point.ele = reader.readElementText().toDouble(); + got_other = true; } else if (reader.name() == QStringLiteral("DistanceMeters")) { crt_point.total_dist = reader.readElementText().toDouble(); + got_other = true; } else if (reader.name() == QStringLiteral("HeartRateBpm")) { reader.readNextStartElement(); - if (reader.name() == QStringLiteral("Value")) + if (reader.name() == QStringLiteral("Value")) { crt_point.hr = reader.readElementText().toDouble(); + got_other = true; + } } else if (reader.name() == QStringLiteral("Cadence")) { crt_point.cad = reader.readElementText().toDouble(); + got_other = true; } else if (reader.name() == QStringLiteral("Temperature")) { crt_point.atemp = reader.readElementText().toDouble(); + got_other = true; } else if (reader.isStartElement() && reader.name() == QStringLiteral("Extensions")) { while (reader.readNext() && !(reader.name() == QStringLiteral("TPX") && reader.tokenType() == QXmlStreamReader::EndElement)) { - if (reader.name() == QStringLiteral("Speed")) + if (reader.name() == QStringLiteral("Speed")) { crt_point.speed = reader.readElementText().toDouble(); - else if (reader.name() == QStringLiteral("Watts")) + got_other = true; + } else if (reader.name() == QStringLiteral("Watts")) { crt_point.power = reader.readElementText().toDouble(); + got_other = true; + } } } } - //now add the point to linked list (but only if increasing time) - if (crt_point.time != GPS_UNINIT && crt_point.time > last_time) { - *gps_list = (gps_point_ll *) calloc(1, sizeof(gps_point_ll)); - if (*gps_list == NULL) - return; - - *count_pts += 1; - (*gps_list)->gp = crt_point; - (*gps_list)->next = NULL; - gps_list = &(*gps_list)->next; - last_time = crt_point.time; - } else if (crt_point.time == GPS_UNINIT) { - //if no time is stored, we make it up starting from 1 - gps_point_ll *tmp = (gps_point_ll *) calloc(1, sizeof(gps_point_ll)); - if (tmp == NULL) - return; - if (no_time_head == NULL) { - no_time_head = tmp; - no_time_prev = tmp; - } else { - no_time_prev->next = tmp; - no_time_prev = tmp; + + bool valid_pt = (valid_datetime && got_other) || (got_latlon == 2); + if (!valid_pt) { + if (print5++ < 5) { + mlt_log_info( + NULL, + "qxlm_parse_gpx: skipping read point[%d], minimum condition: (datetime + " + "something) or (location)", + nr_pt); + _mlt_log_gps_point(nr_pt, crt_point); + if (print5 == 5) + mlt_log_info(NULL, "(will stop printing skipped points)"); } + continue; + } + + //store the point in the array, set synthetic time, we'll deal with invalid times after sorting + if (crt_point.time == GPS_UNINIT || crt_point.time == 0) { + //if no time is read (or bad format), we make it up starting from 1 no_time_count += 1; crt_point.time = no_time_count * 1000; - tmp->gp = crt_point; - tmp->next = NULL; - } else - mlt_log_info(NULL, - "qxml_parse_tcx: skipping point due to time [%d] %f,%f - crt:%u.%u, " - "last:%u.%u\n", - *count_pts, - crt_point.lat, - crt_point.lon, - (unsigned) (crt_point.time / 1000), - (unsigned) (crt_point.time % 1000), - (unsigned) (last_time / 1000), - (unsigned) (last_time % 1000)); + } + gps_points_file.append(crt_point); } } - //if no points had time, we replace the real list with the no_time one - if (*count_pts == 0 && no_time_count > 0) { - mlt_log_info(NULL, - "qxml_parse_gpx: all GPS points are missing time values, inserting each " - "point at 1 second interval!\n"); - *gps_list = no_time_head; - *count_pts = no_time_count; - } else if (*count_pts > 0 && no_time_count > 0) { - //cleanup the no_time list as we're not using it + if (print5 > 5) + mlt_log_info(NULL, "qxml_parse_tcx: skipped %d points (out of %d)", print5, nr_pt); +} + +//tries to parse a .txt file containing GPS data; returns 0 for failure, 1 success +//first line of the file may contain a format helper that starts with #delimiter= +//if no format is found, we try a generic regex for date+time and lat, lon (no other fields) +int q_parse_txt(QTextStream &reader, QVector &gps_points_file, int &no_time_count) +{ + //example from my dashcam (added extra spaces for readability): + //2025/06/12 02:45:01 N:45.123456 E:26.123456 0.0 km/h 0.00 72.50 10 x:-4.13 y:0.27 z:0.49 + //date time lat lon speed (unit) heading altitude satelites xyz_gravity_sensor + //expected format line: + //#delimiter=" " datetime="yyyy/MM/dd" datetime="hh:mm:ss" lat lon speed skip heading altitude + + //regex for yyyy*MM*dd*HH*mm*ss[*sss], any separators (non digits chars), miliseconds optional + QRegularExpression re_date_time( + R"((\d{4})\D?(\d{2})\D?(\d{2})\D?(\d{2})\D?(\d{2})\D?(\d{2})(?:\D?(\d{1,3}))?)"); + QRegularExpression re_date(R"((\d{4})\D?(\d{2})\D?(\d{2}))"); + QRegularExpression re_time(R"((\d{2})\D?(\d{2})\D?(\d{2})(?:\D?(\d{1,3}))?)"); + QRegularExpression re_lat(R"(([NS])[: ]?([+-]?\d{1,3}[\.]\d{4,}))"); + QRegularExpression re_lon(R"(([EW])[: ]?([+-]?\d{1,3}[\.]\d{4,}))"); + + const QString first_keyword = "#delimiter="; + QString format_line, delimiter; + QStringList format_tokens, format_tokens_arguments; + bool regex_mode = false, format_date_time_separated = false; + + //if file doesn't start with "#delimiter=", will try regex for datetime and lat lon + if ((format_line = reader.readLine()).startsWith(first_keyword) == false) { mlt_log_info(NULL, - "qxml_parse_gpx: %d GPS points have no time in original file and have been " - "discarded (total valid points kept: %d)!\n", - no_time_count, - *count_pts); - while (no_time_head) { - gps_point_ll *tmp = no_time_head; - no_time_head = no_time_head->next; - free(tmp); + "no format line found at the start of the file, trying a generic regex " + "for datetime and lat/lon"); + mlt_log_info(NULL, "Note: sample format line: '%s'", qUtf8Printable(sample_format_line)); + reader.seek(0); + regex_mode = true; + } else if (!reader.atEnd()) { + //read and store the content of delimiter="" + int firstQ = format_line.indexOf('"'); + int secondQ = format_line.indexOf('"', firstQ + 1); + if (firstQ != -1 && secondQ != -1) { + delimiter = format_line.mid(firstQ + 1, secondQ - (firstQ + 1)); + if (!delimiter.isEmpty()) { + QString after_delimiter = format_line.mid(secondQ + 1); + format_tokens = after_delimiter.split(delimiter, Qt::SkipEmptyParts); + } else { + mlt_log_warning(NULL, + "empty delimiter string, make sure to enclose it within double " + "quotes. Aborting file."); + return 0; + } + } else { + mlt_log_warning(NULL, + "bad delimiter format, make sure to enclose a character or string " + "within double quotes. Aborting file."); + return 0; + } + + // mlt_log_debug(NULL, + // "q_parse_txt(): format_line='%s', delimiter='%s', format_tokens found=%" PRId64 "", + // qUtf8Printable(format_line), + // qUtf8Printable(delimiter), + // format_tokens.size()); + + //extract datetime with format + format_tokens_arguments.resize(format_tokens.size()); + for (int i = 0; i < format_tokens.size(); i++) { + if (format_tokens[i].contains("datetime=", Qt::CaseInsensitive)) { + int firstQ = format_tokens[i].indexOf('"'); + int secondQ = format_tokens[i].indexOf('"', firstQ + 1); + format_tokens_arguments[i] = format_tokens[i].mid(firstQ + 1, + secondQ - (firstQ + 1)); + format_date_time_separated = true; + } } + } else { //already end of file? this is an invalid file + mlt_log_warning( + NULL, + "file ended too soon, please make sure the GPS points are stored on individual lines."); + return 0; } + + //process the file line by line + int nr_invalid_lines = 0; + QVector warned_of_format(format_tokens.size(), false); + while (!reader.atEnd()) { + QString line = reader.readLine(); + gps_point_raw crt_point = uninit_gps_raw_point; + //a point is valid if it has either (a valid time + at least one other field) or (a valid lat/lon pair) + bool valid_datetime = false, got_other = false; + int got_latlon = 0; + + if (regex_mode) { //using regex to find a datetime and/or a lat+lon in this line + //datetime + auto m = re_date_time.match(line); + if (m.hasMatch()) { + int year = m.captured(1).toInt(); + int month = m.captured(2).toInt(); + int day = m.captured(3).toInt(); + int hour = m.captured(4).toInt(); + int minute = m.captured(5).toInt(); + int second = m.captured(6).toInt(); + int msec = m.captured(7).isEmpty() ? 0 + : m.captured(7).leftJustified(3, '0').toInt(); + QDateTime dt(QDate(year, month, day), QTime(hour, minute, second, msec), Qt::UTC); + if (dt.isValid()) { + crt_point.time = dt.toMSecsSinceEpoch(); + valid_datetime = (crt_point.time != 0); + } + } + //latitude + auto m2 = re_lat.match(line); + if (m2.hasMatch()) { + QString direction = m2.captured(1); + crt_point.lat = m2.captured(2).toDouble(); + if (direction == "S") + crt_point.lat *= -1; + if (is_valid_latitude(crt_point.lat)) { + got_latlon++; + } + crt_point.lat_projected = project_latitude(crt_point.lat); + } + //longitude + auto m3 = re_lon.match(line, m2.capturedEnd()); + if (m3.hasMatch()) { + QString direction = m3.captured(1); + crt_point.lon = m3.captured(2).toDouble(); + if (direction == "W") + crt_point.lon *= -1; + if (is_valid_longitude(crt_point.lon)) { + got_latlon++; + } + } + } else { //format provided by the file + QStringList line_tokens = line.split(delimiter); + QString datetime_assembled, datetime_format_assembled; + bool got_qdate = false, got_qtime = false; + QDate qdate_part; + QTime qtime_part; + + for (int i = 0; i < format_tokens.size() && i < line_tokens.size(); i++) { + // mlt_log_info(NULL, + // "format[%d]: %s, token[%d]; %s", + // i, + // qUtf8Printable(format_tokens[i]), + // i, + // line_tokens[i].toUtf8().constData()); + if (format_tokens[i].contains("datetime=", Qt::CaseInsensitive)) { + //explicit format provided by the user, can have date separated from time + datetime_assembled.append(line_tokens[i]); + datetime_assembled.append(delimiter); + datetime_format_assembled.append(format_tokens_arguments[i]); + datetime_format_assembled.append(delimiter); + } else if (format_tokens[i].contains("datetimeiso", Qt::CaseInsensitive)) { + //exact ISO format + crt_point.time = datetimeXMLstring_to_mseconds(qUtf8Printable(line_tokens[i])); + valid_datetime = (crt_point.time != 0); + } else if (format_tokens[i].contains("datetime", Qt::CaseInsensitive)) { + //generic date+time in one token, will use regex for parsing + auto m = re_date_time.match(line_tokens[i]); + if (m.hasMatch()) { + int year = m.captured(1).toInt(); + int month = m.captured(2).toInt(); + int day = m.captured(3).toInt(); + int hour = m.captured(4).toInt(); + int minute = m.captured(5).toInt(); + int second = m.captured(6).toInt(); + int msec = m.captured(7).isEmpty() + ? 0 + : m.captured(7).leftJustified(3, '0').toInt(); + QDateTime dt(QDate(year, month, day), + QTime(hour, minute, second, msec), + Qt::UTC); + if (dt.isValid()) { + crt_point.time = dt.toMSecsSinceEpoch(); + valid_datetime = (crt_point.time != 0); + } + } + } else if (format_tokens[i].contains("date", Qt::CaseInsensitive)) { + //generic date only in a single token, will use regex for parsing + auto m = re_date.match(line_tokens[i]); + if (m.hasMatch()) { + int year = m.captured(1).toInt(); + int month = m.captured(2).toInt(); + int day = m.captured(3).toInt(); + qdate_part = QDate(year, month, day); + if (qdate_part.isValid()) + got_qdate = true; + } + } else if (format_tokens[i].contains("time", Qt::CaseInsensitive)) { + //generic time only in a single token, will use regex for parsing + auto m = re_time.match(line_tokens[i]); + if (m.hasMatch()) { + int hour = m.captured(1).toInt(); + int minute = m.captured(2).toInt(); + int second = m.captured(3).toInt(); + int msec = m.captured(4).isEmpty() + ? 0 + : m.captured(4).leftJustified(3, '0').toInt(); + qtime_part = QTime(hour, minute, second, msec); + if (qtime_part.isValid()) + got_qtime = true; + } + } else if (format_tokens[i].contains("lat", Qt::CaseInsensitive)) { + double negative = (line_tokens[i].startsWith('s', Qt::CaseInsensitive) ? -1 : 1); + int number_starts = line_tokens[i].indexOf(QRegularExpression("[-+0-9]")); + crt_point.lat = line_tokens[i].mid(number_starts).toDouble() * negative; + crt_point.lat_projected = project_latitude(crt_point.lat); + if (is_valid_latitude(crt_point.lat)) + got_latlon++; + } else if (format_tokens[i].contains("lon", Qt::CaseInsensitive)) { + double negative = (line_tokens[i].startsWith('w', Qt::CaseInsensitive) ? -1 : 1); + int number_starts = line_tokens[i].indexOf(QRegularExpression("[-+0-9]")); + crt_point.lon = line_tokens[i].mid(number_starts).toDouble() * negative; + if (is_valid_longitude(crt_point.lon)) + got_latlon++; + } else if (format_tokens[i].contains("speed", Qt::CaseInsensitive)) { + crt_point.speed = line_tokens[i].toDouble(); + if (i + 1 < line_tokens.size()) { + double ratio = 1; + if (qstring_contains_any_qstringlist(line_tokens[i + 1], + accepted_speed_formats)) + ratio = convert_speed_to_format(1, + qPrintable( + line_tokens[i + 1].toLower())); + crt_point.speed /= ratio; + } + got_other = true; + } else if (format_tokens[i].contains("alt", Qt::CaseInsensitive) + || format_tokens[i].contains("ele", Qt::CaseInsensitive)) { + crt_point.ele = line_tokens[i].toDouble(); + if (i + 1 < line_tokens.size()) { + double ratio = 1; + if (qstring_contains_any_qstringlist(line_tokens[i + 1], + accepted_distance_formats)) + ratio = convert_distance_to_format(1, + qPrintable( + line_tokens[i + 1].toLower())); + crt_point.ele /= ratio; + } + got_other = true; + } else if (format_tokens[i].contains("head", Qt::CaseInsensitive) + || format_tokens[i].contains("bear", Qt::CaseInsensitive)) { + crt_point.bearing = line_tokens[i].toDouble(); + got_other = true; + } else if (format_tokens[i].contains("hr", Qt::CaseInsensitive) + || format_tokens[i].contains("heart", Qt::CaseInsensitive)) { + crt_point.hr = line_tokens[i].toDouble(); + got_other = true; + } else if (format_tokens[i].contains("cad", Qt::CaseInsensitive)) { + crt_point.cad = line_tokens[i].toDouble(); + got_other = true; + } else if (format_tokens[i].contains("temp", Qt::CaseInsensitive)) { + crt_point.atemp = line_tokens[i].toDouble(); + got_other = true; + } else if (format_tokens[i].contains("pow", Qt::CaseInsensitive)) { + crt_point.power = line_tokens[i].toDouble(); + got_other = true; + } else { + if (!warned_of_format[i]) { + mlt_log_info(NULL, + "unknown format string[%d]: '%s', skipping", + i, + qPrintable(format_tokens[i])); + warned_of_format[i] = true; + } + } + } + + //process datetime now after the entire line is parsed: + if (got_qdate && got_qtime) { + QDateTime dt(qdate_part, qtime_part, Qt::UTC); + if (dt.isValid()) { + crt_point.time = dt.toMSecsSinceEpoch(); + valid_datetime = (crt_point.time != 0); + } + } else if (format_date_time_separated) { + crt_point.time = datetimeXMLstring_to_mseconds(qUtf8Printable(datetime_assembled), + qUtf8Printable( + datetime_format_assembled)); + valid_datetime = (crt_point.time != 0); + // mlt_log_info(NULL, + // "format_date_time_separated true, assembled input:'%s', assembled format:'%s', " + // "result=%" PRId64 "", + // qUtf8Printable(datetime_assembled), + // qUtf8Printable(datetime_format_assembled), + // crt_point.time); + } + } //end regex_mode / format_mode + bool valid_pt = (valid_datetime && got_other) || (got_latlon == 2); + if (!valid_pt) { + mlt_log_info(NULL, "q_parse_txt: skipping invalid line: '%s'\n", qUtf8Printable(line)); + if (nr_invalid_lines++ >= 10) { + mlt_log_warning(NULL, "q_parse_txt: 10 invalid lines in a row, aborting file\n"); + return 0; + } + continue; + } else { + nr_invalid_lines = 0; + } + + if (crt_point.time == GPS_UNINIT || crt_point.time == 0) { + //if no time is read (or bad format), we make it up starting from 1 (=1000ms) + no_time_count += 1; + crt_point.time = no_time_count * 1000; + } + gps_points_file.append(crt_point); + } //end of while reader.atEnd() + + return 1; +} + +//checks if there's a file with the same base name as original clip + extension of supported gps +//returns 0 on failure, 1 on success (and fills in auto_filename) +//TODO: do q_parse_file() here even if we process twice?// actually should we keep in cache all read files? +int q_auto_gps_file(const char *clip_path, char *auto_filename) +{ + // mlt_log_info(NULL, + // "q_auto_gps_file: checking for GPS file near '%s'", + // (clip_path ? clip_path : "NULL")); + if (clip_path) { + QFileInfo clip(QString::fromUtf8(clip_path)); + QStringList extensions_to_check = {".gpx", ".GPX", ".tcx", ".TCX", ".txt", ".TXT"}; + for (auto ext : extensions_to_check) { + QString guess = clip.dir().filePath(clip.completeBaseName() + ext); + if (QFile::exists(guess)) { + QByteArray guess_utf8 = guess.toUtf8(); + strncat(auto_filename, guess_utf8.constData(), PATH_MAX - strlen(auto_filename) - 1); + return 1; + } + } + } + return 0; } /* Reads file, parses it and stores the gps values in a gps_point_raw array (allocated inside) @@ -1327,99 +1688,167 @@ void qxml_parse_tcx(QXmlStreamReader &reader, gps_point_ll **gps_list, int *coun * swap_to_180 -> shifts all longitudes around with 180 becoming 0, only use it if user requests it * Returns 0 on failure, 1 on success. */ -int qxml_parse_file(gps_private_data gdata) +int q_parse_file(gps_private_data gdata) { - gps_point_ll *gps_list_head = NULL, *tmp = NULL; - int count_pts = 0, nr = 0; - char *filename = gdata.last_filename; - gps_point_raw *gps_array = NULL; + QVector gps_points_array; + int no_time_count = 0; //for gps tracks without time, used for ordering + QString filename = gdata.last_filename; int &swap_to_180 = *gdata.swap180; - // mlt_log_info(gdata.filter, "in qxml_parse_file, filename: %s, swap_to_180=%d\n", filename, swap_to_180); + // mlt_log_info(gdata.filter, "in q_parse_file, filename: %s\n", qUtf8Printable(filename)); QFile gps_file(filename); if (!gps_file.open(QFile::ReadOnly | QFile::Text)) { - mlt_log_warning(gdata.filter, "qxml_parse_file couldn't read file: %s\n", filename); + mlt_log_warning(gdata.filter, + "q_parse_file couldn't read file: %s\n", + qUtf8Printable(filename)); return 0; } - QXmlStreamReader qxml_reader(&gps_file); - qxml_reader.setNamespaceProcessing(false); - while (!qxml_reader.atEnd() && !qxml_reader.hasError()) { - qxml_reader.readNextStartElement(); - if (qxml_reader.isStartDocument()) { - // mlt_log_info(NULL, "qxml_parse_file: StartDocument found. Encoding=%s\n", qUtf8Printable(qxml_reader.documentEncoding().toString())); - continue; + if (filename.endsWith(".gpx", Qt::CaseInsensitive) + || filename.endsWith(".tcx", Qt::CaseInsensitive)) { + QXmlStreamReader qxml_reader(&gps_file); + qxml_reader.setNamespaceProcessing(false); + while (!qxml_reader.atEnd() && !qxml_reader.hasError()) { + qxml_reader.readNextStartElement(); + if (qxml_reader.isStartDocument()) { + mlt_log_info(NULL, + "q_parse_file: StartDocument found. Encoding=%s\n", + qUtf8Printable(qxml_reader.documentEncoding().toString())); + continue; + } + if (qxml_reader.name() == QStringLiteral("TrainingCenterDatabase")) { + qxml_parse_tcx(qxml_reader, gps_points_array, no_time_count); + } else if (qxml_reader.name() == QStringLiteral("gpx")) { + qxml_parse_gpx(qxml_reader, gps_points_array, no_time_count); + } else { + mlt_log_warning(gdata.filter, + "qxml_parse_file: fail to parse file: %s, unknown root element: " + "%s. Aborting.\n", + qUtf8Printable(filename), + qUtf8Printable(qxml_reader.name().toString())); + return 0; + } } - // mlt_log_info(NULL, " readNext element tag: %s, type=%d\n", qUtf8Printable(qxml_reader.name().toString()), qxml_reader.tokenType()); - if (qxml_reader.name() == QStringLiteral("TrainingCenterDatabase")) { - qxml_parse_tcx(qxml_reader, &gps_list_head, &count_pts); - } else if (qxml_reader.name() == QStringLiteral("gpx")) { - qxml_parse_gpx(qxml_reader, &gps_list_head, &count_pts); - } else { - mlt_log_warning( - gdata.filter, - "qxml_parse_file: fail to parse file: %s, unknown root element: %s. Aborting.\n", - filename, - qUtf8Printable(qxml_reader.name().toString())); + if (qxml_reader.hasError()) { + mlt_log_info(NULL, + "qxml_reader.hasError! line:%u, errString:%s\n", + (unsigned) qxml_reader.lineNumber(), + qUtf8Printable(qxml_reader.errorString())); return 0; } - } - if (qxml_reader.hasError()) { - mlt_log_info(NULL, - "qxml_reader.hasError! line:%u, errString:%s\n", - (unsigned) qxml_reader.lineNumber(), - qUtf8Printable(qxml_reader.errorString())); - return 0; + + qxml_reader.clear(); + } else { + QTextStream qin(&gps_file); + if (q_parse_txt(qin, gps_points_array, no_time_count) == 0) { + mlt_log_error(NULL, + "txt file (%s) processing failed. Make sure it starts with a recognised " + "format line!", + qUtf8Printable(filename)); + mlt_log_info(NULL, "Sample format line: '%s'", qUtf8Printable(sample_format_line)); + return 0; + } + // mlt_log_info(NULL, "printing the read array, size=%" PRId64 "", gps_points_array.size()); + // for (int i=0; i< gps_points_array.size(); i++) + // _mlt_log_gps_point(i, gps_points_array[i]); } - qxml_reader.clear(); //not sure if really needed if not reused + const int gps_pt_count = gps_points_array.size(); + // mlt_log_info(NULL, + // "back from parsing the file(s) '%s', gps_points_array.size()=%d, " + // "no_time_count=%d", + // qPrintable(filename), + // gps_pt_count, + // no_time_count); - if (count_pts < 2) { + if (gps_pt_count < 2) { mlt_log_warning(gdata.filter, - "qxml_parse_file: less than 2 gps points read (%d). Aborting.\n", - count_pts); + "q_parse_file: less than 2 gps points read (%d). Aborting.\n", + gps_pt_count); return 0; } - *gdata.ptr_to_gps_points_r = (gps_point_raw *) calloc(count_pts, sizeof(gps_point_raw)); - gps_array = *gdata.ptr_to_gps_points_r; //just an alias + //make sure points are in chronological order (they usually are) + if (!std::is_sorted(gps_points_array.begin(), gps_points_array.end())) + std::sort(gps_points_array.begin(), gps_points_array.end()); + + //if there are entries with and without time, drop the ones without valid time + bool skip_notimes = false; + if (no_time_count > 0 && no_time_count < gps_pt_count) { + skip_notimes = true; + mlt_log_info(NULL, + "q_parse_file: will skip %d/%d points without valid time", + no_time_count, + gps_pt_count); + } + + //allocate C style array + *gdata.ptr_to_gps_points_r = (gps_point_raw *) calloc(gps_pt_count, sizeof(gps_point_raw)); + gps_point_raw *&gps_array = *gdata.ptr_to_gps_points_r; //alias by reference for realloc if (gps_array == NULL) { mlt_log_error(gdata.filter, - "malloc error (size=%u)\n", - (unsigned) (count_pts * sizeof(gps_point_raw))); - } - *gdata.gps_points_size = count_pts; - - //copy points to private array and free list - while (gps_list_head) { - gps_array[nr++] = gps_list_head->gp; - tmp = gps_list_head; - gps_list_head = gps_list_head->next; - free(tmp); + "malloc error (size=%u), out of memory?\n", + (unsigned) (gps_pt_count * sizeof(gps_point_raw))); + return 0; } + *gdata.gps_points_size = gps_pt_count; - //search to find if we cross the 180 and/or the 0 meridian + //copy points to private array (and skip pts if needed) + int nr = 0; + //also do the 180 crossing check bool crosses0 = false, crosses180 = false; - for (int i = 0; i < *gdata.gps_points_size - 1; i++) { - double crt = gps_array[i].lon; - double next = gps_array[i + 1].lon; - if (crt == GPS_UNINIT || next == GPS_UNINIT) + double last_lon = GPS_UNINIT; + for (auto &pt : gps_points_array) { + if (skip_notimes && pt.time <= no_time_count * 1000) + continue; + if (nr - 1 >= 0 && pt.time == gps_array[nr - 1].time) continue; - if ((crt < 0 && next > 0) || (crt > 0 && next < 0)) { - if (crt - next > 180 || next - crt > 180) - crosses180 = true; - else - crosses0 = true; + gps_array[nr++] = pt; + + //180 crossing logic here + if (last_lon != GPS_UNINIT && pt.lon != GPS_UNINIT) { + if ((last_lon < 0 && pt.lon > 0) || (pt.lon < 0 && last_lon > 0)) { + if (fabs(last_lon - pt.lon) > 180) + crosses180 = true; + else + crosses0 = true; + } + } + if (pt.lon != GPS_UNINIT) + last_lon = pt.lon; + } + + if (nr >= 2) { + gps_point_raw *tmp = (gps_point_raw *) realloc(gps_array, nr * sizeof(gps_point_raw)); + if (tmp != NULL) { + gps_array = tmp; + *gdata.gps_points_size = nr; + } else { + mlt_log_error(gdata.filter, + "realloc error (size=%u), out of memory?\n", + (unsigned) (nr * sizeof(gps_point_raw))); + free(gps_array); + gps_array = NULL; + *gdata.gps_points_size = 0; + return 0; } + } else { + mlt_log_warning(gdata.filter, + "q_parse_file: not enough (<2) valid GPS points after processing " + "(initially: %d). Aborting.\n", + gps_pt_count); + return 0; } - mlt_log_info( - NULL, - "_automatic 180 meridian detection: crosses180=%d, crosses0=%d --> swapping180=%d\n", - crosses180, - crosses0, - (crosses180 && !crosses0)); + + // mlt_log_info( + // NULL, + // "_automatic 180 meridian detection: crosses180=%d, crosses0=%d --> swapping180=%d\n", + // crosses180, + // crosses0, + // (crosses180 && !crosses0)); + //if only 180 is crossed, we swap, otherswise we don't do anything if (crosses180 && !crosses0) { swap_to_180 = 1; @@ -1429,24 +1858,8 @@ int qxml_parse_file(gps_private_data gdata) swap_to_180 = 0; // //debug result: - // for (int i = 0; i < *gdata.gps_points_size; i += 100) { - // gps_point_raw *crt_point = &gps_array[i]; - // mlt_log_info(NULL, - // "_xml_parse_file read point[%d]: time:%d, lat:%f, lon:%f, speed:%f, " - // "distance:%f, ele:%f, hr:%f, bearing:%f, cadence:%f, temp:%f, power:%f\n", - // i, - // crt_point->time / 1000, - // crt_point->lat, - // crt_point->lon, - // crt_point->speed, - // crt_point->total_dist, - // crt_point->ele, - // crt_point->hr, - // crt_point->bearing, - // crt_point->cad, - // crt_point->atemp, - // crt_point->power); - //} + // for (int i = 0; i < *gdata.gps_points_size; i += 100) + // _mlt_log_gps_point(i, gps_array[i]); return 1; } diff --git a/src/modules/qt/gps_parser.h b/src/modules/qt/gps_parser.h index f00805fba..5c8251efa 100644 --- a/src/modules/qt/gps_parser.h +++ b/src/modules/qt/gps_parser.h @@ -22,6 +22,7 @@ #include #include #include +#include #include // #include @@ -31,25 +32,20 @@ #define GPS_UNINIT -9999 #define MAX_PRINT_TIMESTAMP_NS 32 -typedef struct +struct gps_point_raw { double lat, lon, lat_projected, speed, total_dist, ele, hr, bearing, cad, atemp, power; int64_t time; //epoch milliseconds -} gps_point_raw; + bool operator<(const gps_point_raw &pt2) const { return time < pt2.time; } +}; -typedef struct gps_point_raw_list -{ - gps_point_raw gp; - struct gps_point_raw_list *next; -} gps_point_ll; - -typedef struct +struct gps_point_proc { double lat, lon, lat_projected, speed, speed_vertical, speed_3d, total_dist, ele, hr, bearing, cad, atemp, power; int64_t time; double d_elev, elev_up, elev_down, dist_up, dist_down, dist_flat, grade_p; -} gps_point_proc; +}; //0 is a valid value for many fields, use GPS_UNINIT (-9999) to differentiate missing values static const gps_point_raw uninit_gps_raw_point = {.lat = GPS_UNINIT, @@ -88,7 +84,7 @@ static const gps_point_proc uninit_gps_proc_point = {.lat = GPS_UNINIT, .grade_p = GPS_UNINIT}; //structure used to ease argument passing between filter and GPS parser api -typedef struct +struct gps_private_data { gps_point_raw *gps_points_r; //raw gps data from file gps_point_proc *gps_points_p; //processed gps data @@ -105,10 +101,12 @@ typedef struct int last_smooth_lvl; char *last_filename; //gps file fullpath mlt_filter filter; -} gps_private_data; +}; #define has_valid_location(x) (((x).lat) != GPS_UNINIT && ((x).lon) != GPS_UNINIT) #define has_valid_location_ptr(x) ((x) && ((x)->lat) != GPS_UNINIT && ((x)->lon) != GPS_UNINIT) +#define is_valid_latitude(lat) ((lat) >= -90 && (lat) <= 90) +#define is_valid_longitude(lon) ((lon) >= -180 && (lon) <= 180) #define MATH_PI 3.14159265358979323846 #define to_rad(x) ((x) *MATH_PI / 180.0) @@ -133,6 +131,10 @@ void process_gps_smoothing(gps_private_data gdata, char do_processing); double project_latitude(double lat); double unproject_latitude(double lat_projected); +extern QStringList accepted_distance_formats; +extern QStringList accepted_speed_formats; +bool qstring_contains_any_qstringlist(const QString &main_string, const QStringList &list); + double weighted_middle_double( double v1, int64_t t1, double v2, int64_t t2, int64_t new_t, int max_gps_diff_ms); int64_t weighted_middle_int64( @@ -142,9 +144,15 @@ gps_point_proc weighted_middle_point_proc(gps_point_proc *p1, int64_t new_t, int max_gps_diff_ms); -void qxml_parse_gpx(QXmlStreamReader &reader, gps_point_ll **gps_list, int *count_pts); -void qxml_parse_tcx(QXmlStreamReader &reader, gps_point_ll **gps_list, int *count_pts); -int qxml_parse_file(gps_private_data gdata); +void qxml_parse_gpx(QXmlStreamReader &reader, + QVector &gps_points_file, + int &no_time_count); +void qxml_parse_tcx(QXmlStreamReader &reader, + QVector &gps_points_file, + int &no_time_count); +int q_parse_txt(QTextStream &reader, QVector &gps_points_file, int &no_time_count); +int q_auto_gps_file(const char *clip_path, char *auto_filename); +int q_parse_file(gps_private_data gdata); #define swap_180_if_needed(lon) (pdata->swap_180 ? get_180_swapped(lon) : lon) double get_180_swapped(double lon);