From fa6d719d26448825de339b503b0688dec42f8132 Mon Sep 17 00:00:00 2001 From: Daniel F Date: Thu, 1 Oct 2026 17:37:49 +0300 Subject: [PATCH 01/14] gps parser - added working txt reader with prompt(+default) check --- src/modules/qt/gps_parser.cpp | 315 ++++++++++++++++++++++++++++++---- 1 file changed, 280 insertions(+), 35 deletions(-) diff --git a/src/modules/qt/gps_parser.cpp b/src/modules/qt/gps_parser.cpp index 58f6610c5..2cf27d4c2 100644 --- a/src/modules/qt/gps_parser.cpp +++ b/src/modules/qt/gps_parser.cpp @@ -23,11 +23,11 @@ #endif #include "gps_parser.h" #include +#include #include #define _x (const xmlChar *) #define _s (const char *) -#define _qxml_getthestring(s) qUtf8Printable((s).toString()) // Maps are usually in ESPG3857 format, not in GPS format EPSG4326 / WG84. We have to project // the latitude, otherwise big maps have wrong aspect ratio @@ -67,6 +67,23 @@ double get_avg_gps_time_ms(gps_private_data gdata) return (double) tdiff / gp_size; } +/* int64_t datetime_unknown_to_mseconds (const char *text) +{ + //Qt::ISODateWithMs, + //Qt::ISODate, + const QStringList date_formats { + "yyyy/MM/dd HH:mm:ss.zzz", + "yyyy/MM/dd HH:mm:ss", + "yyyy-MM-dd HH:mm:ss.zzz", + "yyyy-MM-dd HH:mm:ss", + "yyyy/MM/dd", + "yyyy-MM-dd", + "HH:mm:ss.zzz", + "HH:mm:ss" + }; + +} */ + /* Converts the datetime string from gps file into seconds since epoch * Note: assumes UTC */ @@ -81,7 +98,11 @@ int64_t datetimeXMLstring_to_mseconds(const char *text, const char *format /* = datetime = QDateTime::fromString(QString(text), Qt::ISODateWithMs); if (!datetime.isValid()) { - mlt_log_warning(NULL, "[filter gpstext] %s failed on string: %.25s\n", __FUNCTION__, text); + mlt_log_warning(NULL, + "datetimeXMLstring_to_mseconds - QDateTime::fromString failed on string: " + "'%.25s' with format: '%s'\n", + text, + (format ? format : "NULL")); return 0; } @@ -1066,7 +1087,7 @@ void qxml_parse_gpx(QXmlStreamReader &reader, gps_point_ll **gps_list, int *coun */ while (!reader.atEnd() && !reader.hasError()) { reader.readNext(); - // mlt_log_info(NULL, " readNext tag: %s type = %d\n", _qxml_getthestring(reader.name()), reader.tokenType()); + // mlt_log_info(NULL, " readNext tag: %s type = %d\n", qUtf8Printable(reader.name()), reader.tokenType()); if (reader.isStartElement() && reader.name() == QStringLiteral("trkpt")) { gps_point_raw crt_point = uninit_gps_raw_point; @@ -1082,7 +1103,7 @@ void qxml_parse_gpx(QXmlStreamReader &reader, gps_point_ll **gps_list, int *coun && !(reader.name() == QStringLiteral("trkpt") && reader.tokenType() == QXmlStreamReader::EndElement)) //until closing trkpt { - // mlt_log_info(NULL, "[trkpt->1] readNext tag: %s, type=%d\n", _qxml_getthestring(reader.name()), reader.tokenType()); + // mlt_log_info(NULL, "[trkpt->1] readNext tag: %s, type=%d\n", qUtf8Printable(reader.name()), reader.tokenType()); if (!reader.isStartElement()) continue; @@ -1211,7 +1232,7 @@ void qxml_parse_tcx(QXmlStreamReader &reader, gps_point_ll **gps_list, int *coun */ while (!reader.atEnd() && !reader.hasError()) { reader.readNext(); - // mlt_log_info(NULL, " readNextStartElement tag: %s\n", _qxml_getthestring(reader.name())); + // mlt_log_info(NULL, " readNextStartElement tag: %s\n", qUtf8Printable(reader.name())); if (reader.isStartElement() && reader.name() == QStringLiteral("Trackpoint")) { gps_point_raw crt_point = uninit_gps_raw_point; while ( @@ -1220,7 +1241,7 @@ void qxml_parse_tcx(QXmlStreamReader &reader, gps_point_ll **gps_list, int *coun && reader.tokenType() == QXmlStreamReader::EndElement)) //loop until we hit the closing { - // mlt_log_info(NULL, "[Trackpoint->1] readNext tag: %s, type=%d\n", _qxml_getthestring(reader.name()), reader.tokenType()); + // mlt_log_info(NULL, "[Trackpoint->1] readNext tag: %s, type=%d\n", qUtf8Printable(reader.name()), reader.tokenType()); if (!reader.isStartElement()) continue; @@ -1322,6 +1343,215 @@ void qxml_parse_tcx(QXmlStreamReader &reader, gps_point_ll **gps_list, int *coun } } +//tries to parse a .txt file containing GPS data; returns 0 for failure, 1 success +//first line of the file must contain a format helper that starts with #delimiter= +int q_parse_txt(QTextStream &reader, gps_point_ll **gps_list, int *count_pts) +{ + int64_t last_time = -1; + //support no time in file because many services (Strava included) do this for exported routes + gps_point_ll *no_time_head = NULL, *no_time_prev = NULL; + int no_time_count = 0; + + //example from my dashcam (added extra spaces for readability): + //2025/06/12 02:45:01 N:45.123456 E:26.123456 0.0 km/h 0.00 72.50 10 x:-4.13 y:0.27 z:0.49 + //date time lat lon speed (unit) heading altitude satelites xyz_gravity_sensor + //expected format line: + //#delimiter=" " timestamp="yyyy/MM/dd" timestamp="hh:mm:ss" lat lon speed skip heading altitude + + //NOTE: there's probably a very limited number of common formats, and 99% of them start with date+time lat lon + // so if there's enough interest in this, it could be automated with a decent success rate + QString default_format_line + = "#delimiter=\" \" timestamp=\"yyyy/MM/dd\" timestamp=\"hh:mm:ss\" latitude longitude " + "speed skip heading altitude"; + + //read the format line, check for delimiter, use default if fails + QString delimiter; + QStringList format_tokens, format_tokens_arguments; + if (!reader.atEnd()) { + QString format_line = reader.readLine(); + QString first_keyword = "#delimiter="; + if (!format_line.startsWith(first_keyword)) { + mlt_log_warning( + NULL, + "no format line found at the start of the file, using the default of: " + "'%s'\n (note: tokens that don't match or are extra at the end are skipped)", + qUtf8Printable(default_format_line)); + format_line = default_format_line; + reader.seek(0); + } + //read and store the content of delimiter=" " + int firstQ = format_line.indexOf('"'); + int secondQ = format_line.indexOf('"', firstQ + 1); + if (firstQ != -1 && secondQ != -1) { + delimiter = format_line.mid(firstQ + 1, secondQ - (firstQ + 1)); + if (!delimiter.isEmpty()) { + QString after_delimiter = format_line.mid(secondQ + 1); + format_tokens = after_delimiter.split(delimiter, Qt::SkipEmptyParts); + } else + return 0; + } else + return 0; + + // mlt_log_debug(NULL, + // "q_parse_txt(): format_line='%s', delimiter='%s', format_tokens found=%lld", + // qUtf8Printable(format_line), + // qUtf8Printable(delimiter), + // format_tokens.size()); + } + + //extract timestamp with format + format_tokens_arguments.resize(format_tokens.size()); + bool date_time_separated = false; + for (int i = 0; i < format_tokens.size(); i++) { + if (format_tokens[i].contains("timestamp=", Qt::CaseInsensitive)) { + int firstQ = format_tokens[i].indexOf('"'); + int secondQ = format_tokens[i].indexOf('"', firstQ + 1); + format_tokens_arguments[i] = format_tokens[i].mid(firstQ + 1, secondQ - (firstQ + 1)); + if (!format_tokens_arguments[i].contains('y', Qt::CaseInsensitive)) + date_time_separated = true; + } + } + + //for each line: for each token in the format_line tokens process the coresponding token from file + while (!reader.atEnd()) { + QString line = reader.readLine(); + QStringList tokens = line.split(delimiter, Qt::SkipEmptyParts); + gps_point_raw crt_point = uninit_gps_raw_point; + QString datetime_assembled, datetime_format_assembled; + + for (int i = 0; i < format_tokens.size() && i < tokens.size(); i++) { + /* mlt_log_info(NULL, + "format[%d]: %s, token[%d]; %s", + i, + qUtf8Printable(format_tokens[i]), + i, + tokens[i].toUtf8().constData()); */ + if (format_tokens[i].contains("timestamp=", Qt::CaseInsensitive)) { + datetime_assembled.append(tokens[i]); + datetime_assembled.append(delimiter); + datetime_format_assembled.append(format_tokens_arguments[i]); + datetime_format_assembled.append(delimiter); + } else if (format_tokens[i].contains("timestamp", Qt::CaseInsensitive)) { + crt_point.time = datetimeXMLstring_to_mseconds(qUtf8Printable(tokens[i])); + } else if (format_tokens[i].contains("lat", Qt::CaseInsensitive)) { + double negative = (tokens[i].startsWith('s', Qt::CaseInsensitive) ? -1 : 1); + //skip to the number + int number_at = tokens[i].indexOf(QRegularExpression("[-+0-9]")); + crt_point.lat = tokens[i].mid(number_at).toDouble() * negative; + crt_point.lat_projected = project_latitude(crt_point.lat); + } else if (format_tokens[i].contains("lon", Qt::CaseInsensitive)) { + double negative = (tokens[i].startsWith('w', Qt::CaseInsensitive) ? -1 : 1); + int number_at = tokens[i].indexOf(QRegularExpression("[-+0-9]")); + crt_point.lon = tokens[i].mid(number_at).toDouble() * negative; + } else if (format_tokens[i].contains("speed", Qt::CaseInsensitive)) { + crt_point.speed = tokens[i].toDouble(); + if (i + 1 < format_tokens.size()) { + if (tokens[i + 1].contains("km", Qt::CaseInsensitive)) + crt_point.speed /= 3.6; + else if (tokens[i].contains("mi", Qt::CaseInsensitive) + || tokens[i].contains("m/h", Qt::CaseInsensitive)) + crt_point.speed /= 2.23693629; + } + } else if (format_tokens[i].contains("alt", Qt::CaseInsensitive) + || format_tokens[i].contains("ele", Qt::CaseInsensitive)) { + crt_point.ele = tokens[i].toDouble(); + } else if (format_tokens[i].contains("head", Qt::CaseInsensitive) + || format_tokens[i].contains("bear", Qt::CaseInsensitive)) { + crt_point.bearing = tokens[i].toDouble(); + } + } + + //process datetime now after the entire line is parsed: + if (date_time_separated) { + crt_point.time = datetimeXMLstring_to_mseconds(qUtf8Printable(datetime_assembled), + qUtf8Printable( + datetime_format_assembled)); + // mlt_log_info(NULL, + // "date_time_separated true, assembled input:'%s', assembled format:'%s', " + // "result=%lld", + // qUtf8Printable(datetime_assembled), + // qUtf8Printable(datetime_format_assembled), + // crt_point.time); + } + + // mlt_log_info(NULL, + // "crt_point parsed txt: time:%lld, lat:%f, lon:%f, speed:%f, " + // "distance:%f, ele:%f, hr:%f, bearing:%f, cadence:%f, temp:%f, power:%f\n", + // crt_point.time, + // crt_point.lat, + // crt_point.lon, + // crt_point.speed, + // crt_point.total_dist, + // crt_point.ele, + // crt_point.hr, + // crt_point.bearing, + // crt_point.cad, + // crt_point.atemp, + // crt_point.power); + + //we parsed a line, now add it to our valid list + if (crt_point.time != GPS_UNINIT && crt_point.time > last_time) { + *gps_list = (gps_point_ll *) calloc(1, sizeof(gps_point_ll)); + if (*gps_list == NULL) + return 0; + *count_pts += 1; + (*gps_list)->gp = crt_point; + (*gps_list)->next = NULL; + gps_list = &(*gps_list)->next; + last_time = crt_point.time; + } else if (crt_point.time == GPS_UNINIT) { + //if no time is stored, we make it up starting from 1 + gps_point_ll *tmp = (gps_point_ll *) calloc(1, sizeof(gps_point_ll)); + if (tmp == NULL) + return 0; + if (no_time_head == NULL) { + no_time_head = tmp; + no_time_prev = tmp; + } else { + no_time_prev->next = tmp; + no_time_prev = tmp; + } + no_time_count += 1; + crt_point.time = no_time_count * 1000; + tmp->gp = crt_point; + tmp->next = NULL; + } else + mlt_log_info(NULL, + "qxml_parse_tcx: skipping point due to time [%d] %f,%f - crt:%u.%u, " + "last:%u.%u\n", + *count_pts, + crt_point.lat, + crt_point.lon, + (unsigned) (crt_point.time / 1000), + (unsigned) (crt_point.time % 1000), + (unsigned) (last_time / 1000), + (unsigned) (last_time % 1000)); + } //end of while reader.atEnd() + + //if no points had time, we replace the real list with the no_time one + if (*count_pts == 0 && no_time_count > 0) { + mlt_log_info(NULL, + "qxml_parse_gpx: all GPS points are missing time values, inserting each " + "point at 1 second interval!\n"); + *gps_list = no_time_head; + *count_pts = no_time_count; + } else if (*count_pts > 0 && no_time_count > 0) { + //cleanup the no_time list as we're not using it + mlt_log_info(NULL, + "qxml_parse_gpx: %d GPS points have no time in original file and have been " + "discarded (total valid points kept: %d)!\n", + no_time_count, + *count_pts); + while (no_time_head) { + gps_point_ll *tmp = no_time_head; + no_time_head = no_time_head->next; + free(tmp); + } + } + + return 1; +} + /* Reads file, parses it and stores the gps values in a gps_point_raw array (allocated inside) * and its size in gps_points_size. * swap_to_180 -> shifts all longitudes around with 180 becoming 0, only use it if user requests it @@ -1331,7 +1561,7 @@ int qxml_parse_file(gps_private_data gdata) { gps_point_ll *gps_list_head = NULL, *tmp = NULL; int count_pts = 0, nr = 0; - char *filename = gdata.last_filename; + QString filename = gdata.last_filename; gps_point_raw *gps_array = NULL; int &swap_to_180 = *gdata.swap180; @@ -1340,41 +1570,56 @@ int qxml_parse_file(gps_private_data gdata) QFile gps_file(filename); if (!gps_file.open(QFile::ReadOnly | QFile::Text)) { - mlt_log_warning(gdata.filter, "qxml_parse_file couldn't read file: %s\n", filename); + mlt_log_warning(gdata.filter, + "qxml_parse_file couldn't read file: %s\n", + qUtf8Printable(filename)); return 0; } - QXmlStreamReader qxml_reader(&gps_file); - qxml_reader.setNamespaceProcessing(false); - while (!qxml_reader.atEnd() && !qxml_reader.hasError()) { - qxml_reader.readNextStartElement(); - if (qxml_reader.isStartDocument()) { - // mlt_log_info(NULL, "qxml_parse_file: StartDocument found. Encoding=%s\n", qUtf8Printable(qxml_reader.documentEncoding().toString())); - continue; + if (filename.endsWith(".txt", Qt::CaseInsensitive)) { + QTextStream qin(&gps_file); + if (q_parse_txt(qin, &gps_list_head, &count_pts) == 0) { + mlt_log_error(NULL, + "txt file (%s) does not start with a recognised format line, can't process!\n\ + \tsample format line: \n#delimiter=\" \" timestamp=\"yyyy/MM/dd\" timestamp=\"hh:mm:ss\" latitude longitude speed skip heading altitude", + qUtf8Printable(filename)); + return 0; } - // mlt_log_info(NULL, " readNext element tag: %s, type=%d\n", qUtf8Printable(qxml_reader.name().toString()), qxml_reader.tokenType()); - if (qxml_reader.name() == QStringLiteral("TrainingCenterDatabase")) { - qxml_parse_tcx(qxml_reader, &gps_list_head, &count_pts); - } else if (qxml_reader.name() == QStringLiteral("gpx")) { - qxml_parse_gpx(qxml_reader, &gps_list_head, &count_pts); - } else { - mlt_log_warning( - gdata.filter, - "qxml_parse_file: fail to parse file: %s, unknown root element: %s. Aborting.\n", - filename, - qUtf8Printable(qxml_reader.name().toString())); + } else { + QXmlStreamReader qxml_reader(&gps_file); + qxml_reader.setNamespaceProcessing(false); + while (!qxml_reader.atEnd() && !qxml_reader.hasError()) { + qxml_reader.readNextStartElement(); + if (qxml_reader.isStartDocument()) { + mlt_log_info(NULL, + "qxml_parse_file: StartDocument found. Encoding=%s\n", + qUtf8Printable(qxml_reader.documentEncoding().toString())); + continue; + } + // mlt_log_info(NULL, " readNext element tag: %s, type=%d\n", qUtf8Printable(qxml_reader.name().toString()), qxml_reader.tokenType()); + if (qxml_reader.name() == QStringLiteral("TrainingCenterDatabase")) { + qxml_parse_tcx(qxml_reader, &gps_list_head, &count_pts); + } else if (qxml_reader.name() == QStringLiteral("gpx")) { + qxml_parse_gpx(qxml_reader, &gps_list_head, &count_pts); + } else { + mlt_log_warning(gdata.filter, + "qxml_parse_file: fail to parse file: %s, unknown root element: " + "%s. Aborting.\n", + qUtf8Printable(filename), + qUtf8Printable(qxml_reader.name().toString())); + return 0; + } + } + if (qxml_reader.hasError()) { + mlt_log_info(NULL, + "qxml_reader.hasError! line:%u, errString:%s\n", + (unsigned) qxml_reader.lineNumber(), + qUtf8Printable(qxml_reader.errorString())); return 0; } - } - if (qxml_reader.hasError()) { - mlt_log_info(NULL, - "qxml_reader.hasError! line:%u, errString:%s\n", - (unsigned) qxml_reader.lineNumber(), - qUtf8Printable(qxml_reader.errorString())); - return 0; - } - qxml_reader.clear(); //not sure if really needed if not reused + qxml_reader.clear(); //not sure if really needed if not reused + } if (count_pts < 2) { mlt_log_warning(gdata.filter, From 804396d3c77b9bcd50c6869a667ae9c4960926bb Mon Sep 17 00:00:00 2001 From: Daniel F Date: Fri, 2 Oct 2026 01:36:03 +0300 Subject: [PATCH 02/14] GPS parser: migrate gps_list to QArray; drop typedef in structs --- src/modules/qt/filter_gpsgraphic.h | 20 +-- src/modules/qt/gps_parser.cpp | 258 ++++++++++++++--------------- src/modules/qt/gps_parser.h | 16 +- 3 files changed, 142 insertions(+), 152 deletions(-) diff --git a/src/modules/qt/filter_gpsgraphic.h b/src/modules/qt/filter_gpsgraphic.h index 9129432cc..a7707ae93 100644 --- a/src/modules/qt/filter_gpsgraphic.h +++ b/src/modules/qt/filter_gpsgraphic.h @@ -33,7 +33,7 @@ struct s_gps_data_bounds namespace { -typedef struct +struct private_data { gps_point_raw *gps_points_r; //raw gps data from file gps_point_proc *gps_points_p; //processed gps data @@ -55,31 +55,31 @@ typedef struct double bg_img_scaled_width, bg_img_scaled_height; QRectF bg_img_matched_rect; int swap_180; -} private_data; +}; } // namespace -typedef enum { +enum gpsg_data_sources { gspg_location_src = 0, gpsg_altitude_src, gpsg_hr_src, gpsg_speed_src -} gpsg_data_sources; +}; -typedef enum { +enum gpsg_data_subids { gpsg_latitude_id = gspg_location_src, gpsg_longitude_id = gpsg_latitude_id + 100 -} gpsg_data_subids; +}; -typedef enum { +enum gpsg_graph_types { gpsg_2d_map_graph = 0, gpsg_crop_center_graph, gpsg_speedometer_graph //gpsg_ladder_graph, //was thinking a vertical ruler/gauge thing for altitude, couldn't design something that actually looks good //gpsg_compass_graph //probably easy to adapt from speedometer but not really that important to be worth the extra code -} gpsg_graph_types; +}; -typedef enum { +enum gpsg_color_styles { gpsg_color_by_solid = 0, gpsg_color_by_solid_past_future, gpsg_color_by_solid_past, @@ -93,7 +93,7 @@ typedef enum { gpsg_color_by_speed_max100, gpsg_color_by_grade_max90, gpsg_color_by_grade_max20 -} gpsg_color_styles; +}; struct s_base_crops { diff --git a/src/modules/qt/gps_parser.cpp b/src/modules/qt/gps_parser.cpp index 2cf27d4c2..9c6b01752 100644 --- a/src/modules/qt/gps_parser.cpp +++ b/src/modules/qt/gps_parser.cpp @@ -100,7 +100,7 @@ int64_t datetimeXMLstring_to_mseconds(const char *text, const char *format /* = if (!datetime.isValid()) { mlt_log_warning(NULL, "datetimeXMLstring_to_mseconds - QDateTime::fromString failed on string: " - "'%.25s' with format: '%s'\n", + "'%.25s' with format: '%s'; returning 0.", text, (format ? format : "NULL")); return 0; @@ -1345,13 +1345,8 @@ void qxml_parse_tcx(QXmlStreamReader &reader, gps_point_ll **gps_list, int *coun //tries to parse a .txt file containing GPS data; returns 0 for failure, 1 success //first line of the file must contain a format helper that starts with #delimiter= -int q_parse_txt(QTextStream &reader, gps_point_ll **gps_list, int *count_pts) +int q_parse_txt(QTextStream &reader, QVector & gps_points_file, int64_t& last_time, bool& all_sorted, int& no_time_count, int&eq_time_count) { - int64_t last_time = -1; - //support no time in file because many services (Strava included) do this for exported routes - gps_point_ll *no_time_head = NULL, *no_time_prev = NULL; - int no_time_count = 0; - //example from my dashcam (added extra spaces for readability): //2025/06/12 02:45:01 N:45.123456 E:26.123456 0.0 km/h 0.00 72.50 10 x:-4.13 y:0.27 z:0.49 //date time lat lon speed (unit) heading altitude satelites xyz_gravity_sensor @@ -1489,65 +1484,20 @@ int q_parse_txt(QTextStream &reader, gps_point_ll **gps_list, int *count_pts) // crt_point.atemp, // crt_point.power); - //we parsed a line, now add it to our valid list - if (crt_point.time != GPS_UNINIT && crt_point.time > last_time) { - *gps_list = (gps_point_ll *) calloc(1, sizeof(gps_point_ll)); - if (*gps_list == NULL) - return 0; - *count_pts += 1; - (*gps_list)->gp = crt_point; - (*gps_list)->next = NULL; - gps_list = &(*gps_list)->next; + //we parsed a line, check the time and store it in the array + if (crt_point.time != GPS_UNINIT && crt_point.time != 0) { + if (crt_point.time < last_time) + all_sorted = false; + else if (crt_point.time == last_time) + eq_time_count +=1; last_time = crt_point.time; - } else if (crt_point.time == GPS_UNINIT) { - //if no time is stored, we make it up starting from 1 - gps_point_ll *tmp = (gps_point_ll *) calloc(1, sizeof(gps_point_ll)); - if (tmp == NULL) - return 0; - if (no_time_head == NULL) { - no_time_head = tmp; - no_time_prev = tmp; - } else { - no_time_prev->next = tmp; - no_time_prev = tmp; - } + } else if (crt_point.time == GPS_UNINIT || crt_point.time == 0) { + //if no time is read (or bad format), we make it up starting from 1 no_time_count += 1; crt_point.time = no_time_count * 1000; - tmp->gp = crt_point; - tmp->next = NULL; - } else - mlt_log_info(NULL, - "qxml_parse_tcx: skipping point due to time [%d] %f,%f - crt:%u.%u, " - "last:%u.%u\n", - *count_pts, - crt_point.lat, - crt_point.lon, - (unsigned) (crt_point.time / 1000), - (unsigned) (crt_point.time % 1000), - (unsigned) (last_time / 1000), - (unsigned) (last_time % 1000)); - } //end of while reader.atEnd() - - //if no points had time, we replace the real list with the no_time one - if (*count_pts == 0 && no_time_count > 0) { - mlt_log_info(NULL, - "qxml_parse_gpx: all GPS points are missing time values, inserting each " - "point at 1 second interval!\n"); - *gps_list = no_time_head; - *count_pts = no_time_count; - } else if (*count_pts > 0 && no_time_count > 0) { - //cleanup the no_time list as we're not using it - mlt_log_info(NULL, - "qxml_parse_gpx: %d GPS points have no time in original file and have been " - "discarded (total valid points kept: %d)!\n", - no_time_count, - *count_pts); - while (no_time_head) { - gps_point_ll *tmp = no_time_head; - no_time_head = no_time_head->next; - free(tmp); } - } + gps_points_file.append(crt_point); + } //end of while reader.atEnd() return 1; } @@ -1559,13 +1509,19 @@ int q_parse_txt(QTextStream &reader, gps_point_ll **gps_list, int *count_pts) */ int qxml_parse_file(gps_private_data gdata) { - gps_point_ll *gps_list_head = NULL, *tmp = NULL; - int count_pts = 0, nr = 0; + QVector gps_points_array; + int64_t last_time = -1; //used to check if points are read already ordered + int no_time_count = 0; //for gps tracks without time, used for ordering + int eq_time_count = 0; //we only support unique times + bool all_sorted = true; //skip the sorting at the end + + // gps_point_ll *gps_list_head = NULL, *tmp = NULL; + // int count_pts = 0, nr = 0; QString filename = gdata.last_filename; - gps_point_raw *gps_array = NULL; + // gps_point_raw *gps_array = NULL; int &swap_to_180 = *gdata.swap180; - // mlt_log_info(gdata.filter, "in qxml_parse_file, filename: %s, swap_to_180=%d\n", filename, swap_to_180); + // mlt_log_info(gdata.filter, "in qxml_parse_file, filename: %s\n", filename); QFile gps_file(filename); @@ -1578,74 +1534,104 @@ int qxml_parse_file(gps_private_data gdata) if (filename.endsWith(".txt", Qt::CaseInsensitive)) { QTextStream qin(&gps_file); - if (q_parse_txt(qin, &gps_list_head, &count_pts) == 0) { + if (q_parse_txt(qin, gps_points_array, last_time, all_sorted, no_time_count, eq_time_count) == 0) { mlt_log_error(NULL, - "txt file (%s) does not start with a recognised format line, can't process!\n\ + "txt file (%s) processing failed. Make sure it starts with a recognised format line!\n\ \tsample format line: \n#delimiter=\" \" timestamp=\"yyyy/MM/dd\" timestamp=\"hh:mm:ss\" latitude longitude speed skip heading altitude", qUtf8Printable(filename)); return 0; } - } else { - QXmlStreamReader qxml_reader(&gps_file); - qxml_reader.setNamespaceProcessing(false); - while (!qxml_reader.atEnd() && !qxml_reader.hasError()) { - qxml_reader.readNextStartElement(); - if (qxml_reader.isStartDocument()) { - mlt_log_info(NULL, - "qxml_parse_file: StartDocument found. Encoding=%s\n", - qUtf8Printable(qxml_reader.documentEncoding().toString())); - continue; - } - // mlt_log_info(NULL, " readNext element tag: %s, type=%d\n", qUtf8Printable(qxml_reader.name().toString()), qxml_reader.tokenType()); - if (qxml_reader.name() == QStringLiteral("TrainingCenterDatabase")) { - qxml_parse_tcx(qxml_reader, &gps_list_head, &count_pts); - } else if (qxml_reader.name() == QStringLiteral("gpx")) { - qxml_parse_gpx(qxml_reader, &gps_list_head, &count_pts); - } else { - mlt_log_warning(gdata.filter, - "qxml_parse_file: fail to parse file: %s, unknown root element: " - "%s. Aborting.\n", - qUtf8Printable(filename), - qUtf8Printable(qxml_reader.name().toString())); - return 0; - } - } - if (qxml_reader.hasError()) { - mlt_log_info(NULL, - "qxml_reader.hasError! line:%u, errString:%s\n", - (unsigned) qxml_reader.lineNumber(), - qUtf8Printable(qxml_reader.errorString())); - return 0; + mlt_log_info(NULL, "printing the read array, size=%lld", gps_points_array.size()); + for (auto pt : gps_points_array) { + mlt_log_info(NULL, ".time:%lld, lat:%f, lon:%f, speed:%f, distance:%f, ele:%f, hr:%f, bearing:%f, cadence:%f, temp:%f, power:%f\n", + pt.time, pt.lat, pt.lon, pt.speed, pt.total_dist, pt.ele, pt.hr, pt.bearing, pt.cad, pt.atemp, pt.power); } - - qxml_reader.clear(); //not sure if really needed if not reused - } - - if (count_pts < 2) { + } + // else { + // QXmlStreamReader qxml_reader(&gps_file); + // qxml_reader.setNamespaceProcessing(false); + // while (!qxml_reader.atEnd() && !qxml_reader.hasError()) { + // qxml_reader.readNextStartElement(); + // if (qxml_reader.isStartDocument()) { + // mlt_log_info(NULL, + // "qxml_parse_file: StartDocument found. Encoding=%s\n", + // qUtf8Printable(qxml_reader.documentEncoding().toString())); + // continue; + // } + // // mlt_log_info(NULL, " readNext element tag: %s, type=%d\n", qUtf8Printable(qxml_reader.name().toString()), qxml_reader.tokenType()); + // if (qxml_reader.name() == QStringLiteral("TrainingCenterDatabase")) { + // qxml_parse_tcx(qxml_reader, &gps_list_head, &count_pts); + // } else if (qxml_reader.name() == QStringLiteral("gpx")) { + // qxml_parse_gpx(qxml_reader, &gps_list_head, &count_pts); + // } else { + // mlt_log_warning(gdata.filter, + // "qxml_parse_file: fail to parse file: %s, unknown root element: " + // "%s. Aborting.\n", + // qUtf8Printable(filename), + // qUtf8Printable(qxml_reader.name().toString())); + // return 0; + // } + // } + // if (qxml_reader.hasError()) { + // mlt_log_info(NULL, + // "qxml_reader.hasError! line:%u, errString:%s\n", + // (unsigned) qxml_reader.lineNumber(), + // qUtf8Printable(qxml_reader.errorString())); + // return 0; + // } + + // qxml_reader.clear(); //not sure if really needed if not reused + // } + + int gps_pt_count = gps_points_array.size(); + mlt_log_info(NULL, "back from parsing the file(s) '%s', gps_points_array.size()=%d, last_time=%lld, all_sorted=%d, no_time_count=%d, eq_time_count=%d", + qPrintable(filename), gps_pt_count, last_time, all_sorted, no_time_count, eq_time_count); + + if (gps_pt_count < 2) { mlt_log_warning(gdata.filter, - "qxml_parse_file: less than 2 gps points read (%d). Aborting.\n", - count_pts); + "qxml_parse_file: less than 2 gps points read (%lld). Aborting.\n", + gps_points_array.size()); return 0; } - *gdata.ptr_to_gps_points_r = (gps_point_raw *) calloc(count_pts, sizeof(gps_point_raw)); - gps_array = *gdata.ptr_to_gps_points_r; //just an alias + //sort if needed + if (!all_sorted) + std::sort(gps_points_array.begin(), gps_points_array.end()); + + //deal with entries without time + bool skip_notimes = false; + if (no_time_count > 0 && no_time_count < gps_pt_count) { + skip_notimes = true; + gps_pt_count -= no_time_count; + mlt_log_info (NULL, "qxml_parse_file: will skip %d/%lld points without valid time", no_time_count, gps_points_array.size()); + } + if (eq_time_count) { + gps_pt_count -= eq_time_count; + mlt_log_info (NULL, "qxml_parse_file: will skip %d/%lld points with duplicate time", eq_time_count, gps_points_array.size()); + } + + //allocate C style array + *gdata.ptr_to_gps_points_r = (gps_point_raw *) calloc(gps_pt_count, sizeof(gps_point_raw)); + gps_point_raw* gps_array = *gdata.ptr_to_gps_points_r; //just an alias if (gps_array == NULL) { mlt_log_error(gdata.filter, "malloc error (size=%u)\n", - (unsigned) (count_pts * sizeof(gps_point_raw))); + (unsigned) (gps_pt_count * sizeof(gps_point_raw))); } - *gdata.gps_points_size = count_pts; - - //copy points to private array and free list - while (gps_list_head) { - gps_array[nr++] = gps_list_head->gp; - tmp = gps_list_head; - gps_list_head = gps_list_head->next; - free(tmp); + *gdata.gps_points_size = gps_pt_count; + + //copy points to private array (skip pts if needed) + int nr = 0; + for (auto pt : gps_points_array) { + if (skip_notimes && pt.time <= no_time_count*1000) + continue; + if (eq_time_count > 0 && nr-1 >= 0 && pt.time == gps_points_array[nr-1].time) + continue; + gps_array[nr++] = pt; } //search to find if we cross the 180 and/or the 0 meridian + //TODO: move this logic inside the array copy above and don't just check crt to next, but [next which has valid .lon] bool crosses0 = false, crosses180 = false; for (int i = 0; i < *gdata.gps_points_size - 1; i++) { double crt = gps_array[i].lon; @@ -1673,25 +1659,25 @@ int qxml_parse_file(gps_private_data gdata) } else swap_to_180 = 0; - // //debug result: - // for (int i = 0; i < *gdata.gps_points_size; i += 100) { - // gps_point_raw *crt_point = &gps_array[i]; - // mlt_log_info(NULL, - // "_xml_parse_file read point[%d]: time:%d, lat:%f, lon:%f, speed:%f, " - // "distance:%f, ele:%f, hr:%f, bearing:%f, cadence:%f, temp:%f, power:%f\n", - // i, - // crt_point->time / 1000, - // crt_point->lat, - // crt_point->lon, - // crt_point->speed, - // crt_point->total_dist, - // crt_point->ele, - // crt_point->hr, - // crt_point->bearing, - // crt_point->cad, - // crt_point->atemp, - // crt_point->power); - //} + //debug result: + for (int i = 0; i < *gdata.gps_points_size; i += 100) { + gps_point_raw *crt_point = &gps_array[i]; + mlt_log_info(NULL, + "_xml_parse_file read point[%d]: time:%lld, lat:%f, lon:%f, speed:%f, " + "distance:%f, ele:%f, hr:%f, bearing:%f, cadence:%f, temp:%f, power:%f\n", + i, + crt_point->time, + crt_point->lat, + crt_point->lon, + crt_point->speed, + crt_point->total_dist, + crt_point->ele, + crt_point->hr, + crt_point->bearing, + crt_point->cad, + crt_point->atemp, + crt_point->power); + } return 1; } diff --git a/src/modules/qt/gps_parser.h b/src/modules/qt/gps_parser.h index f00805fba..d322a43d5 100644 --- a/src/modules/qt/gps_parser.h +++ b/src/modules/qt/gps_parser.h @@ -31,11 +31,15 @@ #define GPS_UNINIT -9999 #define MAX_PRINT_TIMESTAMP_NS 32 -typedef struct +struct gps_point_raw { double lat, lon, lat_projected, speed, total_dist, ele, hr, bearing, cad, atemp, power; int64_t time; //epoch milliseconds -} gps_point_raw; + bool operator<(const gps_point_raw& pt2) const + { + return time < pt2.time; + } +}; typedef struct gps_point_raw_list { @@ -43,13 +47,13 @@ typedef struct gps_point_raw_list struct gps_point_raw_list *next; } gps_point_ll; -typedef struct +struct gps_point_proc { double lat, lon, lat_projected, speed, speed_vertical, speed_3d, total_dist, ele, hr, bearing, cad, atemp, power; int64_t time; double d_elev, elev_up, elev_down, dist_up, dist_down, dist_flat, grade_p; -} gps_point_proc; +}; //0 is a valid value for many fields, use GPS_UNINIT (-9999) to differentiate missing values static const gps_point_raw uninit_gps_raw_point = {.lat = GPS_UNINIT, @@ -88,7 +92,7 @@ static const gps_point_proc uninit_gps_proc_point = {.lat = GPS_UNINIT, .grade_p = GPS_UNINIT}; //structure used to ease argument passing between filter and GPS parser api -typedef struct +struct gps_private_data { gps_point_raw *gps_points_r; //raw gps data from file gps_point_proc *gps_points_p; //processed gps data @@ -105,7 +109,7 @@ typedef struct int last_smooth_lvl; char *last_filename; //gps file fullpath mlt_filter filter; -} gps_private_data; +} ; #define has_valid_location(x) (((x).lat) != GPS_UNINIT && ((x).lon) != GPS_UNINIT) #define has_valid_location_ptr(x) ((x) && ((x)->lat) != GPS_UNINIT && ((x)->lon) != GPS_UNINIT) From 1caaa824eb20542c67066c9c13eb7b0112f344bd Mon Sep 17 00:00:00 2001 From: Daniel F Date: Fri, 2 Oct 2026 21:21:49 +0300 Subject: [PATCH 03/14] GPS parser: auto match date time location in txt via regexes --- src/modules/qt/filter_gpsgraphic.cpp | 4 +- src/modules/qt/filter_gpstext.cpp | 2 +- src/modules/qt/gps_parser.cpp | 505 ++++++++++++++++++++------- src/modules/qt/gps_parser.h | 15 +- 4 files changed, 385 insertions(+), 141 deletions(-) diff --git a/src/modules/qt/filter_gpsgraphic.cpp b/src/modules/qt/filter_gpsgraphic.cpp index b19f57aca..528790a7e 100644 --- a/src/modules/qt/filter_gpsgraphic.cpp +++ b/src/modules/qt/filter_gpsgraphic.cpp @@ -633,7 +633,7 @@ static void process_filter_properties(mlt_filter filter, mlt_frame frame) / 1000); } -//checks if a new file is selected and if so, does private_data cleanup and calls qxml_parse_file +//checks if a new file is selected and if so, does private_data cleanup and calls q_parse_file //+ computes some one time pdata values static void process_file(mlt_filter filter, mlt_frame frame) { @@ -652,7 +652,7 @@ static void process_file(mlt_filter filter, mlt_frame frame) strncpy(pdata->last_filename, filename, sizeof(pdata->last_filename) - 1); pdata->last_filename[sizeof(pdata->last_filename) - 1] = '\0'; - if (qxml_parse_file(filter_to_gps_data(filter)) == 1) { + if (q_parse_file(filter_to_gps_data(filter)) == 1) { get_first_gps_time(filter_to_gps_data(filter)); get_last_gps_time(filter_to_gps_data(filter)); diff --git a/src/modules/qt/filter_gpstext.cpp b/src/modules/qt/filter_gpstext.cpp index 0a281d57f..6565f6a00 100644 --- a/src/modules/qt/filter_gpstext.cpp +++ b/src/modules/qt/filter_gpstext.cpp @@ -616,7 +616,7 @@ static void process_file(mlt_filter filter, mlt_frame frame) strncpy(pdata->last_filename, filename, sizeof(pdata->last_filename) - 1); pdata->last_filename[sizeof(pdata->last_filename) - 1] = '\0'; - if (qxml_parse_file(filter_to_gps_data(filter)) == 1) { + if (q_parse_file(filter_to_gps_data(filter)) == 1) { get_first_gps_time(filter_to_gps_data(filter)); get_last_gps_time(filter_to_gps_data(filter)); diff --git a/src/modules/qt/gps_parser.cpp b/src/modules/qt/gps_parser.cpp index 9c6b01752..0d088e1d1 100644 --- a/src/modules/qt/gps_parser.cpp +++ b/src/modules/qt/gps_parser.cpp @@ -29,6 +29,38 @@ #define _x (const xmlChar *) #define _s (const char *) +QStringList accepted_distance_formats + = {"km", "kilometer", "mi", "mile", "nm", "nautical", "ft", "feet"}; + +QStringList accepted_speed_formats = {"km", + "km/h", + "kilo", + "ms", + "m/s", + "meter", + "mmin", + "m/min", + "ftmin", + "ft/min", + "mi", + "mi/h", + "mile", + "kn", + "nm/h", + "knots", + "ft", + "ft/s", + "feet"}; + +bool qstring_contains_any_qstringlist(const QString &main_string, const QStringList &list) +{ + for (const auto &s : list) { + if (main_string.contains(s, Qt::CaseInsensitive)) + return true; + } + return false; +} + // Maps are usually in ESPG3857 format, not in GPS format EPSG4326 / WG84. We have to project // the latitude, otherwise big maps have wrong aspect ratio double project_latitude(double lat) @@ -399,8 +431,7 @@ double convert_speed_to_format(double x, const char *format) if (x == GPS_UNINIT) return x; //order is important as short keywords will match anywhere (ms in kms and mi in mi[nutes]) - if (format == NULL || strstr(format, "kms") || strstr(format, "km/s") - || strstr(format, "kilometer")) + if (format == NULL || strstr(format, "km") || strstr(format, "km/h") || strstr(format, "kilo")) return x * 3.6; //default km/h if (strstr(format, "ms") || strstr(format, "m/s") || strstr(format, "meter")) return x; @@ -1345,7 +1376,7 @@ void qxml_parse_tcx(QXmlStreamReader &reader, gps_point_ll **gps_list, int *coun //tries to parse a .txt file containing GPS data; returns 0 for failure, 1 success //first line of the file must contain a format helper that starts with #delimiter= -int q_parse_txt(QTextStream &reader, QVector & gps_points_file, int64_t& last_time, bool& all_sorted, int& no_time_count, int&eq_time_count) +int q_parse_txt(QTextStream &reader, QVector &gps_points_file, int &no_time_count) { //example from my dashcam (added extra spaces for readability): //2025/06/12 02:45:01 N:45.123456 E:26.123456 0.0 km/h 0.00 72.50 10 x:-4.13 y:0.27 z:0.49 @@ -1358,23 +1389,28 @@ int q_parse_txt(QTextStream &reader, QVector & gps_points_file, i QString default_format_line = "#delimiter=\" \" timestamp=\"yyyy/MM/dd\" timestamp=\"hh:mm:ss\" latitude longitude " "speed skip heading altitude"; - - //read the format line, check for delimiter, use default if fails - QString delimiter; + //regex for yyyy*MM*dd*HH*mm*ss[*sss], any separators (non digits chars), miliseconds optional + QRegularExpression re_date_time( + R"((\d{4})\D?(\d{2})\D?(\d{2})\D?(\d{2})\D?(\d{2})\D?(\d{2})(?:\D?(\d{1,3}))?)"); + QRegularExpression re_date(R"((\d{4})\D?(\d{2})\D?(\d{2}))"); + QRegularExpression re_time(R"((\d{2})\D?(\d{2})\D?(\d{2})(?:\D?(\d{1,3}))?)"); + QRegularExpression re_lat(R"(([NS])[: ]?([+-]?\d{1,3}[\.,]\d{6}))"); + QRegularExpression re_lon(R"(([EW])[: ]?([+-]?\d{1,3}[\.,]\d{6}))"); + + QString format_line, first_keyword = "#delimiter=", delimiter; QStringList format_tokens, format_tokens_arguments; - if (!reader.atEnd()) { - QString format_line = reader.readLine(); - QString first_keyword = "#delimiter="; - if (!format_line.startsWith(first_keyword)) { - mlt_log_warning( - NULL, - "no format line found at the start of the file, using the default of: " - "'%s'\n (note: tokens that don't match or are extra at the end are skipped)", - qUtf8Printable(default_format_line)); - format_line = default_format_line; - reader.seek(0); - } - //read and store the content of delimiter=" " + bool regex_mode = false, format_date_time_separated = false; + + //if file doesn't start with "#delimiter=", will try regex for datetime and lat lon + if ((format_line = reader.readLine()).startsWith(first_keyword) == false) { + mlt_log_info(NULL, + "no format line found at the start of the file, trying a generic regex " + "for datetime and lat/lon"); + mlt_log_info(NULL, "Note: sample format line: '%s'", qUtf8Printable(default_format_line)); + reader.seek(0); + regex_mode = true; + } else if (!reader.atEnd()) { + //read and store the content of delimiter="" int firstQ = format_line.indexOf('"'); int secondQ = format_line.indexOf('"', firstQ + 1); if (firstQ != -1 && secondQ != -1) { @@ -1382,92 +1418,243 @@ int q_parse_txt(QTextStream &reader, QVector & gps_points_file, i if (!delimiter.isEmpty()) { QString after_delimiter = format_line.mid(secondQ + 1); format_tokens = after_delimiter.split(delimiter, Qt::SkipEmptyParts); - } else + } else { + mlt_log_warning(NULL, + "empty delimiter string, make sure to enclose it within double " + "quotes. Aborting file."); return 0; - } else + } + } else { + mlt_log_warning(NULL, + "bad delimiter format, make sure to enclose a character or string " + "within double quotes. Aborting file."); return 0; + } // mlt_log_debug(NULL, // "q_parse_txt(): format_line='%s', delimiter='%s', format_tokens found=%lld", // qUtf8Printable(format_line), // qUtf8Printable(delimiter), // format_tokens.size()); - } - //extract timestamp with format - format_tokens_arguments.resize(format_tokens.size()); - bool date_time_separated = false; - for (int i = 0; i < format_tokens.size(); i++) { - if (format_tokens[i].contains("timestamp=", Qt::CaseInsensitive)) { - int firstQ = format_tokens[i].indexOf('"'); - int secondQ = format_tokens[i].indexOf('"', firstQ + 1); - format_tokens_arguments[i] = format_tokens[i].mid(firstQ + 1, secondQ - (firstQ + 1)); - if (!format_tokens_arguments[i].contains('y', Qt::CaseInsensitive)) - date_time_separated = true; + //extract timestamp with format + format_tokens_arguments.resize(format_tokens.size()); + for (int i = 0; i < format_tokens.size(); i++) { + if (format_tokens[i].contains("timestamp=", Qt::CaseInsensitive)) { + int firstQ = format_tokens[i].indexOf('"'); + int secondQ = format_tokens[i].indexOf('"', firstQ + 1); + format_tokens_arguments[i] = format_tokens[i].mid(firstQ + 1, + secondQ - (firstQ + 1)); + if (!format_tokens_arguments[i].contains('y', Qt::CaseInsensitive)) + format_date_time_separated = true; + } } + } else { //already end of file? this is an invalid file + mlt_log_warning( + NULL, + "file ended too soon, please make sure the GPS points are stored on individual lines."); + return 0; } - //for each line: for each token in the format_line tokens process the coresponding token from file + //process the file line by line + int nr_invalid_lines = 0; + QVector warned_of_format(format_tokens.size(), false); while (!reader.atEnd()) { - QString line = reader.readLine(); - QStringList tokens = line.split(delimiter, Qt::SkipEmptyParts); + QString line = reader.readLine(256); gps_point_raw crt_point = uninit_gps_raw_point; - QString datetime_assembled, datetime_format_assembled; - - for (int i = 0; i < format_tokens.size() && i < tokens.size(); i++) { - /* mlt_log_info(NULL, - "format[%d]: %s, token[%d]; %s", - i, - qUtf8Printable(format_tokens[i]), - i, - tokens[i].toUtf8().constData()); */ - if (format_tokens[i].contains("timestamp=", Qt::CaseInsensitive)) { - datetime_assembled.append(tokens[i]); - datetime_assembled.append(delimiter); - datetime_format_assembled.append(format_tokens_arguments[i]); - datetime_format_assembled.append(delimiter); - } else if (format_tokens[i].contains("timestamp", Qt::CaseInsensitive)) { - crt_point.time = datetimeXMLstring_to_mseconds(qUtf8Printable(tokens[i])); - } else if (format_tokens[i].contains("lat", Qt::CaseInsensitive)) { - double negative = (tokens[i].startsWith('s', Qt::CaseInsensitive) ? -1 : 1); - //skip to the number - int number_at = tokens[i].indexOf(QRegularExpression("[-+0-9]")); - crt_point.lat = tokens[i].mid(number_at).toDouble() * negative; + + //a point is valid if it has either (a valid time + at least one other field) or (a valid lat/lon pair) + bool valid_datetime = false, got_other = false; + int got_latlon = 0; + + if (regex_mode) { //using regex to find a datetime and/or a lat+lon in this line + //datetime + auto m = re_date_time.match(line); + if (m.hasMatch()) { + int year = m.captured(1).toInt(); + int month = m.captured(2).toInt(); + int day = m.captured(3).toInt(); + int hour = m.captured(4).toInt(); + int minute = m.captured(5).toInt(); + int second = m.captured(6).toInt(); + int msec = m.captured(7).isEmpty() ? 0 + : m.captured(7).leftJustified(3, '0').toInt(); + QDateTime dt(QDate(year, month, day), QTime(hour, minute, second, msec), Qt::UTC); + if (dt.isValid()) { + crt_point.time = dt.toMSecsSinceEpoch(); + valid_datetime = (crt_point.time != 0); + } + } + //latitude + auto m2 = re_lat.match(line); + if (m2.hasMatch()) { + QString direction = m2.captured(1); + crt_point.lat = m2.captured(2).toDouble(); + if (direction == "S") + crt_point.lat *= -1; + if (is_valid_latitude(crt_point.lat)) { + got_latlon++; + } crt_point.lat_projected = project_latitude(crt_point.lat); - } else if (format_tokens[i].contains("lon", Qt::CaseInsensitive)) { - double negative = (tokens[i].startsWith('w', Qt::CaseInsensitive) ? -1 : 1); - int number_at = tokens[i].indexOf(QRegularExpression("[-+0-9]")); - crt_point.lon = tokens[i].mid(number_at).toDouble() * negative; - } else if (format_tokens[i].contains("speed", Qt::CaseInsensitive)) { - crt_point.speed = tokens[i].toDouble(); - if (i + 1 < format_tokens.size()) { - if (tokens[i + 1].contains("km", Qt::CaseInsensitive)) - crt_point.speed /= 3.6; - else if (tokens[i].contains("mi", Qt::CaseInsensitive) - || tokens[i].contains("m/h", Qt::CaseInsensitive)) - crt_point.speed /= 2.23693629; + } + //longitude + auto m3 = re_lon.match(line, m2.capturedEnd()); + if (m3.hasMatch()) { + QString direction = m3.captured(1); + crt_point.lon = m3.captured(2).toDouble(); + if (direction == "W") + crt_point.lon *= -1; + if (is_valid_longitude(crt_point.lon)) { + got_latlon++; + } + } + } else { //format provided by the file + QStringList line_tokens = line.split(delimiter, Qt::SkipEmptyParts); + QString datetime_assembled, datetime_format_assembled; + bool got_qdate = false, got_qtime = false; + QDate qdate_part; + QTime qtime_part; + + for (int i = 0; i < format_tokens.size() && i < line_tokens.size(); i++) { + // mlt_log_info(NULL, + // "format[%d]: %s, token[%d]; %s", + // i, + // qUtf8Printable(format_tokens[i]), + // i, + // line_tokens[i].toUtf8().constData()); + if (format_tokens[i].contains("datetime=", Qt::CaseInsensitive)) { + datetime_assembled.append(line_tokens[i]); + datetime_assembled.append(delimiter); + datetime_format_assembled.append(format_tokens_arguments[i]); + datetime_format_assembled.append(delimiter); + } else if (format_tokens[i].contains("datetimeiso", Qt::CaseInsensitive)) { + crt_point.time = datetimeXMLstring_to_mseconds(qUtf8Printable(line_tokens[i])); + valid_datetime = (crt_point.time != 0); + } else if (format_tokens[i].contains("datetime", Qt::CaseInsensitive)) { + auto m = re_date_time.match(line_tokens[i]); + if (m.hasMatch()) { + int year = m.captured(1).toInt(); + int month = m.captured(2).toInt(); + int day = m.captured(3).toInt(); + int hour = m.captured(4).toInt(); + int minute = m.captured(5).toInt(); + int second = m.captured(6).toInt(); + int msec = m.captured(7).isEmpty() + ? 0 + : m.captured(7).leftJustified(3, '0').toInt(); + QDateTime dt(QDate(year, month, day), + QTime(hour, minute, second, msec), + Qt::UTC); + if (dt.isValid()) { + crt_point.time = dt.toMSecsSinceEpoch(); + valid_datetime = (crt_point.time != 0); + } + } + } else if (format_tokens[i].contains("date", Qt::CaseInsensitive)) { + auto m = re_date.match(line_tokens[i]); + if (m.hasMatch()) { + int year = m.captured(1).toInt(); + int month = m.captured(2).toInt(); + int day = m.captured(3).toInt(); + qdate_part = QDate(year, month, day); + if (qdate_part.isValid()) + got_qdate = true; + } + } else if (format_tokens[i].contains("time", Qt::CaseInsensitive)) { + auto m = re_time.match(line_tokens[i]); + if (m.hasMatch()) { + int hour = m.captured(1).toInt(); + int minute = m.captured(2).toInt(); + int second = m.captured(3).toInt(); + int msec = m.captured(4).isEmpty() + ? 0 + : m.captured(4).leftJustified(3, '0').toInt(); + qtime_part = QTime(hour, minute, second, msec); + if (qtime_part.isValid()) + got_qtime = true; + } + } else if (format_tokens[i].contains("lat", Qt::CaseInsensitive)) { + double negative = (line_tokens[i].startsWith('s', Qt::CaseInsensitive) ? -1 : 1); + int number_at = line_tokens[i].indexOf(QRegularExpression("[-+0-9]")); + crt_point.lat = line_tokens[i].mid(number_at).toDouble() * negative; + crt_point.lat_projected = project_latitude(crt_point.lat); + if (is_valid_latitude(crt_point.lat)) + got_latlon++; + } else if (format_tokens[i].contains("lon", Qt::CaseInsensitive)) { + double negative = (line_tokens[i].startsWith('w', Qt::CaseInsensitive) ? -1 : 1); + int number_at = line_tokens[i].indexOf(QRegularExpression("[-+0-9]")); + crt_point.lon = line_tokens[i].mid(number_at).toDouble() * negative; + if (is_valid_longitude(crt_point.lon)) + got_latlon++; + } else if (format_tokens[i].contains("speed", Qt::CaseInsensitive)) { + crt_point.speed = line_tokens[i].toDouble(); + mlt_log_info(NULL, "speed: token='%s', token.toDouble=%f", qPrintable(line_tokens[i]), crt_point.speed); + if (i + 1 < line_tokens.size()) { + double ratio = 1; + if (qstring_contains_any_qstringlist(line_tokens[i + 1], accepted_speed_formats)) + ratio = convert_speed_to_format(1, qPrintable(line_tokens[i + 1])); + crt_point.speed /= ratio; + } + got_other = true; + } else if (format_tokens[i].contains("alt", Qt::CaseInsensitive) + || format_tokens[i].contains("ele", Qt::CaseInsensitive)) { + crt_point.ele = line_tokens[i].toDouble(); + if (i + 1 < line_tokens.size()) { + double ratio = 1; + if (qstring_contains_any_qstringlist(line_tokens[i + 1], + accepted_distance_formats)) + ratio = convert_distance_to_format(1, qPrintable(line_tokens[i + 1])); + crt_point.speed /= ratio; + } + got_other = true; + } else if (format_tokens[i].contains("head", Qt::CaseInsensitive) + || format_tokens[i].contains("bear", Qt::CaseInsensitive)) { + crt_point.bearing = line_tokens[i].toDouble(); + got_other = true; + } else if (format_tokens[i].contains("hr", Qt::CaseInsensitive) + || format_tokens[i].contains("heart", Qt::CaseInsensitive)) { + crt_point.hr = line_tokens[i].toDouble(); + got_other = true; + } else if (format_tokens[i].contains("cad", Qt::CaseInsensitive)) { + crt_point.cad = line_tokens[i].toDouble(); + got_other = true; + } else if (format_tokens[i].contains("temp", Qt::CaseInsensitive)) { + crt_point.atemp = line_tokens[i].toDouble(); + got_other = true; + } else if (format_tokens[i].contains("pow", Qt::CaseInsensitive)) { + crt_point.power = line_tokens[i].toDouble(); + got_other = true; + } else { + if (!warned_of_format[i]) { + mlt_log_info(NULL, + "unknown format string[%d]: '%s', skipping", + i, + qPrintable(format_tokens[i])); + warned_of_format[i] = true; + } } - } else if (format_tokens[i].contains("alt", Qt::CaseInsensitive) - || format_tokens[i].contains("ele", Qt::CaseInsensitive)) { - crt_point.ele = tokens[i].toDouble(); - } else if (format_tokens[i].contains("head", Qt::CaseInsensitive) - || format_tokens[i].contains("bear", Qt::CaseInsensitive)) { - crt_point.bearing = tokens[i].toDouble(); } - } - //process datetime now after the entire line is parsed: - if (date_time_separated) { - crt_point.time = datetimeXMLstring_to_mseconds(qUtf8Printable(datetime_assembled), - qUtf8Printable( - datetime_format_assembled)); - // mlt_log_info(NULL, - // "date_time_separated true, assembled input:'%s', assembled format:'%s', " - // "result=%lld", - // qUtf8Printable(datetime_assembled), - // qUtf8Printable(datetime_format_assembled), - // crt_point.time); - } + //process datetime now after the entire line is parsed: + if (got_qdate && got_qtime) { + QDateTime dt(qdate_part, qtime_part); + if (dt.isValid()) { + crt_point.time = dt.toMSecsSinceEpoch(); + valid_datetime = (crt_point.time != 0); + } + } else if (format_date_time_separated) { + crt_point.time = datetimeXMLstring_to_mseconds(qUtf8Printable(datetime_assembled), + qUtf8Printable( + datetime_format_assembled)); + valid_datetime = (crt_point.time != 0); + // mlt_log_info(NULL, + // "format_date_time_separated true, assembled input:'%s', assembled format:'%s', " + // "result=%lld", + // qUtf8Printable(datetime_assembled), + // qUtf8Printable(datetime_format_assembled), + // crt_point.time); + } + } //end regex_mode / format_mode // mlt_log_info(NULL, // "crt_point parsed txt: time:%lld, lat:%f, lon:%f, speed:%f, " @@ -1484,14 +1671,27 @@ int q_parse_txt(QTextStream &reader, QVector & gps_points_file, i // crt_point.atemp, // crt_point.power); + bool valid_line = (valid_datetime && got_other) || (got_latlon == 2); + if (!valid_line) { + mlt_log_info(NULL, "q_parse_txt: skipping invalid line: '%s'\n", qUtf8Printable(line)); + if (nr_invalid_lines++ >= 10) { + mlt_log_warning(NULL, "q_parse_txt: 10 invalid lines in a row, aborting file\n"); + return 0; + } + continue; + } else { + nr_invalid_lines = 0; + } + //we parsed a line, check the time and store it in the array - if (crt_point.time != GPS_UNINIT && crt_point.time != 0) { - if (crt_point.time < last_time) - all_sorted = false; - else if (crt_point.time == last_time) - eq_time_count +=1; - last_time = crt_point.time; - } else if (crt_point.time == GPS_UNINIT || crt_point.time == 0) { + // if (crt_point.time != GPS_UNINIT && crt_point.time != 0) { + // if (crt_point.time < last_time) + // all_sorted = false; + // // else if (crt_point.time == last_time) + // // eq_time_count += 1; //TODO: if multiple file support this won't be enough + // last_time = crt_point.time; + // } else + if (crt_point.time == GPS_UNINIT || crt_point.time == 0) { //if no time is read (or bad format), we make it up starting from 1 no_time_count += 1; crt_point.time = no_time_count * 1000; @@ -1507,13 +1707,10 @@ int q_parse_txt(QTextStream &reader, QVector & gps_points_file, i * swap_to_180 -> shifts all longitudes around with 180 becoming 0, only use it if user requests it * Returns 0 on failure, 1 on success. */ -int qxml_parse_file(gps_private_data gdata) +int q_parse_file(gps_private_data gdata) { - QVector gps_points_array; - int64_t last_time = -1; //used to check if points are read already ordered - int no_time_count = 0; //for gps tracks without time, used for ordering - int eq_time_count = 0; //we only support unique times - bool all_sorted = true; //skip the sorting at the end + QVector gps_points_array; + int no_time_count = 0; //for gps tracks without time, used for ordering // gps_point_ll *gps_list_head = NULL, *tmp = NULL; // int count_pts = 0, nr = 0; @@ -1521,32 +1718,47 @@ int qxml_parse_file(gps_private_data gdata) // gps_point_raw *gps_array = NULL; int &swap_to_180 = *gdata.swap180; - // mlt_log_info(gdata.filter, "in qxml_parse_file, filename: %s\n", filename); + // mlt_log_info(gdata.filter, "in q_parse_file, filename: %s\n", filename); QFile gps_file(filename); if (!gps_file.open(QFile::ReadOnly | QFile::Text)) { mlt_log_warning(gdata.filter, - "qxml_parse_file couldn't read file: %s\n", + "q_parse_file couldn't read file: %s\n", qUtf8Printable(filename)); return 0; } if (filename.endsWith(".txt", Qt::CaseInsensitive)) { QTextStream qin(&gps_file); - if (q_parse_txt(qin, gps_points_array, last_time, all_sorted, no_time_count, eq_time_count) == 0) { + if (q_parse_txt(qin, gps_points_array, no_time_count) == 0) { mlt_log_error(NULL, - "txt file (%s) processing failed. Make sure it starts with a recognised format line!\n\ - \tsample format line: \n#delimiter=\" \" timestamp=\"yyyy/MM/dd\" timestamp=\"hh:mm:ss\" latitude longitude speed skip heading altitude", + "txt file (%s) processing failed. Make sure it starts with a recognised " + "format line!", qUtf8Printable(filename)); + mlt_log_info(NULL, + "Sample format line: \n#delimiter=\" \" timestamp=\"yyyy/MM/dd\" " + "timestamp=\"hh:mm:ss\" latitude longitude speed skip heading altitude"); return 0; } mlt_log_info(NULL, "printing the read array, size=%lld", gps_points_array.size()); - for (auto pt : gps_points_array) { - mlt_log_info(NULL, ".time:%lld, lat:%f, lon:%f, speed:%f, distance:%f, ele:%f, hr:%f, bearing:%f, cadence:%f, temp:%f, power:%f\n", - pt.time, pt.lat, pt.lon, pt.speed, pt.total_dist, pt.ele, pt.hr, pt.bearing, pt.cad, pt.atemp, pt.power); + for (auto &pt : gps_points_array) { + mlt_log_info(NULL, + ".time:%lld, lat:%f, lon:%f, speed:%f, distance:%f, ele:%f, hr:%f, " + "bearing:%f, cadence:%f, temp:%f, power:%f\n", + pt.time, + pt.lat, + pt.lon, + pt.speed, + pt.total_dist, + pt.ele, + pt.hr, + pt.bearing, + pt.cad, + pt.atemp, + pt.power); } - } + } // else { // QXmlStreamReader qxml_reader(&gps_file); // qxml_reader.setNamespaceProcessing(false); @@ -1554,7 +1766,7 @@ int qxml_parse_file(gps_private_data gdata) // qxml_reader.readNextStartElement(); // if (qxml_reader.isStartDocument()) { // mlt_log_info(NULL, - // "qxml_parse_file: StartDocument found. Encoding=%s\n", + // "q_parse_file: StartDocument found. Encoding=%s\n", // qUtf8Printable(qxml_reader.documentEncoding().toString())); // continue; // } @@ -1565,7 +1777,7 @@ int qxml_parse_file(gps_private_data gdata) // qxml_parse_gpx(qxml_reader, &gps_list_head, &count_pts); // } else { // mlt_log_warning(gdata.filter, - // "qxml_parse_file: fail to parse file: %s, unknown root element: " + // "q_parse_file: fail to parse file: %s, unknown root element: " // "%s. Aborting.\n", // qUtf8Printable(filename), // qUtf8Printable(qxml_reader.name().toString())); @@ -1583,52 +1795,81 @@ int qxml_parse_file(gps_private_data gdata) // qxml_reader.clear(); //not sure if really needed if not reused // } - int gps_pt_count = gps_points_array.size(); - mlt_log_info(NULL, "back from parsing the file(s) '%s', gps_points_array.size()=%d, last_time=%lld, all_sorted=%d, no_time_count=%d, eq_time_count=%d", - qPrintable(filename), gps_pt_count, last_time, all_sorted, no_time_count, eq_time_count); + const int gps_pt_count = gps_points_array.size(); + mlt_log_info(NULL, + "back from parsing the file(s) '%s', gps_points_array.size()=%d, " + "no_time_count=%d", + qPrintable(filename), + gps_pt_count, + no_time_count); if (gps_pt_count < 2) { mlt_log_warning(gdata.filter, - "qxml_parse_file: less than 2 gps points read (%lld). Aborting.\n", - gps_points_array.size()); + "q_parse_file: less than 2 gps points read (%d). Aborting.\n", + gps_pt_count); return 0; } - //sort if needed - if (!all_sorted) - std::sort(gps_points_array.begin(), gps_points_array.end()); + //make sure points are in chronological order + std::sort(gps_points_array.begin(), gps_points_array.end()); - //deal with entries without time + //if there are entries with and without time, drop the ones without - unless all of them bool skip_notimes = false; if (no_time_count > 0 && no_time_count < gps_pt_count) { skip_notimes = true; - gps_pt_count -= no_time_count; - mlt_log_info (NULL, "qxml_parse_file: will skip %d/%lld points without valid time", no_time_count, gps_points_array.size()); - } - if (eq_time_count) { - gps_pt_count -= eq_time_count; - mlt_log_info (NULL, "qxml_parse_file: will skip %d/%lld points with duplicate time", eq_time_count, gps_points_array.size()); + // gps_pt_count -= no_time_count; + mlt_log_info(NULL, + "q_parse_file: will skip %d/%d points without valid time", + no_time_count, + gps_pt_count); } //allocate C style array *gdata.ptr_to_gps_points_r = (gps_point_raw *) calloc(gps_pt_count, sizeof(gps_point_raw)); - gps_point_raw* gps_array = *gdata.ptr_to_gps_points_r; //just an alias + gps_point_raw *&gps_array = *gdata.ptr_to_gps_points_r; //alias by reference for realloc if (gps_array == NULL) { mlt_log_error(gdata.filter, - "malloc error (size=%u)\n", + "malloc error (size=%u), out of memory?\n", (unsigned) (gps_pt_count * sizeof(gps_point_raw))); + return 0; } *gdata.gps_points_size = gps_pt_count; - //copy points to private array (skip pts if needed) + //copy points to private array (and skip pts if needed) int nr = 0; - for (auto pt : gps_points_array) { - if (skip_notimes && pt.time <= no_time_count*1000) + for (auto &pt : gps_points_array) { + if (skip_notimes && pt.time <= no_time_count * 1000) continue; - if (eq_time_count > 0 && nr-1 >= 0 && pt.time == gps_points_array[nr-1].time) + if (nr - 1 >= 0 && pt.time == gps_array[nr - 1].time) continue; gps_array[nr++] = pt; } + if (nr >= 2) { + gps_point_raw *tmp = (gps_point_raw *) realloc(gps_array, nr * sizeof(gps_point_raw)); + if (tmp != NULL) { + gps_array = tmp; + *gdata.gps_points_size = nr; + } else { + mlt_log_error(gdata.filter, + "realloc error (size=%u), out of memory?\n", + (unsigned) (nr * sizeof(gps_point_raw))); + free(gps_array); + gps_array = NULL; + *gdata.gps_points_size = 0; + return 0; + } + } else { + mlt_log_warning(gdata.filter, + "q_parse_file: not enough (<2) valid GPS points after processing " + "(initially: %d). Aborting.\n", + gps_pt_count); + return 0; + } + + mlt_log_warning(NULL, + "*gdata.ptr_to_gps_points_r=%p, gps_array=%p", + *gdata.ptr_to_gps_points_r, + gps_array); //search to find if we cross the 180 and/or the 0 meridian //TODO: move this logic inside the array copy above and don't just check crt to next, but [next which has valid .lon] diff --git a/src/modules/qt/gps_parser.h b/src/modules/qt/gps_parser.h index d322a43d5..cfa472282 100644 --- a/src/modules/qt/gps_parser.h +++ b/src/modules/qt/gps_parser.h @@ -35,10 +35,7 @@ struct gps_point_raw { double lat, lon, lat_projected, speed, total_dist, ele, hr, bearing, cad, atemp, power; int64_t time; //epoch milliseconds - bool operator<(const gps_point_raw& pt2) const - { - return time < pt2.time; - } + bool operator<(const gps_point_raw &pt2) const { return time < pt2.time; } }; typedef struct gps_point_raw_list @@ -109,10 +106,12 @@ struct gps_private_data int last_smooth_lvl; char *last_filename; //gps file fullpath mlt_filter filter; -} ; +}; #define has_valid_location(x) (((x).lat) != GPS_UNINIT && ((x).lon) != GPS_UNINIT) #define has_valid_location_ptr(x) ((x) && ((x)->lat) != GPS_UNINIT && ((x)->lon) != GPS_UNINIT) +#define is_valid_latitude(lat) ((lat) >= -90 && (lat) <= 90) +#define is_valid_longitude(lon) ((lon) >= -180 && (lon) <= 180) #define MATH_PI 3.14159265358979323846 #define to_rad(x) ((x) *MATH_PI / 180.0) @@ -137,6 +136,10 @@ void process_gps_smoothing(gps_private_data gdata, char do_processing); double project_latitude(double lat); double unproject_latitude(double lat_projected); +extern QStringList accepted_distance_formats; +extern QStringList accepted_speed_formats; +bool qstring_contains_any_qstringlist(const QString &main_string, const QStringList &list); + double weighted_middle_double( double v1, int64_t t1, double v2, int64_t t2, int64_t new_t, int max_gps_diff_ms); int64_t weighted_middle_int64( @@ -148,7 +151,7 @@ gps_point_proc weighted_middle_point_proc(gps_point_proc *p1, void qxml_parse_gpx(QXmlStreamReader &reader, gps_point_ll **gps_list, int *count_pts); void qxml_parse_tcx(QXmlStreamReader &reader, gps_point_ll **gps_list, int *count_pts); -int qxml_parse_file(gps_private_data gdata); +int q_parse_file(gps_private_data gdata); #define swap_180_if_needed(lon) (pdata->swap_180 ? get_180_swapped(lon) : lon) double get_180_swapped(double lon); From cd0d259364de24eb2811e76a80c3d20d999dcee3 Mon Sep 17 00:00:00 2001 From: Daniel F Date: Fri, 2 Oct 2026 22:20:28 +0300 Subject: [PATCH 04/14] GPS parser: migrate gpx and tcx to QVector --- src/modules/qt/gps_parser.cpp | 400 +++++++++++++++------------------- 1 file changed, 173 insertions(+), 227 deletions(-) diff --git a/src/modules/qt/gps_parser.cpp b/src/modules/qt/gps_parser.cpp index 0d088e1d1..7e10c8dd1 100644 --- a/src/modules/qt/gps_parser.cpp +++ b/src/modules/qt/gps_parser.cpp @@ -1093,12 +1093,10 @@ void process_gps_smoothing(gps_private_data gdata, char do_processing) //File parsing with QT's XML stream reader // Parses a .gpx file into a gps_point_raw linked list -void qxml_parse_gpx(QXmlStreamReader &reader, gps_point_ll **gps_list, int *count_pts) +void qxml_parse_gpx(QXmlStreamReader &reader, + QVector &gps_points_file, + int &no_time_count) { - int64_t last_time = -1; - //support no time in file because many services (Strava included) do this for exported routes - gps_point_ll *no_time_head = NULL, *no_time_prev = NULL; - int no_time_count = 0; /* // sample point from .GPX file (version 1.0) + TrackPointExtension @@ -1116,19 +1114,27 @@ void qxml_parse_gpx(QXmlStreamReader &reader, gps_point_ll **gps_list, int *coun */ + + int nr_pt = 0; while (!reader.atEnd() && !reader.hasError()) { reader.readNext(); // mlt_log_info(NULL, " readNext tag: %s type = %d\n", qUtf8Printable(reader.name()), reader.tokenType()); if (reader.isStartElement() && reader.name() == QStringLiteral("trkpt")) { gps_point_raw crt_point = uninit_gps_raw_point; + bool valid_datetime = false, got_other = false; + int got_latlon = 0; + nr_pt++; QXmlStreamAttributes attributes = reader.attributes(); if (attributes.hasAttribute("lat")) { crt_point.lat = attributes.value("lat").toDouble(); crt_point.lat_projected = project_latitude(crt_point.lat); + got_latlon++; } - if (attributes.hasAttribute("lon")) + if (attributes.hasAttribute("lon")) { crt_point.lon = attributes.value("lon").toDouble(); + got_latlon++; + } while (reader.readNext() && !(reader.name() == QStringLiteral("trkpt") @@ -1138,106 +1144,72 @@ void qxml_parse_gpx(QXmlStreamReader &reader, gps_point_ll **gps_list, int *coun if (!reader.isStartElement()) continue; - if (reader.name() == QStringLiteral("ele")) + if (reader.name() == QStringLiteral("ele")) { crt_point.ele = reader.readElementText().toDouble(); - else if (reader.name() == QStringLiteral("time")) + got_other = true; + } else if (reader.name() == QStringLiteral("time")) { crt_point.time = datetimeXMLstring_to_mseconds( qUtf8Printable(reader.readElementText())); - else if (reader.name() == QStringLiteral("speed")) + valid_datetime = (crt_point.time != 0); + } else if (reader.name() == QStringLiteral("speed")) { crt_point.speed = reader.readElementText().toDouble(); - else if (reader.name() == QStringLiteral("course")) + got_other = true; + } else if (reader.name() == QStringLiteral("course")) { crt_point.bearing = reader.readElementText().toDouble(); - else if (reader.name() == QStringLiteral("extensions")) { + got_other = true; + } else if (reader.name() == QStringLiteral("extensions")) { while (reader.readNext() && !(reader.name() == QStringLiteral("extensions") && reader.tokenType() == QXmlStreamReader::EndElement)) { - if (reader.name() == QStringLiteral("power")) + if (reader.name() == QStringLiteral("power")) { crt_point.power = reader.readElementText().toDouble(); - else if (reader.name() == QStringLiteral("TrackPointExtension")) { + got_other = true; + } else if (reader.name() == QStringLiteral("TrackPointExtension")) { while (reader.readNext() && !(reader.name() == QStringLiteral("TrackPointExtension") && reader.tokenType() == QXmlStreamReader::EndElement)) { - if (reader.name() == QStringLiteral("hr")) + if (reader.name() == QStringLiteral("hr")) { crt_point.hr = reader.readElementText().toDouble(); - else if (reader.name() == QStringLiteral("cad")) + got_other = true; + } else if (reader.name() == QStringLiteral("cad")) { crt_point.cad = reader.readElementText().toDouble(); - else if (reader.name() == QStringLiteral("atemp")) + got_other = true; + } else if (reader.name() == QStringLiteral("atemp")) { crt_point.atemp = reader.readElementText().toDouble(); + got_other = true; + } } } } } } - //now add the point to linked list (but only if increasing time) - if (crt_point.time != GPS_UNINIT && crt_point.time > last_time) { - *gps_list = (gps_point_ll *) calloc(1, sizeof(gps_point_ll)); - if (*gps_list == NULL) - return; - - *count_pts += 1; - (*gps_list)->gp = crt_point; - (*gps_list)->next = NULL; - gps_list = &(*gps_list)->next; - last_time = crt_point.time; - } else if (crt_point.time == GPS_UNINIT) { - //if no time is stored, we make it up starting from 1 - gps_point_ll *tmp = (gps_point_ll *) calloc(1, sizeof(gps_point_ll)); - if (tmp == NULL) - return; - if (no_time_head == NULL) { - no_time_head = tmp; - no_time_prev = tmp; - } else { - no_time_prev->next = tmp; - no_time_prev = tmp; - } + + bool valid_pt = (valid_datetime && got_other) || (got_latlon == 2); + if (!valid_pt) { + mlt_log_info( + NULL, + "qxlm_parse_gpx: skipping read point[%d], minimum condition: (datetime + " + "something) or (location)", + nr_pt); + continue; + } + + //store the point in the array, set synthetic time, we'll deal with invalid times after sorting + if (crt_point.time == GPS_UNINIT || crt_point.time == 0) { + //if no time is read (or bad format), we make it up starting from 1 no_time_count += 1; crt_point.time = no_time_count * 1000; - tmp->gp = crt_point; - tmp->next = NULL; - } else { - mlt_log_info(NULL, - "qxml_parse_gpx: skipping point due to time [%d] %f,%f - crt:%u.%u, " - "last:%u.%u\n", - *count_pts, - crt_point.lat, - crt_point.lon, - (unsigned) (crt_point.time / 1000), - (unsigned) (crt_point.time % 1000), - (unsigned) (last_time / 1000), - (unsigned) (last_time % 1000)); } - } - } - //if no points had time, we replace the real list with the no_time one - if (*count_pts == 0 && no_time_count > 0) { - mlt_log_info(NULL, - "qxml_parse_gpx: all GPS points are missing time values, inserting each " - "point at 1 second interval!\n"); - *gps_list = no_time_head; - *count_pts = no_time_count; - } else if (*count_pts > 0 && no_time_count > 0) { - mlt_log_info(NULL, - "qxml_parse_gpx: %d GPS points have no time in original file and have been " - "discarded (total valid points kept: %d)!\n", - no_time_count, - *count_pts); - //cleanup the no_time list as we're not using it - while (no_time_head) { - gps_point_ll *tmp = no_time_head; - no_time_head = no_time_head->next; - free(tmp); + gps_points_file.append(crt_point); } } } // Parses a .tcx file into a gps_point_raw linked list -void qxml_parse_tcx(QXmlStreamReader &reader, gps_point_ll **gps_list, int *count_pts) +void qxml_parse_tcx(QXmlStreamReader &reader, + QVector &gps_points_file, + int &no_time_count) { - int64_t last_time = -1; - //support no time in file because many services (Strava included) do this for exported routes - gps_point_ll *no_time_head = NULL, *no_time_prev = NULL; - int no_time_count = 0; /* //sample point from .TCX file @@ -1261,11 +1233,17 @@ void qxml_parse_tcx(QXmlStreamReader &reader, gps_point_ll **gps_list, int *coun */ + + int nr_pt = 0; while (!reader.atEnd() && !reader.hasError()) { reader.readNext(); // mlt_log_info(NULL, " readNextStartElement tag: %s\n", qUtf8Printable(reader.name())); if (reader.isStartElement() && reader.name() == QStringLiteral("Trackpoint")) { gps_point_raw crt_point = uninit_gps_raw_point; + bool valid_datetime = false, got_other = false; + int got_latlon = 0; + nr_pt++; + while ( reader.readNext() && !(reader.name() == QStringLiteral("Trackpoint") @@ -1276,100 +1254,73 @@ void qxml_parse_tcx(QXmlStreamReader &reader, gps_point_ll **gps_list, int *coun if (!reader.isStartElement()) continue; - if (reader.name() == QStringLiteral("Time")) + if (reader.name() == QStringLiteral("Time")) { crt_point.time = datetimeXMLstring_to_mseconds( qUtf8Printable(reader.readElementText())); - else if (reader.name() == QStringLiteral("Position")) { + valid_datetime = (crt_point.time != 0); + } else if (reader.name() == QStringLiteral("Position")) { reader.readNextStartElement(); if (reader.name() == QStringLiteral("LatitudeDegrees")) { crt_point.lat = reader.readElementText().toDouble(); crt_point.lat_projected = project_latitude(crt_point.lat); + got_latlon++; } reader.readNextStartElement(); - if (reader.name() == QStringLiteral("LongitudeDegrees")) + if (reader.name() == QStringLiteral("LongitudeDegrees")) { crt_point.lon = reader.readElementText().toDouble(); + got_latlon++; + } } else if (reader.name() == QStringLiteral("AltitudeMeters")) { crt_point.ele = reader.readElementText().toDouble(); + got_other = true; } else if (reader.name() == QStringLiteral("DistanceMeters")) { crt_point.total_dist = reader.readElementText().toDouble(); + got_other = true; } else if (reader.name() == QStringLiteral("HeartRateBpm")) { reader.readNextStartElement(); - if (reader.name() == QStringLiteral("Value")) + if (reader.name() == QStringLiteral("Value")) { crt_point.hr = reader.readElementText().toDouble(); + got_other = true; + } } else if (reader.name() == QStringLiteral("Cadence")) { crt_point.cad = reader.readElementText().toDouble(); + got_other = true; } else if (reader.name() == QStringLiteral("Temperature")) { crt_point.atemp = reader.readElementText().toDouble(); + got_other = true; } else if (reader.isStartElement() && reader.name() == QStringLiteral("Extensions")) { while (reader.readNext() && !(reader.name() == QStringLiteral("TPX") && reader.tokenType() == QXmlStreamReader::EndElement)) { - if (reader.name() == QStringLiteral("Speed")) + if (reader.name() == QStringLiteral("Speed")) { crt_point.speed = reader.readElementText().toDouble(); - else if (reader.name() == QStringLiteral("Watts")) + got_other = true; + } else if (reader.name() == QStringLiteral("Watts")) { crt_point.power = reader.readElementText().toDouble(); + got_other = true; + } } } } - //now add the point to linked list (but only if increasing time) - if (crt_point.time != GPS_UNINIT && crt_point.time > last_time) { - *gps_list = (gps_point_ll *) calloc(1, sizeof(gps_point_ll)); - if (*gps_list == NULL) - return; - - *count_pts += 1; - (*gps_list)->gp = crt_point; - (*gps_list)->next = NULL; - gps_list = &(*gps_list)->next; - last_time = crt_point.time; - } else if (crt_point.time == GPS_UNINIT) { - //if no time is stored, we make it up starting from 1 - gps_point_ll *tmp = (gps_point_ll *) calloc(1, sizeof(gps_point_ll)); - if (tmp == NULL) - return; - if (no_time_head == NULL) { - no_time_head = tmp; - no_time_prev = tmp; - } else { - no_time_prev->next = tmp; - no_time_prev = tmp; - } + + bool valid_pt = (valid_datetime && got_other) || (got_latlon == 2); + if (!valid_pt) { + mlt_log_info( + NULL, + "qxlm_parse_tcx: skipping read point[%d], minimum condition: (datetime + " + "something) or (location)", + nr_pt); + continue; + } + + //store the point in the array, set synthetic time, we'll deal with invalid times after sorting + if (crt_point.time == GPS_UNINIT || crt_point.time == 0) { + //if no time is read (or bad format), we make it up starting from 1 no_time_count += 1; crt_point.time = no_time_count * 1000; - tmp->gp = crt_point; - tmp->next = NULL; - } else - mlt_log_info(NULL, - "qxml_parse_tcx: skipping point due to time [%d] %f,%f - crt:%u.%u, " - "last:%u.%u\n", - *count_pts, - crt_point.lat, - crt_point.lon, - (unsigned) (crt_point.time / 1000), - (unsigned) (crt_point.time % 1000), - (unsigned) (last_time / 1000), - (unsigned) (last_time % 1000)); - } - } - //if no points had time, we replace the real list with the no_time one - if (*count_pts == 0 && no_time_count > 0) { - mlt_log_info(NULL, - "qxml_parse_gpx: all GPS points are missing time values, inserting each " - "point at 1 second interval!\n"); - *gps_list = no_time_head; - *count_pts = no_time_count; - } else if (*count_pts > 0 && no_time_count > 0) { - //cleanup the no_time list as we're not using it - mlt_log_info(NULL, - "qxml_parse_gpx: %d GPS points have no time in original file and have been " - "discarded (total valid points kept: %d)!\n", - no_time_count, - *count_pts); - while (no_time_head) { - gps_point_ll *tmp = no_time_head; - no_time_head = no_time_head->next; - free(tmp); + } + gps_points_file.append(crt_point); } } } @@ -1382,12 +1333,12 @@ int q_parse_txt(QTextStream &reader, QVector &gps_points_file, in //2025/06/12 02:45:01 N:45.123456 E:26.123456 0.0 km/h 0.00 72.50 10 x:-4.13 y:0.27 z:0.49 //date time lat lon speed (unit) heading altitude satelites xyz_gravity_sensor //expected format line: - //#delimiter=" " timestamp="yyyy/MM/dd" timestamp="hh:mm:ss" lat lon speed skip heading altitude + //#delimiter=" " datetime="yyyy/MM/dd" datetime="hh:mm:ss" lat lon speed skip heading altitude //NOTE: there's probably a very limited number of common formats, and 99% of them start with date+time lat lon // so if there's enough interest in this, it could be automated with a decent success rate QString default_format_line - = "#delimiter=\" \" timestamp=\"yyyy/MM/dd\" timestamp=\"hh:mm:ss\" latitude longitude " + = "#delimiter=\" \" datetime=\"yyyy/MM/dd\" datetime=\"hh:mm:ss\" latitude longitude " "speed skip heading altitude"; //regex for yyyy*MM*dd*HH*mm*ss[*sss], any separators (non digits chars), miliseconds optional QRegularExpression re_date_time( @@ -1437,10 +1388,10 @@ int q_parse_txt(QTextStream &reader, QVector &gps_points_file, in // qUtf8Printable(delimiter), // format_tokens.size()); - //extract timestamp with format + //extract datetime with format format_tokens_arguments.resize(format_tokens.size()); for (int i = 0; i < format_tokens.size(); i++) { - if (format_tokens[i].contains("timestamp=", Qt::CaseInsensitive)) { + if (format_tokens[i].contains("datetime=", Qt::CaseInsensitive)) { int firstQ = format_tokens[i].indexOf('"'); int secondQ = format_tokens[i].indexOf('"', firstQ + 1); format_tokens_arguments[i] = format_tokens[i].mid(firstQ + 1, @@ -1588,10 +1539,10 @@ int q_parse_txt(QTextStream &reader, QVector &gps_points_file, in got_latlon++; } else if (format_tokens[i].contains("speed", Qt::CaseInsensitive)) { crt_point.speed = line_tokens[i].toDouble(); - mlt_log_info(NULL, "speed: token='%s', token.toDouble=%f", qPrintable(line_tokens[i]), crt_point.speed); if (i + 1 < line_tokens.size()) { double ratio = 1; - if (qstring_contains_any_qstringlist(line_tokens[i + 1], accepted_speed_formats)) + if (qstring_contains_any_qstringlist(line_tokens[i + 1], + accepted_speed_formats)) ratio = convert_speed_to_format(1, qPrintable(line_tokens[i + 1])); crt_point.speed /= ratio; } @@ -1602,7 +1553,7 @@ int q_parse_txt(QTextStream &reader, QVector &gps_points_file, in if (i + 1 < line_tokens.size()) { double ratio = 1; if (qstring_contains_any_qstringlist(line_tokens[i + 1], - accepted_distance_formats)) + accepted_distance_formats)) ratio = convert_distance_to_format(1, qPrintable(line_tokens[i + 1])); crt_point.speed /= ratio; } @@ -1671,8 +1622,8 @@ int q_parse_txt(QTextStream &reader, QVector &gps_points_file, in // crt_point.atemp, // crt_point.power); - bool valid_line = (valid_datetime && got_other) || (got_latlon == 2); - if (!valid_line) { + bool valid_pt = (valid_datetime && got_other) || (got_latlon == 2); + if (!valid_pt) { mlt_log_info(NULL, "q_parse_txt: skipping invalid line: '%s'\n", qUtf8Printable(line)); if (nr_invalid_lines++ >= 10) { mlt_log_warning(NULL, "q_parse_txt: 10 invalid lines in a row, aborting file\n"); @@ -1729,7 +1680,42 @@ int q_parse_file(gps_private_data gdata) return 0; } - if (filename.endsWith(".txt", Qt::CaseInsensitive)) { + if (filename.endsWith(".gpx", Qt::CaseInsensitive) + || filename.endsWith(".tcx", Qt::CaseInsensitive)) { + QXmlStreamReader qxml_reader(&gps_file); + qxml_reader.setNamespaceProcessing(false); + while (!qxml_reader.atEnd() && !qxml_reader.hasError()) { + qxml_reader.readNextStartElement(); + if (qxml_reader.isStartDocument()) { + mlt_log_info(NULL, + "q_parse_file: StartDocument found. Encoding=%s\n", + qUtf8Printable(qxml_reader.documentEncoding().toString())); + continue; + } + if (qxml_reader.name() == QStringLiteral("TrainingCenterDatabase")) { + qxml_parse_tcx(qxml_reader, gps_points_array, no_time_count); + } else if (qxml_reader.name() == QStringLiteral("gpx")) { + qxml_parse_gpx(qxml_reader, gps_points_array, no_time_count); + } else { + mlt_log_warning(gdata.filter, + "qxml_parse_file: fail to parse file: %s, unknown root element: " + "%s. Aborting.\n", + qUtf8Printable(filename), + qUtf8Printable(qxml_reader.name().toString())); + return 0; + } + } + if (qxml_reader.hasError()) { + mlt_log_info(NULL, + "qxml_reader.hasError! line:%u, errString:%s\n", + (unsigned) qxml_reader.lineNumber(), + qUtf8Printable(qxml_reader.errorString())); + return 0; + } + + qxml_reader.clear(); + } else { + //if (filename.endsWith(".txt", Qt::CaseInsensitive)) { QTextStream qin(&gps_file); if (q_parse_txt(qin, gps_points_array, no_time_count) == 0) { mlt_log_error(NULL, @@ -1737,63 +1723,28 @@ int q_parse_file(gps_private_data gdata) "format line!", qUtf8Printable(filename)); mlt_log_info(NULL, - "Sample format line: \n#delimiter=\" \" timestamp=\"yyyy/MM/dd\" " - "timestamp=\"hh:mm:ss\" latitude longitude speed skip heading altitude"); + "Sample format line: \n#delimiter=\" \" datetime=\"yyyy/MM/dd\" " + "datetime=\"hh:mm:ss\" latitude longitude speed skip heading altitude"); return 0; } - mlt_log_info(NULL, "printing the read array, size=%lld", gps_points_array.size()); - for (auto &pt : gps_points_array) { - mlt_log_info(NULL, - ".time:%lld, lat:%f, lon:%f, speed:%f, distance:%f, ele:%f, hr:%f, " - "bearing:%f, cadence:%f, temp:%f, power:%f\n", - pt.time, - pt.lat, - pt.lon, - pt.speed, - pt.total_dist, - pt.ele, - pt.hr, - pt.bearing, - pt.cad, - pt.atemp, - pt.power); - } + // mlt_log_info(NULL, "printing the read array, size=%lld", gps_points_array.size()); + // for (auto &pt : gps_points_array) { + // mlt_log_info(NULL, + // ".time:%lld, lat:%f, lon:%f, speed:%f, distance:%f, ele:%f, hr:%f, " + // "bearing:%f, cadence:%f, temp:%f, power:%f\n", + // pt.time, + // pt.lat, + // pt.lon, + // pt.speed, + // pt.total_dist, + // pt.ele, + // pt.hr, + // pt.bearing, + // pt.cad, + // pt.atemp, + // pt.power); + // } } - // else { - // QXmlStreamReader qxml_reader(&gps_file); - // qxml_reader.setNamespaceProcessing(false); - // while (!qxml_reader.atEnd() && !qxml_reader.hasError()) { - // qxml_reader.readNextStartElement(); - // if (qxml_reader.isStartDocument()) { - // mlt_log_info(NULL, - // "q_parse_file: StartDocument found. Encoding=%s\n", - // qUtf8Printable(qxml_reader.documentEncoding().toString())); - // continue; - // } - // // mlt_log_info(NULL, " readNext element tag: %s, type=%d\n", qUtf8Printable(qxml_reader.name().toString()), qxml_reader.tokenType()); - // if (qxml_reader.name() == QStringLiteral("TrainingCenterDatabase")) { - // qxml_parse_tcx(qxml_reader, &gps_list_head, &count_pts); - // } else if (qxml_reader.name() == QStringLiteral("gpx")) { - // qxml_parse_gpx(qxml_reader, &gps_list_head, &count_pts); - // } else { - // mlt_log_warning(gdata.filter, - // "q_parse_file: fail to parse file: %s, unknown root element: " - // "%s. Aborting.\n", - // qUtf8Printable(filename), - // qUtf8Printable(qxml_reader.name().toString())); - // return 0; - // } - // } - // if (qxml_reader.hasError()) { - // mlt_log_info(NULL, - // "qxml_reader.hasError! line:%u, errString:%s\n", - // (unsigned) qxml_reader.lineNumber(), - // qUtf8Printable(qxml_reader.errorString())); - // return 0; - // } - - // qxml_reader.clear(); //not sure if really needed if not reused - // } const int gps_pt_count = gps_points_array.size(); mlt_log_info(NULL, @@ -1866,11 +1817,6 @@ int q_parse_file(gps_private_data gdata) return 0; } - mlt_log_warning(NULL, - "*gdata.ptr_to_gps_points_r=%p, gps_array=%p", - *gdata.ptr_to_gps_points_r, - gps_array); - //search to find if we cross the 180 and/or the 0 meridian //TODO: move this logic inside the array copy above and don't just check crt to next, but [next which has valid .lon] bool crosses0 = false, crosses180 = false; @@ -1900,25 +1846,25 @@ int q_parse_file(gps_private_data gdata) } else swap_to_180 = 0; - //debug result: - for (int i = 0; i < *gdata.gps_points_size; i += 100) { - gps_point_raw *crt_point = &gps_array[i]; - mlt_log_info(NULL, - "_xml_parse_file read point[%d]: time:%lld, lat:%f, lon:%f, speed:%f, " - "distance:%f, ele:%f, hr:%f, bearing:%f, cadence:%f, temp:%f, power:%f\n", - i, - crt_point->time, - crt_point->lat, - crt_point->lon, - crt_point->speed, - crt_point->total_dist, - crt_point->ele, - crt_point->hr, - crt_point->bearing, - crt_point->cad, - crt_point->atemp, - crt_point->power); - } + // //debug result: + // for (int i = 0; i < *gdata.gps_points_size; i += 100) { + // gps_point_raw *crt_point = &gps_array[i]; + // mlt_log_info(NULL, + // "_xml_parse_file read point[%d]: time:%lld, lat:%f, lon:%f, speed:%f, " + // "distance:%f, ele:%f, hr:%f, bearing:%f, cadence:%f, temp:%f, power:%f\n", + // i, + // crt_point->time, + // crt_point->lat, + // crt_point->lon, + // crt_point->speed, + // crt_point->total_dist, + // crt_point->ele, + // crt_point->hr, + // crt_point->bearing, + // crt_point->cad, + // crt_point->atemp, + // crt_point->power); + // } return 1; } From 4516971b0ca10c4eb96205c8ee0e9a4d7e670343 Mon Sep 17 00:00:00 2001 From: DanielF <3215812+dany123@users.noreply.github.com> Date: Thu, 15 Oct 2026 22:35:17 +0300 Subject: [PATCH 05/14] clang-format --- src/modules/qt/filter_gpsgraphic.h | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/modules/qt/filter_gpsgraphic.h b/src/modules/qt/filter_gpsgraphic.h index a7707ae93..1e5ff55c4 100644 --- a/src/modules/qt/filter_gpsgraphic.h +++ b/src/modules/qt/filter_gpsgraphic.h @@ -59,12 +59,7 @@ struct private_data } // namespace -enum gpsg_data_sources { - gspg_location_src = 0, - gpsg_altitude_src, - gpsg_hr_src, - gpsg_speed_src -}; +enum gpsg_data_sources { gspg_location_src = 0, gpsg_altitude_src, gpsg_hr_src, gpsg_speed_src }; enum gpsg_data_subids { gpsg_latitude_id = gspg_location_src, From 37c34c9bb774e07321fbec77a9d54c6c84a92c04 Mon Sep 17 00:00:00 2001 From: DanielF <3215812+dany123@users.noreply.github.com> Date: Tue, 20 Oct 2026 18:45:57 +0300 Subject: [PATCH 06/14] small fixes --- src/modules/qt/gps_parser.cpp | 51 +++++++++++++++-------------------- 1 file changed, 22 insertions(+), 29 deletions(-) diff --git a/src/modules/qt/gps_parser.cpp b/src/modules/qt/gps_parser.cpp index 7e10c8dd1..e017bc3d8 100644 --- a/src/modules/qt/gps_parser.cpp +++ b/src/modules/qt/gps_parser.cpp @@ -61,6 +61,10 @@ bool qstring_contains_any_qstringlist(const QString &main_string, const QStringL return false; } +const QString sample_format_line + = "#delimiter=\" \" datetime=\"yyyy/MM/dd\" datetime=\"hh:mm:ss\" latitude longitude " + "speed skip heading altitude"; + // Maps are usually in ESPG3857 format, not in GPS format EPSG4326 / WG84. We have to project // the latitude, otherwise big maps have wrong aspect ratio double project_latitude(double lat) @@ -1326,7 +1330,8 @@ void qxml_parse_tcx(QXmlStreamReader &reader, } //tries to parse a .txt file containing GPS data; returns 0 for failure, 1 success -//first line of the file must contain a format helper that starts with #delimiter= +//first line of the file may contain a format helper that starts with #delimiter= +//if no format is found, we try a generic regex for date+time and lat, lon (no other fields) int q_parse_txt(QTextStream &reader, QVector &gps_points_file, int &no_time_count) { //example from my dashcam (added extra spaces for readability): @@ -1335,11 +1340,6 @@ int q_parse_txt(QTextStream &reader, QVector &gps_points_file, in //expected format line: //#delimiter=" " datetime="yyyy/MM/dd" datetime="hh:mm:ss" lat lon speed skip heading altitude - //NOTE: there's probably a very limited number of common formats, and 99% of them start with date+time lat lon - // so if there's enough interest in this, it could be automated with a decent success rate - QString default_format_line - = "#delimiter=\" \" datetime=\"yyyy/MM/dd\" datetime=\"hh:mm:ss\" latitude longitude " - "speed skip heading altitude"; //regex for yyyy*MM*dd*HH*mm*ss[*sss], any separators (non digits chars), miliseconds optional QRegularExpression re_date_time( R"((\d{4})\D?(\d{2})\D?(\d{2})\D?(\d{2})\D?(\d{2})\D?(\d{2})(?:\D?(\d{1,3}))?)"); @@ -1350,14 +1350,14 @@ int q_parse_txt(QTextStream &reader, QVector &gps_points_file, in QString format_line, first_keyword = "#delimiter=", delimiter; QStringList format_tokens, format_tokens_arguments; - bool regex_mode = false, format_date_time_separated = false; + bool regex_mode = false; //if file doesn't start with "#delimiter=", will try regex for datetime and lat lon if ((format_line = reader.readLine()).startsWith(first_keyword) == false) { mlt_log_info(NULL, "no format line found at the start of the file, trying a generic regex " "for datetime and lat/lon"); - mlt_log_info(NULL, "Note: sample format line: '%s'", qUtf8Printable(default_format_line)); + mlt_log_info(NULL, "Note: sample format line: '%s'", qUtf8Printable(sample_format_line)); reader.seek(0); regex_mode = true; } else if (!reader.atEnd()) { @@ -1396,8 +1396,6 @@ int q_parse_txt(QTextStream &reader, QVector &gps_points_file, in int secondQ = format_tokens[i].indexOf('"', firstQ + 1); format_tokens_arguments[i] = format_tokens[i].mid(firstQ + 1, secondQ - (firstQ + 1)); - if (!format_tokens_arguments[i].contains('y', Qt::CaseInsensitive)) - format_date_time_separated = true; } } } else { //already end of file? this is an invalid file @@ -1409,11 +1407,11 @@ int q_parse_txt(QTextStream &reader, QVector &gps_points_file, in //process the file line by line int nr_invalid_lines = 0; + bool format_date_time_separated = false; QVector warned_of_format(format_tokens.size(), false); while (!reader.atEnd()) { QString line = reader.readLine(256); gps_point_raw crt_point = uninit_gps_raw_point; - //a point is valid if it has either (a valid time + at least one other field) or (a valid lat/lon pair) bool valid_datetime = false, got_other = false; int got_latlon = 0; @@ -1474,14 +1472,18 @@ int q_parse_txt(QTextStream &reader, QVector &gps_points_file, in // i, // line_tokens[i].toUtf8().constData()); if (format_tokens[i].contains("datetime=", Qt::CaseInsensitive)) { + //explicit format provided by the user, can have date separated from time datetime_assembled.append(line_tokens[i]); datetime_assembled.append(delimiter); datetime_format_assembled.append(format_tokens_arguments[i]); datetime_format_assembled.append(delimiter); + format_date_time_separated = true; } else if (format_tokens[i].contains("datetimeiso", Qt::CaseInsensitive)) { + //exact ISO format crt_point.time = datetimeXMLstring_to_mseconds(qUtf8Printable(line_tokens[i])); valid_datetime = (crt_point.time != 0); } else if (format_tokens[i].contains("datetime", Qt::CaseInsensitive)) { + //generic date+time in one token, will use regex for parsing auto m = re_date_time.match(line_tokens[i]); if (m.hasMatch()) { int year = m.captured(1).toInt(); @@ -1502,6 +1504,7 @@ int q_parse_txt(QTextStream &reader, QVector &gps_points_file, in } } } else if (format_tokens[i].contains("date", Qt::CaseInsensitive)) { + //generic date only in a single token, will use regex for parsing auto m = re_date.match(line_tokens[i]); if (m.hasMatch()) { int year = m.captured(1).toInt(); @@ -1512,6 +1515,7 @@ int q_parse_txt(QTextStream &reader, QVector &gps_points_file, in got_qdate = true; } } else if (format_tokens[i].contains("time", Qt::CaseInsensitive)) { + //generic time only in a single token, will use regex for parsing auto m = re_time.match(line_tokens[i]); if (m.hasMatch()) { int hour = m.captured(1).toInt(); @@ -1526,15 +1530,15 @@ int q_parse_txt(QTextStream &reader, QVector &gps_points_file, in } } else if (format_tokens[i].contains("lat", Qt::CaseInsensitive)) { double negative = (line_tokens[i].startsWith('s', Qt::CaseInsensitive) ? -1 : 1); - int number_at = line_tokens[i].indexOf(QRegularExpression("[-+0-9]")); - crt_point.lat = line_tokens[i].mid(number_at).toDouble() * negative; + int number_starts = line_tokens[i].indexOf(QRegularExpression("[-+0-9]")); + crt_point.lat = line_tokens[i].mid(number_starts).toDouble() * negative; crt_point.lat_projected = project_latitude(crt_point.lat); if (is_valid_latitude(crt_point.lat)) got_latlon++; } else if (format_tokens[i].contains("lon", Qt::CaseInsensitive)) { double negative = (line_tokens[i].startsWith('w', Qt::CaseInsensitive) ? -1 : 1); - int number_at = line_tokens[i].indexOf(QRegularExpression("[-+0-9]")); - crt_point.lon = line_tokens[i].mid(number_at).toDouble() * negative; + int number_starts = line_tokens[i].indexOf(QRegularExpression("[-+0-9]")); + crt_point.lon = line_tokens[i].mid(number_starts).toDouble() * negative; if (is_valid_longitude(crt_point.lon)) got_latlon++; } else if (format_tokens[i].contains("speed", Qt::CaseInsensitive)) { @@ -1555,7 +1559,7 @@ int q_parse_txt(QTextStream &reader, QVector &gps_points_file, in if (qstring_contains_any_qstringlist(line_tokens[i + 1], accepted_distance_formats)) ratio = convert_distance_to_format(1, qPrintable(line_tokens[i + 1])); - crt_point.speed /= ratio; + crt_point.ele /= ratio; } got_other = true; } else if (format_tokens[i].contains("head", Qt::CaseInsensitive) @@ -1634,16 +1638,8 @@ int q_parse_txt(QTextStream &reader, QVector &gps_points_file, in nr_invalid_lines = 0; } - //we parsed a line, check the time and store it in the array - // if (crt_point.time != GPS_UNINIT && crt_point.time != 0) { - // if (crt_point.time < last_time) - // all_sorted = false; - // // else if (crt_point.time == last_time) - // // eq_time_count += 1; //TODO: if multiple file support this won't be enough - // last_time = crt_point.time; - // } else if (crt_point.time == GPS_UNINIT || crt_point.time == 0) { - //if no time is read (or bad format), we make it up starting from 1 + //if no time is read (or bad format), we make it up starting from 1 (=1000ms) no_time_count += 1; crt_point.time = no_time_count * 1000; } @@ -1722,9 +1718,7 @@ int q_parse_file(gps_private_data gdata) "txt file (%s) processing failed. Make sure it starts with a recognised " "format line!", qUtf8Printable(filename)); - mlt_log_info(NULL, - "Sample format line: \n#delimiter=\" \" datetime=\"yyyy/MM/dd\" " - "datetime=\"hh:mm:ss\" latitude longitude speed skip heading altitude"); + mlt_log_info(NULL, "Sample format line: '%s'", qUtf8Printable(sample_format_line)); return 0; } // mlt_log_info(NULL, "printing the read array, size=%lld", gps_points_array.size()); @@ -1768,7 +1762,6 @@ int q_parse_file(gps_private_data gdata) bool skip_notimes = false; if (no_time_count > 0 && no_time_count < gps_pt_count) { skip_notimes = true; - // gps_pt_count -= no_time_count; mlt_log_info(NULL, "q_parse_file: will skip %d/%d points without valid time", no_time_count, From ce916d52ee20ec0bd634337cfbb242e6f41c6ca7 Mon Sep 17 00:00:00 2001 From: Daniel F Date: Mon, 5 Oct 2026 15:38:05 +0300 Subject: [PATCH 07/14] GPS filters: auto load companion gps file if found (+some optimizations) --- src/modules/qt/filter_gpsgraphic.cpp | 32 +++- src/modules/qt/filter_gpsgraphic.h | 1 + src/modules/qt/filter_gpstext.cpp | 33 +++- src/modules/qt/gps_parser.cpp | 215 +++++++++++++++------------ src/modules/qt/gps_parser.h | 2 + 5 files changed, 174 insertions(+), 109 deletions(-) diff --git a/src/modules/qt/filter_gpsgraphic.cpp b/src/modules/qt/filter_gpsgraphic.cpp index 528790a7e..644d4d3dc 100644 --- a/src/modules/qt/filter_gpsgraphic.cpp +++ b/src/modules/qt/filter_gpsgraphic.cpp @@ -640,19 +640,39 @@ static void process_file(mlt_filter filter, mlt_frame frame) private_data *pdata = (private_data *) filter->child; mlt_properties properties = MLT_FILTER_PROPERTIES(filter); char *filename = mlt_properties_get(properties, "resource"); - - //if there's no file selected just return - if (!filename || !strcmp(filename, "")) - return; + char auto_filename[PATH_MAX] = {0}; + bool try_auto_filename = false; + + //if there's no file selected, try to find one, and just return if failure or already tried + if (!filename || !strcmp(filename, "")) { + if (pdata->checked_gps_companion) + return; + mlt_log_info(NULL, "checking for gps companion"); + + //get the parent clip's path + mlt_producer producer = mlt_producer_cut_parent(mlt_frame_get_original_producer(frame)); + char *clip_path = mlt_properties_get(MLT_PRODUCER_PROPERTIES(producer), "resource"); + bool ret = q_auto_gps_file(clip_path, auto_filename); + pdata->checked_gps_companion = true; + + if (ret) { + filename = auto_filename; + try_auto_filename = true; + } else + return; + } //check if the file has been changed, if not, current data is ok, do nothing if (strcmp(pdata->last_filename, filename)) { - // mlt_log_info(filter, "Reading new file: last_filename (%s) != entered_filename (%s), swap_180 = %d\n", pdata->last_filename, filename, swap); + // mlt_log_info(filter, "Reading new file: last_filename (%s) != entered_filename (%s), swap_180 = %d\n", pdata->last_filename, filename, pdata->swap_180); default_priv_data(pdata); + strncpy(pdata->last_filename, filename, sizeof(pdata->last_filename) - 1); pdata->last_filename[sizeof(pdata->last_filename) - 1] = '\0'; if (q_parse_file(filter_to_gps_data(filter)) == 1) { + if (try_auto_filename) + mlt_properties_set_string(properties, "resource", filename); get_first_gps_time(filter_to_gps_data(filter)); get_last_gps_time(filter_to_gps_data(filter)); @@ -665,6 +685,8 @@ static void process_file(mlt_filter filter, mlt_frame frame) //store name in pdata or it'll retry reading every frame if bad file strncpy(pdata->last_filename, filename, sizeof(pdata->last_filename) - 1); pdata->last_filename[sizeof(pdata->last_filename) - 1] = '\0'; + if (try_auto_filename) + pdata->checked_gps_companion = true; } } } diff --git a/src/modules/qt/filter_gpsgraphic.h b/src/modules/qt/filter_gpsgraphic.h index 1e5ff55c4..7769aedeb 100644 --- a/src/modules/qt/filter_gpsgraphic.h +++ b/src/modules/qt/filter_gpsgraphic.h @@ -55,6 +55,7 @@ struct private_data double bg_img_scaled_width, bg_img_scaled_height; QRectF bg_img_matched_rect; int swap_180; + bool checked_gps_companion; }; } // namespace diff --git a/src/modules/qt/filter_gpstext.cpp b/src/modules/qt/filter_gpstext.cpp index 6565f6a00..10d87b7f2 100644 --- a/src/modules/qt/filter_gpstext.cpp +++ b/src/modules/qt/filter_gpstext.cpp @@ -36,11 +36,13 @@ typedef struct int64_t gps_offset; int64_t gps_proc_start_t; //process only points after this time (epoch milliseconds) char last_read_gps_processing_start_time[20]; + char last_read_gps_processing_start_time[20]; double speed_multiplier; double updates_per_second; char last_filename[PATH_MAX]; //gps file fullpath char interpolated; int swap_180; + bool checked_gps_companion; } private_data; } // namespace @@ -129,7 +131,6 @@ static int get_next_token(char *str, int *pos, char *token, int *is_keyword) */ static int64_t extract_offset_time_ms_keyword(char *keyword) { - //mlt_log_info(NULL, "extract_offset_time_keyword: in keyword: %s\n", keyword); char *end = NULL; if (keyword == NULL) return 0; @@ -601,22 +602,42 @@ static void process_file(mlt_filter filter, mlt_frame frame) private_data *pdata = (private_data *) filter->child; mlt_properties properties = MLT_FILTER_PROPERTIES(filter); char *filename = mlt_properties_get(properties, "resource"); + char auto_filename[PATH_MAX] = {0}; + bool try_auto_filename = false; + if (filename == NULL) filename = mlt_properties_get(properties, "gps.file"); /* for backwards compatibility with v1 */ - //if there's no file selected just return - if (!filename || !strcmp(filename, "")) - return; + //if there's no file selected, try to find one, and just return if failure or already tried + if (!filename || !strcmp(filename, "")) { + if (pdata->checked_gps_companion) + return; + + //get the parent clip's path + mlt_producer producer = mlt_producer_cut_parent(mlt_frame_get_original_producer(frame)); + char *clip_path = mlt_properties_get(MLT_PRODUCER_PROPERTIES(producer), "resource"); + bool ret = q_auto_gps_file(clip_path, auto_filename); + pdata->checked_gps_companion = true; + + if (ret) { + filename = auto_filename; + try_auto_filename = true; + } else + return; + } + //TODO: fix Ui with valid file from autoload //check if the file has been changed, if not, current data is ok, do nothing if (strcmp(pdata->last_filename, filename)) { - // mlt_log_info(filter, "Reading new file: last_filename (%s) != entered_filename (%s), swap_180 = %d\n", pdata->last_filename, filename, swap); default_priv_data(pdata); + strncpy(pdata->last_filename, filename, sizeof(pdata->last_filename) - 1); pdata->last_filename[sizeof(pdata->last_filename) - 1] = '\0'; if (q_parse_file(filter_to_gps_data(filter)) == 1) { + if (try_auto_filename) + mlt_properties_set_string(properties, "resource", filename); get_first_gps_time(filter_to_gps_data(filter)); get_last_gps_time(filter_to_gps_data(filter)); @@ -628,6 +649,8 @@ static void process_file(mlt_filter filter, mlt_frame frame) //store name in pdata or it'll retry reading every frame if bad file strncpy(pdata->last_filename, filename, sizeof(pdata->last_filename) - 1); pdata->last_filename[sizeof(pdata->last_filename) - 1] = '\0'; + if (try_auto_filename) + pdata->checked_gps_companion = true; } } } diff --git a/src/modules/qt/gps_parser.cpp b/src/modules/qt/gps_parser.cpp index e017bc3d8..fae7e9aa6 100644 --- a/src/modules/qt/gps_parser.cpp +++ b/src/modules/qt/gps_parser.cpp @@ -23,6 +23,8 @@ #endif #include "gps_parser.h" #include +#include +#include #include #include @@ -85,6 +87,29 @@ double get_180_swapped(double lon) return lon + (lon > 0 ? -180 : 180); } +inline void _mlt_log_gps_point(const int i, const gps_point_raw &crt_point) +{ + char str_time[25] = {0}; + mseconds_to_timestring(crt_point.time, NULL, str_time); + mlt_log_info(NULL, + "gps_point[%d]: time:%lld (%s), lat:%f, lon:%f, lat_projected:%f, speed:%f, " + "distance:%f, ele:%f, hr:%f, bearing:%f, cadence:%f, temp:%f, power:%f\n", + i, + crt_point.time, + str_time, + crt_point.lat, + crt_point.lon, + crt_point.lat_projected, + crt_point.speed, + crt_point.total_dist, + crt_point.ele, + crt_point.hr, + crt_point.bearing, + crt_point.cad, + crt_point.atemp, + crt_point.power); +} + //testing max valid time between 2 points = 10 * avg_gps_time, better option would be a user entered value int get_max_gps_diff_ms(gps_private_data gdata) { @@ -1119,7 +1144,7 @@ void qxml_parse_gpx(QXmlStreamReader &reader, */ - int nr_pt = 0; + int nr_pt = 0, print5 = 0; while (!reader.atEnd() && !reader.hasError()) { reader.readNext(); // mlt_log_info(NULL, " readNext tag: %s type = %d\n", qUtf8Printable(reader.name()), reader.tokenType()); @@ -1190,11 +1215,16 @@ void qxml_parse_gpx(QXmlStreamReader &reader, bool valid_pt = (valid_datetime && got_other) || (got_latlon == 2); if (!valid_pt) { - mlt_log_info( - NULL, - "qxlm_parse_gpx: skipping read point[%d], minimum condition: (datetime + " - "something) or (location)", - nr_pt); + if (print5++ < 5) { + mlt_log_info( + NULL, + "qxlm_parse_gpx: skipping read point[%d], minimum condition: (datetime + " + "something) or (location)", + nr_pt); + _mlt_log_gps_point(nr_pt, crt_point); + if (print5 == 5) + mlt_log_info(NULL, "(will stop printing skipped points)"); + } continue; } @@ -1207,6 +1237,8 @@ void qxml_parse_gpx(QXmlStreamReader &reader, gps_points_file.append(crt_point); } } + if (print5 > 5) + mlt_log_info(NULL, "qxml_parse_tcx: skipped %d points (out of %d)", print5, nr_pt); } // Parses a .tcx file into a gps_point_raw linked list @@ -1238,7 +1270,7 @@ void qxml_parse_tcx(QXmlStreamReader &reader, */ - int nr_pt = 0; + int nr_pt = 0, print5 = 0; while (!reader.atEnd() && !reader.hasError()) { reader.readNext(); // mlt_log_info(NULL, " readNextStartElement tag: %s\n", qUtf8Printable(reader.name())); @@ -1310,11 +1342,16 @@ void qxml_parse_tcx(QXmlStreamReader &reader, bool valid_pt = (valid_datetime && got_other) || (got_latlon == 2); if (!valid_pt) { - mlt_log_info( - NULL, - "qxlm_parse_tcx: skipping read point[%d], minimum condition: (datetime + " - "something) or (location)", - nr_pt); + if (print5++ < 5) { + mlt_log_info( + NULL, + "qxlm_parse_gpx: skipping read point[%d], minimum condition: (datetime + " + "something) or (location)", + nr_pt); + _mlt_log_gps_point(nr_pt, crt_point); + if (print5 == 5) + mlt_log_info(NULL, "(will stop printing skipped points)"); + } continue; } @@ -1327,6 +1364,8 @@ void qxml_parse_tcx(QXmlStreamReader &reader, gps_points_file.append(crt_point); } } + if (print5 > 5) + mlt_log_info(NULL, "qxml_parse_tcx: skipped %d points (out of %d)", print5, nr_pt); } //tries to parse a .txt file containing GPS data; returns 0 for failure, 1 success @@ -1428,7 +1467,9 @@ int q_parse_txt(QTextStream &reader, QVector &gps_points_file, in int second = m.captured(6).toInt(); int msec = m.captured(7).isEmpty() ? 0 : m.captured(7).leftJustified(3, '0').toInt(); - QDateTime dt(QDate(year, month, day), QTime(hour, minute, second, msec), Qt::UTC); + QDateTime dt(QDate(year, month, day), + QTime(hour, minute, second, msec), + QTimeZone::UTC); if (dt.isValid()) { crt_point.time = dt.toMSecsSinceEpoch(); valid_datetime = (crt_point.time != 0); @@ -1477,6 +1518,7 @@ int q_parse_txt(QTextStream &reader, QVector &gps_points_file, in datetime_assembled.append(delimiter); datetime_format_assembled.append(format_tokens_arguments[i]); datetime_format_assembled.append(delimiter); +//TODO: is this check necessary? we already know from before if this matches or not but could there be a valid case where we have both this and the other time token? format_date_time_separated = true; } else if (format_tokens[i].contains("datetimeiso", Qt::CaseInsensitive)) { //exact ISO format @@ -1497,7 +1539,7 @@ int q_parse_txt(QTextStream &reader, QVector &gps_points_file, in : m.captured(7).leftJustified(3, '0').toInt(); QDateTime dt(QDate(year, month, day), QTime(hour, minute, second, msec), - Qt::UTC); + QTimeZone::UTC); if (dt.isValid()) { crt_point.time = dt.toMSecsSinceEpoch(); valid_datetime = (crt_point.time != 0); @@ -1610,22 +1652,6 @@ int q_parse_txt(QTextStream &reader, QVector &gps_points_file, in // crt_point.time); } } //end regex_mode / format_mode - - // mlt_log_info(NULL, - // "crt_point parsed txt: time:%lld, lat:%f, lon:%f, speed:%f, " - // "distance:%f, ele:%f, hr:%f, bearing:%f, cadence:%f, temp:%f, power:%f\n", - // crt_point.time, - // crt_point.lat, - // crt_point.lon, - // crt_point.speed, - // crt_point.total_dist, - // crt_point.ele, - // crt_point.hr, - // crt_point.bearing, - // crt_point.cad, - // crt_point.atemp, - // crt_point.power); - bool valid_pt = (valid_datetime && got_other) || (got_latlon == 2); if (!valid_pt) { mlt_log_info(NULL, "q_parse_txt: skipping invalid line: '%s'\n", qUtf8Printable(line)); @@ -1649,6 +1675,29 @@ int q_parse_txt(QTextStream &reader, QVector &gps_points_file, in return 1; } +//checks if there's a file with the same base name as original clip + extension of supported gps +//returns 0 on failure, 1 on success (and fills in auto_filename) +//TODO: maybe also q_parse_file() here - but cache the result so we don't read twice +int q_auto_gps_file(const char *clip_path, char *auto_filename) +{ + // mlt_log_info(NULL, + // "q_auto_gps_file: checking for GPS file near '%s'", + // (clip_path ? clip_path : "NULL")); + if (clip_path) { + QFileInfo clip(QString::fromUtf8(clip_path)); + QStringList exentsions_to_check = {".gpx", ".tcx", ".txt"}; + for (auto ext : exentsions_to_check) { + QString guess = clip.dir().filePath(clip.completeBaseName() + ext); + // mlt_log_info(NULL, "q_auto_gps_file: checking for '%s'", qPrintable(guess)); + if (QFile::exists(guess)) { + strncat(auto_filename, guess.toUtf8(), PATH_MAX); + return 1; + } + } + } + return 0; +} + /* Reads file, parses it and stores the gps values in a gps_point_raw array (allocated inside) * and its size in gps_points_size. * swap_to_180 -> shifts all longitudes around with 180 becoming 0, only use it if user requests it @@ -1658,11 +1707,7 @@ int q_parse_file(gps_private_data gdata) { QVector gps_points_array; int no_time_count = 0; //for gps tracks without time, used for ordering - - // gps_point_ll *gps_list_head = NULL, *tmp = NULL; - // int count_pts = 0, nr = 0; QString filename = gdata.last_filename; - // gps_point_raw *gps_array = NULL; int &swap_to_180 = *gdata.swap180; // mlt_log_info(gdata.filter, "in q_parse_file, filename: %s\n", filename); @@ -1711,7 +1756,6 @@ int q_parse_file(gps_private_data gdata) qxml_reader.clear(); } else { - //if (filename.endsWith(".txt", Qt::CaseInsensitive)) { QTextStream qin(&gps_file); if (q_parse_txt(qin, gps_points_array, no_time_count) == 0) { mlt_log_error(NULL, @@ -1722,31 +1766,17 @@ int q_parse_file(gps_private_data gdata) return 0; } // mlt_log_info(NULL, "printing the read array, size=%lld", gps_points_array.size()); - // for (auto &pt : gps_points_array) { - // mlt_log_info(NULL, - // ".time:%lld, lat:%f, lon:%f, speed:%f, distance:%f, ele:%f, hr:%f, " - // "bearing:%f, cadence:%f, temp:%f, power:%f\n", - // pt.time, - // pt.lat, - // pt.lon, - // pt.speed, - // pt.total_dist, - // pt.ele, - // pt.hr, - // pt.bearing, - // pt.cad, - // pt.atemp, - // pt.power); - // } + // for (int i=0; i< gps_points_array.size(); i++) + // _mlt_log_gps_point(i, gps_points_array[i]); } const int gps_pt_count = gps_points_array.size(); - mlt_log_info(NULL, - "back from parsing the file(s) '%s', gps_points_array.size()=%d, " - "no_time_count=%d", - qPrintable(filename), - gps_pt_count, - no_time_count); + // mlt_log_info(NULL, + // "back from parsing the file(s) '%s', gps_points_array.size()=%d, " + // "no_time_count=%d", + // qPrintable(filename), + // gps_pt_count, + // no_time_count); if (gps_pt_count < 2) { mlt_log_warning(gdata.filter, @@ -1755,10 +1785,11 @@ int q_parse_file(gps_private_data gdata) return 0; } - //make sure points are in chronological order - std::sort(gps_points_array.begin(), gps_points_array.end()); + //make sure points are in chronological order (they usually are) + if (!std::is_sorted(gps_points_array.begin(), gps_points_array.end())) + std::sort(gps_points_array.begin(), gps_points_array.end()); - //if there are entries with and without time, drop the ones without - unless all of them + //if there are entries with and without time, drop the ones without valid time bool skip_notimes = false; if (no_time_count > 0 && no_time_count < gps_pt_count) { skip_notimes = true; @@ -1781,13 +1812,29 @@ int q_parse_file(gps_private_data gdata) //copy points to private array (and skip pts if needed) int nr = 0; + //also do the 180 crossing check + bool crosses0 = false, crosses180 = false; + double last_lon = GPS_UNINIT; for (auto &pt : gps_points_array) { if (skip_notimes && pt.time <= no_time_count * 1000) continue; if (nr - 1 >= 0 && pt.time == gps_array[nr - 1].time) continue; gps_array[nr++] = pt; + + //180 crossing logic here + if (last_lon != GPS_UNINIT && pt.lon != GPS_UNINIT) { + if ((last_lon < 0 && pt.lon > 0) || (pt.lon < 0 && last_lon > 0)) { + if (fabs(last_lon - pt.lon) > 180) + crosses180 = true; + else + crosses0 = true; + } + } + if (pt.lon != GPS_UNINIT) + last_lon = pt.lon; } + if (nr >= 2) { gps_point_raw *tmp = (gps_point_raw *) realloc(gps_array, nr * sizeof(gps_point_raw)); if (tmp != NULL) { @@ -1810,27 +1857,13 @@ int q_parse_file(gps_private_data gdata) return 0; } - //search to find if we cross the 180 and/or the 0 meridian - //TODO: move this logic inside the array copy above and don't just check crt to next, but [next which has valid .lon] - bool crosses0 = false, crosses180 = false; - for (int i = 0; i < *gdata.gps_points_size - 1; i++) { - double crt = gps_array[i].lon; - double next = gps_array[i + 1].lon; - if (crt == GPS_UNINIT || next == GPS_UNINIT) - continue; - if ((crt < 0 && next > 0) || (crt > 0 && next < 0)) { - if (crt - next > 180 || next - crt > 180) - crosses180 = true; - else - crosses0 = true; - } - } - mlt_log_info( - NULL, - "_automatic 180 meridian detection: crosses180=%d, crosses0=%d --> swapping180=%d\n", - crosses180, - crosses0, - (crosses180 && !crosses0)); + // mlt_log_info( + // NULL, + // "_automatic 180 meridian detection: crosses180=%d, crosses0=%d --> swapping180=%d\n", + // crosses180, + // crosses0, + // (crosses180 && !crosses0)); + //if only 180 is crossed, we swap, otherswise we don't do anything if (crosses180 && !crosses0) { swap_to_180 = 1; @@ -1840,24 +1873,8 @@ int q_parse_file(gps_private_data gdata) swap_to_180 = 0; // //debug result: - // for (int i = 0; i < *gdata.gps_points_size; i += 100) { - // gps_point_raw *crt_point = &gps_array[i]; - // mlt_log_info(NULL, - // "_xml_parse_file read point[%d]: time:%lld, lat:%f, lon:%f, speed:%f, " - // "distance:%f, ele:%f, hr:%f, bearing:%f, cadence:%f, temp:%f, power:%f\n", - // i, - // crt_point->time, - // crt_point->lat, - // crt_point->lon, - // crt_point->speed, - // crt_point->total_dist, - // crt_point->ele, - // crt_point->hr, - // crt_point->bearing, - // crt_point->cad, - // crt_point->atemp, - // crt_point->power); - // } + // for (int i = 0; i < *gdata.gps_points_size; i += 100) + // _mlt_log_gps_point(i, gps_array[i]); return 1; } diff --git a/src/modules/qt/gps_parser.h b/src/modules/qt/gps_parser.h index cfa472282..21bb93be2 100644 --- a/src/modules/qt/gps_parser.h +++ b/src/modules/qt/gps_parser.h @@ -19,6 +19,7 @@ #ifndef GPS_PARSER_H #define GPS_PARSER_H +#include #include #include #include @@ -151,6 +152,7 @@ gps_point_proc weighted_middle_point_proc(gps_point_proc *p1, void qxml_parse_gpx(QXmlStreamReader &reader, gps_point_ll **gps_list, int *count_pts); void qxml_parse_tcx(QXmlStreamReader &reader, gps_point_ll **gps_list, int *count_pts); +int q_auto_gps_file(const char *clip_path, char *auto_filename); int q_parse_file(gps_private_data gdata); #define swap_180_if_needed(lon) (pdata->swap_180 ? get_180_swapped(lon) : lon) From 75193404dc0bc9dac894d54034493de17bb99d01 Mon Sep 17 00:00:00 2001 From: Daniel F Date: Tue, 6 Oct 2026 03:02:48 +0300 Subject: [PATCH 08/14] GPS filters: update default rect to something smaller --- src/modules/qt/filter_gpsgraphic.cpp | 3 +-- src/modules/qt/filter_gpstext.cpp | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/modules/qt/filter_gpsgraphic.cpp b/src/modules/qt/filter_gpsgraphic.cpp index 644d4d3dc..09fd8cad3 100644 --- a/src/modules/qt/filter_gpsgraphic.cpp +++ b/src/modules/qt/filter_gpsgraphic.cpp @@ -647,7 +647,6 @@ static void process_file(mlt_filter filter, mlt_frame frame) if (!filename || !strcmp(filename, "")) { if (pdata->checked_gps_companion) return; - mlt_log_info(NULL, "checking for gps companion"); //get the parent clip's path mlt_producer producer = mlt_producer_cut_parent(mlt_frame_get_original_producer(frame)); @@ -762,7 +761,7 @@ mlt_filter filter_gpsgraphic_init(mlt_profile profile, mlt_properties_set_int(properties, "show_now_text", 1); mlt_properties_set_double(properties, "angle", 0); mlt_properties_set_int(properties, "thickness", 5); - mlt_properties_set(properties, "rect", "10% 10% 30% 30%"); + mlt_properties_set(properties, "rect", "70% 70% 25% 25%"); mlt_properties_set_int(properties, "show_grid", 0); mlt_properties_set(properties, "legend_unit", ""); mlt_properties_set_int(properties, "draw_individual_dots", 0); diff --git a/src/modules/qt/filter_gpstext.cpp b/src/modules/qt/filter_gpstext.cpp index 10d87b7f2..fc0eede3c 100644 --- a/src/modules/qt/filter_gpstext.cpp +++ b/src/modules/qt/filter_gpstext.cpp @@ -741,7 +741,7 @@ mlt_filter filter_gpstext_init(mlt_profile profile, mlt_service_type type, const "Altitude: #gps_elev#m\n\n" "GPS time: #gps_datetime_now# UTC\n" "GPS location: #gps_lat#, #gps_lon#"); - mlt_properties_set_string(my_properties, "geometry", "10%/10%:80%x80%:100%"); + mlt_properties_set_string(my_properties, "geometry", "10%/55%:80%x35%:100%"); mlt_properties_set_string(my_properties, "family", "Sans"); mlt_properties_set_string(my_properties, "size", "26"); mlt_properties_set_string(my_properties, "weight", "400"); From abf6e671ab53fa6fbdf57b8a127fb4970a126a13 Mon Sep 17 00:00:00 2001 From: DanielF <3215812+dany123@users.noreply.github.com> Date: Wed, 21 Oct 2026 09:57:14 +0300 Subject: [PATCH 09/14] rebase cleanup, minor fixes and tweaks --- src/modules/qt/filter_gpsgraphic.cpp | 3 +- src/modules/qt/filter_gpstext.cpp | 8 ++--- src/modules/qt/gps_parser.cpp | 46 ++++++++++------------------ src/modules/qt/gps_parser.h | 16 +++++----- 4 files changed, 27 insertions(+), 46 deletions(-) diff --git a/src/modules/qt/filter_gpsgraphic.cpp b/src/modules/qt/filter_gpsgraphic.cpp index 09fd8cad3..c51f04dd2 100644 --- a/src/modules/qt/filter_gpsgraphic.cpp +++ b/src/modules/qt/filter_gpsgraphic.cpp @@ -663,7 +663,6 @@ static void process_file(mlt_filter filter, mlt_frame frame) //check if the file has been changed, if not, current data is ok, do nothing if (strcmp(pdata->last_filename, filename)) { - // mlt_log_info(filter, "Reading new file: last_filename (%s) != entered_filename (%s), swap_180 = %d\n", pdata->last_filename, filename, pdata->swap_180); default_priv_data(pdata); strncpy(pdata->last_filename, filename, sizeof(pdata->last_filename) - 1); @@ -761,7 +760,7 @@ mlt_filter filter_gpsgraphic_init(mlt_profile profile, mlt_properties_set_int(properties, "show_now_text", 1); mlt_properties_set_double(properties, "angle", 0); mlt_properties_set_int(properties, "thickness", 5); - mlt_properties_set(properties, "rect", "70% 70% 25% 25%"); + mlt_properties_set(properties, "rect", "70% 70% 20% 20%"); mlt_properties_set_int(properties, "show_grid", 0); mlt_properties_set(properties, "legend_unit", ""); mlt_properties_set_int(properties, "draw_individual_dots", 0); diff --git a/src/modules/qt/filter_gpstext.cpp b/src/modules/qt/filter_gpstext.cpp index fc0eede3c..7ec4b8c6f 100644 --- a/src/modules/qt/filter_gpstext.cpp +++ b/src/modules/qt/filter_gpstext.cpp @@ -24,7 +24,7 @@ namespace { -typedef struct +struct private_data { gps_point_raw *gps_points_r; //raw gps data from file gps_point_proc *gps_points_p; //processed gps data @@ -36,14 +36,13 @@ typedef struct int64_t gps_offset; int64_t gps_proc_start_t; //process only points after this time (epoch milliseconds) char last_read_gps_processing_start_time[20]; - char last_read_gps_processing_start_time[20]; double speed_multiplier; double updates_per_second; char last_filename[PATH_MAX]; //gps file fullpath char interpolated; int swap_180; bool checked_gps_companion; -} private_data; +}; } // namespace @@ -626,7 +625,6 @@ static void process_file(mlt_filter filter, mlt_frame frame) } else return; } - //TODO: fix Ui with valid file from autoload //check if the file has been changed, if not, current data is ok, do nothing if (strcmp(pdata->last_filename, filename)) { @@ -741,7 +739,7 @@ mlt_filter filter_gpstext_init(mlt_profile profile, mlt_service_type type, const "Altitude: #gps_elev#m\n\n" "GPS time: #gps_datetime_now# UTC\n" "GPS location: #gps_lat#, #gps_lon#"); - mlt_properties_set_string(my_properties, "geometry", "10%/55%:80%x35%:100%"); + mlt_properties_set_string(my_properties, "geometry", "10%/50%:80%x40%:100%"); mlt_properties_set_string(my_properties, "family", "Sans"); mlt_properties_set_string(my_properties, "size", "26"); mlt_properties_set_string(my_properties, "weight", "400"); diff --git a/src/modules/qt/gps_parser.cpp b/src/modules/qt/gps_parser.cpp index fae7e9aa6..d16ec621b 100644 --- a/src/modules/qt/gps_parser.cpp +++ b/src/modules/qt/gps_parser.cpp @@ -128,23 +128,6 @@ double get_avg_gps_time_ms(gps_private_data gdata) return (double) tdiff / gp_size; } -/* int64_t datetime_unknown_to_mseconds (const char *text) -{ - //Qt::ISODateWithMs, - //Qt::ISODate, - const QStringList date_formats { - "yyyy/MM/dd HH:mm:ss.zzz", - "yyyy/MM/dd HH:mm:ss", - "yyyy-MM-dd HH:mm:ss.zzz", - "yyyy-MM-dd HH:mm:ss", - "yyyy/MM/dd", - "yyyy-MM-dd", - "HH:mm:ss.zzz", - "HH:mm:ss" - }; - -} */ - /* Converts the datetime string from gps file into seconds since epoch * Note: assumes UTC */ @@ -1384,12 +1367,13 @@ int q_parse_txt(QTextStream &reader, QVector &gps_points_file, in R"((\d{4})\D?(\d{2})\D?(\d{2})\D?(\d{2})\D?(\d{2})\D?(\d{2})(?:\D?(\d{1,3}))?)"); QRegularExpression re_date(R"((\d{4})\D?(\d{2})\D?(\d{2}))"); QRegularExpression re_time(R"((\d{2})\D?(\d{2})\D?(\d{2})(?:\D?(\d{1,3}))?)"); - QRegularExpression re_lat(R"(([NS])[: ]?([+-]?\d{1,3}[\.,]\d{6}))"); - QRegularExpression re_lon(R"(([EW])[: ]?([+-]?\d{1,3}[\.,]\d{6}))"); + QRegularExpression re_lat(R"(([NS])[: ]?([+-]?\d{1,3}[\.]\d{6}))"); + QRegularExpression re_lon(R"(([EW])[: ]?([+-]?\d{1,3}[\.]\d{6}))"); - QString format_line, first_keyword = "#delimiter=", delimiter; + const QString first_keyword = "#delimiter="; + QString format_line, delimiter; QStringList format_tokens, format_tokens_arguments; - bool regex_mode = false; + bool regex_mode = false, format_date_time_separated = false; //if file doesn't start with "#delimiter=", will try regex for datetime and lat lon if ((format_line = reader.readLine()).startsWith(first_keyword) == false) { @@ -1435,6 +1419,7 @@ int q_parse_txt(QTextStream &reader, QVector &gps_points_file, in int secondQ = format_tokens[i].indexOf('"', firstQ + 1); format_tokens_arguments[i] = format_tokens[i].mid(firstQ + 1, secondQ - (firstQ + 1)); + format_date_time_separated = true; } } } else { //already end of file? this is an invalid file @@ -1446,10 +1431,9 @@ int q_parse_txt(QTextStream &reader, QVector &gps_points_file, in //process the file line by line int nr_invalid_lines = 0; - bool format_date_time_separated = false; QVector warned_of_format(format_tokens.size(), false); while (!reader.atEnd()) { - QString line = reader.readLine(256); + QString line = reader.readLine(); gps_point_raw crt_point = uninit_gps_raw_point; //a point is valid if it has either (a valid time + at least one other field) or (a valid lat/lon pair) bool valid_datetime = false, got_other = false; @@ -1499,7 +1483,7 @@ int q_parse_txt(QTextStream &reader, QVector &gps_points_file, in } } } else { //format provided by the file - QStringList line_tokens = line.split(delimiter, Qt::SkipEmptyParts); + QStringList line_tokens = line.split(delimiter); QString datetime_assembled, datetime_format_assembled; bool got_qdate = false, got_qtime = false; QDate qdate_part; @@ -1518,8 +1502,6 @@ int q_parse_txt(QTextStream &reader, QVector &gps_points_file, in datetime_assembled.append(delimiter); datetime_format_assembled.append(format_tokens_arguments[i]); datetime_format_assembled.append(delimiter); -//TODO: is this check necessary? we already know from before if this matches or not but could there be a valid case where we have both this and the other time token? - format_date_time_separated = true; } else if (format_tokens[i].contains("datetimeiso", Qt::CaseInsensitive)) { //exact ISO format crt_point.time = datetimeXMLstring_to_mseconds(qUtf8Printable(line_tokens[i])); @@ -1589,7 +1571,9 @@ int q_parse_txt(QTextStream &reader, QVector &gps_points_file, in double ratio = 1; if (qstring_contains_any_qstringlist(line_tokens[i + 1], accepted_speed_formats)) - ratio = convert_speed_to_format(1, qPrintable(line_tokens[i + 1])); + ratio = convert_speed_to_format(1, + qPrintable( + line_tokens[i + 1].toLower())); crt_point.speed /= ratio; } got_other = true; @@ -1600,7 +1584,9 @@ int q_parse_txt(QTextStream &reader, QVector &gps_points_file, in double ratio = 1; if (qstring_contains_any_qstringlist(line_tokens[i + 1], accepted_distance_formats)) - ratio = convert_distance_to_format(1, qPrintable(line_tokens[i + 1])); + ratio = convert_distance_to_format(1, + qPrintable( + line_tokens[i + 1].toLower())); crt_point.ele /= ratio; } got_other = true; @@ -1677,7 +1663,7 @@ int q_parse_txt(QTextStream &reader, QVector &gps_points_file, in //checks if there's a file with the same base name as original clip + extension of supported gps //returns 0 on failure, 1 on success (and fills in auto_filename) -//TODO: maybe also q_parse_file() here - but cache the result so we don't read twice +//TODO: do q_parse_file() here even if we process twice?// actually should we keep in cache all read files? int q_auto_gps_file(const char *clip_path, char *auto_filename) { // mlt_log_info(NULL, @@ -1690,7 +1676,7 @@ int q_auto_gps_file(const char *clip_path, char *auto_filename) QString guess = clip.dir().filePath(clip.completeBaseName() + ext); // mlt_log_info(NULL, "q_auto_gps_file: checking for '%s'", qPrintable(guess)); if (QFile::exists(guess)) { - strncat(auto_filename, guess.toUtf8(), PATH_MAX); + strncat(auto_filename, guess.toUtf8(), PATH_MAX - 1); return 1; } } diff --git a/src/modules/qt/gps_parser.h b/src/modules/qt/gps_parser.h index 21bb93be2..44ba0824d 100644 --- a/src/modules/qt/gps_parser.h +++ b/src/modules/qt/gps_parser.h @@ -19,7 +19,6 @@ #ifndef GPS_PARSER_H #define GPS_PARSER_H -#include #include #include #include @@ -39,12 +38,6 @@ struct gps_point_raw bool operator<(const gps_point_raw &pt2) const { return time < pt2.time; } }; -typedef struct gps_point_raw_list -{ - gps_point_raw gp; - struct gps_point_raw_list *next; -} gps_point_ll; - struct gps_point_proc { double lat, lon, lat_projected, speed, speed_vertical, speed_3d, total_dist, ele, hr, bearing, @@ -150,8 +143,13 @@ gps_point_proc weighted_middle_point_proc(gps_point_proc *p1, int64_t new_t, int max_gps_diff_ms); -void qxml_parse_gpx(QXmlStreamReader &reader, gps_point_ll **gps_list, int *count_pts); -void qxml_parse_tcx(QXmlStreamReader &reader, gps_point_ll **gps_list, int *count_pts); +void qxml_parse_gpx(QXmlStreamReader &reader, + QVector &gps_points_file, + int &no_time_count); +void qxml_parse_tcx(QXmlStreamReader &reader, + QVector &gps_points_file, + int &no_time_count); +int q_parse_txt(QTextStream &reader, QVector &gps_points_file, int &no_time_count); int q_auto_gps_file(const char *clip_path, char *auto_filename); int q_parse_file(gps_private_data gdata); From c52a7e48006bcc2fa4cd349cf92304824e5712d9 Mon Sep 17 00:00:00 2001 From: DanielF <3215812+dany123@users.noreply.github.com> Date: Wed, 21 Oct 2026 21:15:15 +0300 Subject: [PATCH 10/14] fix header, string to char and timezone --- src/modules/qt/gps_parser.cpp | 10 +++++----- src/modules/qt/gps_parser.h | 1 + 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/modules/qt/gps_parser.cpp b/src/modules/qt/gps_parser.cpp index d16ec621b..4ff30f1bd 100644 --- a/src/modules/qt/gps_parser.cpp +++ b/src/modules/qt/gps_parser.cpp @@ -1620,7 +1620,7 @@ int q_parse_txt(QTextStream &reader, QVector &gps_points_file, in //process datetime now after the entire line is parsed: if (got_qdate && got_qtime) { - QDateTime dt(qdate_part, qtime_part); + QDateTime dt(qdate_part, qtime_part, QTimeZone::UTC); if (dt.isValid()) { crt_point.time = dt.toMSecsSinceEpoch(); valid_datetime = (crt_point.time != 0); @@ -1671,12 +1671,12 @@ int q_auto_gps_file(const char *clip_path, char *auto_filename) // (clip_path ? clip_path : "NULL")); if (clip_path) { QFileInfo clip(QString::fromUtf8(clip_path)); - QStringList exentsions_to_check = {".gpx", ".tcx", ".txt"}; - for (auto ext : exentsions_to_check) { + QStringList extensions_to_check = {".gpx", ".GPX", ".tcx", ".TCX", ".txt", ".TXT"}; + for (auto ext : extensions_to_check) { QString guess = clip.dir().filePath(clip.completeBaseName() + ext); - // mlt_log_info(NULL, "q_auto_gps_file: checking for '%s'", qPrintable(guess)); if (QFile::exists(guess)) { - strncat(auto_filename, guess.toUtf8(), PATH_MAX - 1); + QByteArray guess_utf8 = guess.toUtf8(); + strncat(auto_filename, guess_utf8.constData(), PATH_MAX - strlen(auto_filename) - 1); return 1; } } diff --git a/src/modules/qt/gps_parser.h b/src/modules/qt/gps_parser.h index 44ba0824d..5c8251efa 100644 --- a/src/modules/qt/gps_parser.h +++ b/src/modules/qt/gps_parser.h @@ -22,6 +22,7 @@ #include #include #include +#include #include // #include From 0c9e9b7f06aaed4bfb1c0d6e047ad534b16d8bfe Mon Sep 17 00:00:00 2001 From: DanielF <3215812+dany123@users.noreply.github.com> Date: Wed, 21 Oct 2026 21:57:24 +0300 Subject: [PATCH 11/14] fix int64 lld and timezone --- src/modules/qt/gps_parser.cpp | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/src/modules/qt/gps_parser.cpp b/src/modules/qt/gps_parser.cpp index 4ff30f1bd..547c3653c 100644 --- a/src/modules/qt/gps_parser.cpp +++ b/src/modules/qt/gps_parser.cpp @@ -92,7 +92,8 @@ inline void _mlt_log_gps_point(const int i, const gps_point_raw &crt_point) char str_time[25] = {0}; mseconds_to_timestring(crt_point.time, NULL, str_time); mlt_log_info(NULL, - "gps_point[%d]: time:%lld (%s), lat:%f, lon:%f, lat_projected:%f, speed:%f, " + "gps_point[%d]: time:%" PRId64 + " (%s), lat:%f, lon:%f, lat_projected:%f, speed:%f, " "distance:%f, ele:%f, hr:%f, bearing:%f, cadence:%f, temp:%f, power:%f\n", i, crt_point.time, @@ -150,7 +151,7 @@ int64_t datetimeXMLstring_to_mseconds(const char *text, const char *format /* = return 0; } -#if QT_VERSION > QT_VERSION_CHECK(6, 5, 0) +#if QT_VERSION >= QT_VERSION_CHECK(6, 5, 0) datetime.setTimeZone(QTimeZone::UTC); #else datetime.setTimeZone(QTimeZone(0)); @@ -1406,7 +1407,7 @@ int q_parse_txt(QTextStream &reader, QVector &gps_points_file, in } // mlt_log_debug(NULL, - // "q_parse_txt(): format_line='%s', delimiter='%s', format_tokens found=%lld", + // "q_parse_txt(): format_line='%s', delimiter='%s', format_tokens found=%" PRId64 "", // qUtf8Printable(format_line), // qUtf8Printable(delimiter), // format_tokens.size()); @@ -1451,9 +1452,7 @@ int q_parse_txt(QTextStream &reader, QVector &gps_points_file, in int second = m.captured(6).toInt(); int msec = m.captured(7).isEmpty() ? 0 : m.captured(7).leftJustified(3, '0').toInt(); - QDateTime dt(QDate(year, month, day), - QTime(hour, minute, second, msec), - QTimeZone::UTC); + QDateTime dt(QDate(year, month, day), QTime(hour, minute, second, msec), Qt::UTC); if (dt.isValid()) { crt_point.time = dt.toMSecsSinceEpoch(); valid_datetime = (crt_point.time != 0); @@ -1521,7 +1520,7 @@ int q_parse_txt(QTextStream &reader, QVector &gps_points_file, in : m.captured(7).leftJustified(3, '0').toInt(); QDateTime dt(QDate(year, month, day), QTime(hour, minute, second, msec), - QTimeZone::UTC); + Qt::UTC); if (dt.isValid()) { crt_point.time = dt.toMSecsSinceEpoch(); valid_datetime = (crt_point.time != 0); @@ -1620,7 +1619,7 @@ int q_parse_txt(QTextStream &reader, QVector &gps_points_file, in //process datetime now after the entire line is parsed: if (got_qdate && got_qtime) { - QDateTime dt(qdate_part, qtime_part, QTimeZone::UTC); + QDateTime dt(qdate_part, qtime_part, Qt::UTC); if (dt.isValid()) { crt_point.time = dt.toMSecsSinceEpoch(); valid_datetime = (crt_point.time != 0); @@ -1632,7 +1631,7 @@ int q_parse_txt(QTextStream &reader, QVector &gps_points_file, in valid_datetime = (crt_point.time != 0); // mlt_log_info(NULL, // "format_date_time_separated true, assembled input:'%s', assembled format:'%s', " - // "result=%lld", + // "result=%" PRId64 "", // qUtf8Printable(datetime_assembled), // qUtf8Printable(datetime_format_assembled), // crt_point.time); @@ -1751,7 +1750,7 @@ int q_parse_file(gps_private_data gdata) mlt_log_info(NULL, "Sample format line: '%s'", qUtf8Printable(sample_format_line)); return 0; } - // mlt_log_info(NULL, "printing the read array, size=%lld", gps_points_array.size()); + // mlt_log_info(NULL, "printing the read array, size=%" PRId64 "", gps_points_array.size()); // for (int i=0; i< gps_points_array.size(); i++) // _mlt_log_gps_point(i, gps_points_array[i]); } From c62acbf359a3982554a78fa68f5c8b987056a8b1 Mon Sep 17 00:00:00 2001 From: DanielF <3215812+dany123@users.noreply.github.com> Date: Thu, 22 Oct 2026 15:10:32 +0300 Subject: [PATCH 12/14] update regex, update default rect --- src/modules/qt/filter_gpsgraphic.cpp | 2 +- src/modules/qt/filter_gpstext.cpp | 2 +- src/modules/qt/gps_parser.cpp | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/modules/qt/filter_gpsgraphic.cpp b/src/modules/qt/filter_gpsgraphic.cpp index c51f04dd2..76d18a280 100644 --- a/src/modules/qt/filter_gpsgraphic.cpp +++ b/src/modules/qt/filter_gpsgraphic.cpp @@ -760,7 +760,7 @@ mlt_filter filter_gpsgraphic_init(mlt_profile profile, mlt_properties_set_int(properties, "show_now_text", 1); mlt_properties_set_double(properties, "angle", 0); mlt_properties_set_int(properties, "thickness", 5); - mlt_properties_set(properties, "rect", "70% 70% 20% 20%"); + mlt_properties_set(properties, "rect", "75% 75% 20% 20%"); mlt_properties_set_int(properties, "show_grid", 0); mlt_properties_set(properties, "legend_unit", ""); mlt_properties_set_int(properties, "draw_individual_dots", 0); diff --git a/src/modules/qt/filter_gpstext.cpp b/src/modules/qt/filter_gpstext.cpp index 7ec4b8c6f..d421041b0 100644 --- a/src/modules/qt/filter_gpstext.cpp +++ b/src/modules/qt/filter_gpstext.cpp @@ -739,7 +739,7 @@ mlt_filter filter_gpstext_init(mlt_profile profile, mlt_service_type type, const "Altitude: #gps_elev#m\n\n" "GPS time: #gps_datetime_now# UTC\n" "GPS location: #gps_lat#, #gps_lon#"); - mlt_properties_set_string(my_properties, "geometry", "10%/50%:80%x40%:100%"); + mlt_properties_set_string(my_properties, "geometry", "5% 50% 45% 45% 100%"); mlt_properties_set_string(my_properties, "family", "Sans"); mlt_properties_set_string(my_properties, "size", "26"); mlt_properties_set_string(my_properties, "weight", "400"); diff --git a/src/modules/qt/gps_parser.cpp b/src/modules/qt/gps_parser.cpp index 547c3653c..1eb9e4ed0 100644 --- a/src/modules/qt/gps_parser.cpp +++ b/src/modules/qt/gps_parser.cpp @@ -1368,8 +1368,8 @@ int q_parse_txt(QTextStream &reader, QVector &gps_points_file, in R"((\d{4})\D?(\d{2})\D?(\d{2})\D?(\d{2})\D?(\d{2})\D?(\d{2})(?:\D?(\d{1,3}))?)"); QRegularExpression re_date(R"((\d{4})\D?(\d{2})\D?(\d{2}))"); QRegularExpression re_time(R"((\d{2})\D?(\d{2})\D?(\d{2})(?:\D?(\d{1,3}))?)"); - QRegularExpression re_lat(R"(([NS])[: ]?([+-]?\d{1,3}[\.]\d{6}))"); - QRegularExpression re_lon(R"(([EW])[: ]?([+-]?\d{1,3}[\.]\d{6}))"); + QRegularExpression re_lat(R"(([NS])[: ]?([+-]?\d{1,3}[\.]\d{4,}))"); + QRegularExpression re_lon(R"(([EW])[: ]?([+-]?\d{1,3}[\.]\d{4,}))"); const QString first_keyword = "#delimiter="; QString format_line, delimiter; @@ -1695,7 +1695,7 @@ int q_parse_file(gps_private_data gdata) QString filename = gdata.last_filename; int &swap_to_180 = *gdata.swap180; - // mlt_log_info(gdata.filter, "in q_parse_file, filename: %s\n", filename); + // mlt_log_info(gdata.filter, "in q_parse_file, filename: %s\n", qUtf8Printable(filename)); QFile gps_file(filename); From 33aa3611674794c366c411acf70bc525ea2a443d Mon Sep 17 00:00:00 2001 From: DanielF <3215812+dany123@users.noreply.github.com> Date: Fri, 23 Oct 2026 18:54:42 +0300 Subject: [PATCH 13/14] update yml with txt file support --- src/modules/qt/filter_gpsgraphic.yml | 5 ++-- src/modules/qt/filter_gpstext.yml | 42 ++++++++++++++++++++++++++-- 2 files changed, 42 insertions(+), 5 deletions(-) diff --git a/src/modules/qt/filter_gpsgraphic.yml b/src/modules/qt/filter_gpsgraphic.yml index d405818e0..b4bc77b7c 100644 --- a/src/modules/qt/filter_gpsgraphic.yml +++ b/src/modules/qt/filter_gpsgraphic.yml @@ -2,7 +2,7 @@ schema_version: 7.2 type: filter identifier: gpsgraphic title: GPS Graphic -version: 4 +version: 5 copyright: Meltytech, LLC creator: Daniel F license: LGPLv2.1 @@ -17,7 +17,8 @@ parameters: - identifier: resource title: GPS File/URL description: > - The fullpath of file containing location (GPS) data. Supported extensions: .gpx, .tcx. + The fullpath of the file containing location (GPS) data. Supported extensions: .gpx, .tcx .txt + (see the gpstext yml for .txt support details) type: string required: yes readonly: no diff --git a/src/modules/qt/filter_gpstext.yml b/src/modules/qt/filter_gpstext.yml index eeacdb096..63ea51425 100644 --- a/src/modules/qt/filter_gpstext.yml +++ b/src/modules/qt/filter_gpstext.yml @@ -2,7 +2,7 @@ schema_version: 7.2 type: filter identifier: gpstext title: GPS Text -version: 9 +version: 10 copyright: Meltytech, LLC creator: Daniel F license: LGPLv2.1 @@ -81,8 +81,44 @@ parameters: - identifier: resource title: GPS File/URL - description: > - "The fullpath of file containing location (GPS) data. Supported extensions: .gpx, .tcx." + description: | + The fullpath of the file containing location (GPS) data. Supported extensions: .gpx, .tcx. .txt. + Note: while .gpx and .tcx are international standards, .txt files do not generally contain + details on how and what GPS details are storing yet many devices (mainly dashcams) use them + anyway and record positions in a human-readable format. + Also note that while text files usually have the .txt extension, any input file that is not + ending in .gpx or .tcx will be passed to the generic text_parser, so as long as it is text + (i.e. not binary) and it has one entry per line, there is a good chance + At this point there are 2 modes for parsing text files: + 1) automatic regex: + - this was introduced as a no hassle alternative for just loading a reasonable formatted + plaintext file. It only looks for datetime and a latitude/longitude pair. + -it searches for this pattern: yyyy*MM*dd*HH*mm*ss[.sss] (with optional 1-3 digit miliseconds) and + if found, it parses it as the date-time; any non-digit separators can exist between these fields + so both 2026/10/23_12:23:34 and 20261023122334 are found correctly + -it then searches for latitude (and after it, a longitude) with a regex matching 0.1234 to 123.45678910 + (so 1-3 digits followed by a dot then 4+ digits). Optionally, in front it can have NSEW or +- + (see the q_parse_txt() function in gps_parser.cpp for the exact QT regexes used) + 2) manual format line: + - regex is limited and can fail, so if you want extra fields or if your date is non standard + (like month before day) there is another option, you need to manually edit the .txt file and + insert a format string like the one below as the first line. + #delimiter=" " datetime="yyyy/MM/dd" datetime="hh:mm:ss" latitude longitude speed skip heading altitude + -the "#delimiter=" keyword is mandatory at the start, and it will treat everything between the "" as + the delimiter. Common values are space " " and comma "," + -datetime= is a special keyword that takes an argument between "" which will be passed + to QDateTime::fromString, so use the patterns QT accepts + -specially for date and time there are also the "date" and "time" keywords that will attempt + to separately match yyyy*MM*dd and hh*mm*ss*[.ms], "datetime" which will do them together, + and datetimeiso which will match ISO 8601 extended format ("2026-10-23T18:25:20.739" with + optional timezone) + -all supported keywords: datetime= datetimeiso datetime date time + lat lon speed alt ele head bear hr heart cad temp pow + (yes, you can also write latitude and it will match "lat", and yes, alt and ele are the same thing) + -Note: any format/keyword name not recognized will be skipped along with it's corresponding column value, + in my example above the "skip" keyword simply skips over the unit which follows the speed; + -also note that everything after the last keyword is skipped so you don't need to bother with the + entire line in the file type: string required: yes readonly: no From f6ac811feb4a31704169f47a9cc3c41465a5f2e0 Mon Sep 17 00:00:00 2001 From: DanielF <3215812+dany123@users.noreply.github.com> Date: Sat, 24 Oct 2026 11:33:57 +0300 Subject: [PATCH 14/14] fix yaml --- src/modules/qt/filter_gpstext.yml | 64 +++++++++++++++---------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/src/modules/qt/filter_gpstext.yml b/src/modules/qt/filter_gpstext.yml index 63ea51425..85898e331 100644 --- a/src/modules/qt/filter_gpstext.yml +++ b/src/modules/qt/filter_gpstext.yml @@ -83,42 +83,42 @@ parameters: title: GPS File/URL description: | The fullpath of the file containing location (GPS) data. Supported extensions: .gpx, .tcx. .txt. - Note: while .gpx and .tcx are international standards, .txt files do not generally contain - details on how and what GPS details are storing yet many devices (mainly dashcams) use them - anyway and record positions in a human-readable format. + Note: while .gpx and .tcx are international standards, .txt files do not generally contain info + on how and what GPS details are storing, yet many devices (mainly dashcams) use them anyway and + record positions in a human-readable format. Also note that while text files usually have the .txt extension, any input file that is not - ending in .gpx or .tcx will be passed to the generic text_parser, so as long as it is text - (i.e. not binary) and it has one entry per line, there is a good chance + ending in .gpx or .tcx will be passed to the generic text_parser, so as long as it is text (i.e. + not binary) and it has one entry per line, there is a good chance it will work. At this point there are 2 modes for parsing text files: - 1) automatic regex: - - this was introduced as a no hassle alternative for just loading a reasonable formatted + 1) Automatic regex: + * This was introduced as a no hassle alternative for just loading a reasonable formatted plaintext file. It only looks for datetime and a latitude/longitude pair. - -it searches for this pattern: yyyy*MM*dd*HH*mm*ss[.sss] (with optional 1-3 digit miliseconds) and - if found, it parses it as the date-time; any non-digit separators can exist between these fields - so both 2026/10/23_12:23:34 and 20261023122334 are found correctly - -it then searches for latitude (and after it, a longitude) with a regex matching 0.1234 to 123.45678910 - (so 1-3 digits followed by a dot then 4+ digits). Optionally, in front it can have NSEW or +- - (see the q_parse_txt() function in gps_parser.cpp for the exact QT regexes used) - 2) manual format line: - - regex is limited and can fail, so if you want extra fields or if your date is non standard - (like month before day) there is another option, you need to manually edit the .txt file and - insert a format string like the one below as the first line. + * It searches for this pattern: yyyy*MM*dd*HH*mm*ss[.sss] (optional 1-3 digit miliseconds) and + if found, it parses it as the date-time; any non-digit separators can exist between these fields, + so both 2026/10/23_12:23:34 and 20261023122334 are found correctly. + * It then searches for latitude (and after it, a longitude) with a regex matching 0.1234 to + 123.45678910 (so 1-3 digits followed by a dot then 4+ digits). Optionally starts with NSEW or +-. + * See the q_parse_txt() function in gps_parser.cpp for the exact QT regexes used). + 2) Manual format line: + * Regex is limited and can fail, so if you want extra fields or if your date is non standard + (like month before day) there is another option: you need to manually edit the .txt file and + insert a format string like the one below as the first line: #delimiter=" " datetime="yyyy/MM/dd" datetime="hh:mm:ss" latitude longitude speed skip heading altitude - -the "#delimiter=" keyword is mandatory at the start, and it will treat everything between the "" as - the delimiter. Common values are space " " and comma "," - -datetime= is a special keyword that takes an argument between "" which will be passed - to QDateTime::fromString, so use the patterns QT accepts - -specially for date and time there are also the "date" and "time" keywords that will attempt - to separately match yyyy*MM*dd and hh*mm*ss*[.ms], "datetime" which will do them together, - and datetimeiso which will match ISO 8601 extended format ("2026-10-23T18:25:20.739" with - optional timezone) - -all supported keywords: datetime= datetimeiso datetime date time - lat lon speed alt ele head bear hr heart cad temp pow - (yes, you can also write latitude and it will match "lat", and yes, alt and ele are the same thing) - -Note: any format/keyword name not recognized will be skipped along with it's corresponding column value, - in my example above the "skip" keyword simply skips over the unit which follows the speed; - -also note that everything after the last keyword is skipped so you don't need to bother with the - entire line in the file + which corresponds to: + 2025/06/12 02:45:01 N:45.123456 E:26.123456 0.0 km/h 0.00 72.50 10 x:-4.13 y:0.27 z:0.49 + * The "#delimiter=" keyword is mandatory at the start, and it will treat everything between the "" + as the delimiter. Common values are space (" ") and comma (","). + * datetime= is a special keyword that takes an argument between "" which will be passed to + QDateTime::fromString, so use the patterns QT accepts. + * For date and time, there are also the "date" and "time" keywords that will attempt to separately + match yyyy*MM*dd and hh*mm*ss*[.ms], "datetime" which will do them together, and datetimeiso which + will match ISO 8601 extended format ("2026-10-23T18:25:20.739" with optional timezone). + * Supported keywords: datetime= datetimeiso datetime date time lat lon speed alt ele head bear hr + heart cad temp pow. + * Note: any format/keyword name not recognized will be skipped along with it's corresponding column + value. In my example above, the "skip" keyword simply skips over the unit which follows the speed. + * Also note that everything after the last keyword is skipped so you don't need to bother with the + entire line in the file. type: string required: yes readonly: no