Program.cs

using System.Diagnostics;
using System.Text.RegularExpressions;


namespace ImportWC
{
	static partial class Program
	{
		public static  Cumulus Cumulus { get; set; } = new Cumulus();
		public static string Location { get; set; } = string.Empty;

		private static ConsoleColor defConsoleColour;

		public static string WcDataPath { get; set; } = string.Empty;
		public static string WcConfigTemp { get; set; } = string.Empty;
		public static string WcConfigDew { get; set; } = string.Empty;
		public static string WcConfigWind { get; set; } = string.Empty;
		public static string WcConfigPress { get; set; } = string.Empty;
		public static string WcConfigRain { get; set; } = string.Empty;

		static readonly Regex TabSplit = TabSplitRegex();

		public static double RainCounter = 0;
		public static double RainToday = 0;

		public static RainCounterAccumulator RainAccumulator { get; private set; } = new RainCounterAccumulator();


		static int Main()
		{
			// Tell the user what is happening

			TextWriterTraceListener myTextListener = new TextWriterTraceListener($"MXdiags{Path.DirectorySeparatorChar}ImportWC-{DateTime.Now:yyyyMMdd-HHmmss}.txt", "WClog");
			Trace.Listeners.Add(myTextListener);
			Trace.AutoFlush = true;

			defConsoleColour = Console.ForegroundColor;

			var fullVer = System.Reflection.Assembly.GetExecutingAssembly().GetName().Version ?? new Version(0, 0, 0, 0);
			var version = $"{fullVer.Major}.{fullVer.Minor}.{fullVer.Build}";
			LogMessage("ImportWC v." + version);
			Console.WriteLine("ImportWC v." + version);

			// Read the config file
			ReadWeatherCatConfig();

			LogMessage("Processing started");
			Console.WriteLine();
			Console.WriteLine($"Processing started: {DateTime.Now:U}");
			Console.WriteLine();

			// get the location of the exe - we will assume this is in the Cumulus root folder
			Location = AppDomain.CurrentDomain.BaseDirectory;

			RainAccumulator = new RainCounterAccumulator();

			// Check meteo day
			if (Cumulus.RolloverHour != 0)
			{
				LogMessage("Cumulus is not configured for a midnight rollover, so Import cannot create any day file entries");
				LogConsole("Cumulus is not configured for a midnight rollover, so no day file entries will be created", ConsoleColor.DarkYellow);
				LogConsole("You must run CreateMissing after this Import to create the day file entries", ConsoleColor.DarkYellow);
			}
			else
			{
				LogMessage("Cumulus is configured for a midnight rollover, Import will create day file entries");
				LogConsole("Cumulus is configured for a midnight rollover, so day file entries will be created", ConsoleColor.Cyan);
				LogConsole("You must still run CreateMissing after this Import to add missing details to those day file entries", ConsoleColor.Cyan);
			}
			Console.WriteLine();

			// Find all the wlk files
			// naming convention: MM_WeatherCatData.cat
			LogMessage("Searching for cat files");
			Console.WriteLine("Searching for cat log files...");

			if (!Directory.Exists(WcDataPath))
			{
				LogMessage($"The source directory '{WcDataPath}' does not exist, aborting");
				LogConsole($"The source directory '{WcDataPath}' does not exist, aborting", ConsoleColor.Red);
				Environment.Exit(1);
			}

			var dirInfo = new DirectoryInfo(WcDataPath);
			var wcFiles = dirInfo.GetFiles("*_WeatherCatData.cat", SearchOption.AllDirectories);

			LogMessage($"Found {wcFiles.Length} cat log files");
			LogConsole($"Found {wcFiles.Length} cat log files", defConsoleColour);

			// sort the file list
			var wcList = wcFiles
				.OrderBy(fi => ExtractYear(fi))
				.ThenBy(fi => ExtractMonth(fi))
				.ToList();

			var year = 0;
			var lastYear = 0;
			var month = 0;

			foreach (var cat in wcList)
			{
				string[] lines;

				try
				{
					lines = File.ReadAllLines(cat.FullName);
				}
				catch (Exception ex)
				{
					LogMessage($"Error opening file {cat.FullName} - {ex.Message}");
					LogConsole($"Error opening file {cat.FullName} - {ex.Message}", ConsoleColor.Red);
					LogConsole("Skipping to next file", defConsoleColour);
					// abort this file
					continue;
				}

				// get the year/month from the filename
				// year is folder name containg the file
				// month is the first 1 or 2 characters of the filename followed by an underscore
				// eg /wc/2024/5_WeatherCatData.cat

				var name = new DirectoryInfo(cat.FullName);

				if (name.Parent == null || !int.TryParse(name.Parent.Name, out year))
				{
					LogMessage($"Error parsing year from {cat.FullName}");
					LogConsole($"Error parsing year from {cat.FullName}", ConsoleColor.Red);
					LogConsole("Skipping to next file", defConsoleColour);
					// abort this file
					continue;
				}

				if (!int.TryParse(cat.Name.Split('_')[0], out month))
				{
					LogMessage($"Error parsing month from {cat.FullName}");
					LogConsole($"Error parsing month from {cat.FullName}", ConsoleColor.Red);
					LogConsole("Skipping to next file", defConsoleColour);
					// abort this file
					continue;
				}

				LogConsole($"Processing {year} {cat.Name}...", ConsoleColor.Gray);
				LogMessage($"Processing {cat.FullName}...");

				LogMessage($"  {year} {cat.Name} contains {lines.Length} lines");


				if (year != lastYear)
				{
					lastYear = year;
				}

				var started = false;
				var fileVersion = 0;

				foreach (var line in lines)
				{
					// we want to start reading data after the line containing "VERS:"

					if (!started && !line.StartsWith("VERS:"))
					{
						continue;
					}
					else
					{
						if (!started)
						{
							fileVersion = int.Parse(line.Split(':')[1]);
							started = true;
							continue;
						}
						else if (line.Length == 0)
						{
							// skip blank lines afer VERS:
							continue;
						}
					}


					var rec = new WeatherCatRecord(year, month, line);

					if (rec.IsValid)
					{
						LogFile.AddRecord(rec);

						if (rec.HasExtraData)
						{
							ExtraLogFile.AddRecord(rec);
						}

						if (rec.HasSynthData)
						{
							CustomLogFile.AddRecord(rec);
						}
					}
				}

				// Now try and find the corresponding hours file
				// Find all the wlk files
				// naming convention MM_WeatherCatData.cat.hrs
				var hrFileName = Path.Combine(cat.DirectoryName!, month + "_WeatherCatData.cat.hrs");
				LogMessage("Searching for corresponding hour file: " + month + "_WeatherCatData.cat.hrs");
				Console.WriteLine($"Searching for corresponding hour file: {month + "_WeatherCatData.cat.hrs"}...");

				if (File.Exists(hrFileName))
				{
					LogConsole($"Processing {year} {month + "_WeatherCatData.cat.hrs"}...", ConsoleColor.Gray);
					LogMessage($"Processing {hrFileName}...");

					UpdateSunshineFromHoursFile(hrFileName, year, month);
				}
				else
				{
					LogConsole($"Error {year} {month + "_WeatherCatData.cat.hrs"} file not found, skipping sunshine hours for this month", ConsoleColor.Red);
					LogMessage($"Error {hrFileName} file not found, skipping sunshine hours for this month");
				}

				// Write out the log file
				if (LogFile.RecordsCount > 0)
				{
					LogFile.WriteLogFile();
					LogFile.Initialise();
				}

				if (ExtraLogFile.RecordsCount > 0)
				{
					ExtraLogFile.WriteLogFile();
					ExtraLogFile.Initialise();
				}

				if (CustomLogFile.RecordsCount > 0)
				{
					CustomLogFile.WriteLogFile();
					CustomLogFile.Initialise();
				}
			}


			return 0;
		}

		private static void UpdateSunshineFromHoursFile(string path, int year, int month)
		{
			// The WeatherCat hours file contains 270+ columns - There is no available header file
			// Each data column is separated by one or more tab characters
			// There is one file per month, contained in the year folder
			// Naming convertion: [MonthNum]_WeatherCatData.cat.hrs
			//                  eg. 8_WeatherCatData.cat.hrs
			//
			// Records
			// 0 - Day of month
			// 1 - Hour of day (0-23)
			// ...
			// 272 - Total sunsine hours for the hour

			// Pre-index the LogFile Records for efficency
			var index = new Dictionary<(int day, int hour), List<LogFileRec>>();

			foreach (var kvp in LogFile.Records)
			{
				var t = kvp.Key;
				var key = (t.Day, t.Hour);

				if (!index.TryGetValue(key, out var list))
				{
					list = new List<LogFileRec>();
					index[key] = list;
				}

				list.Add(kvp.Value);
			}

			double cumulative = 0;
			int? currentDay = null;

			foreach (var line in File.ReadLines(path))
			{
				if (string.IsNullOrWhiteSpace(line))
					continue;

				var cols = TabSplit.Split(line);

				if (cols.Length <= 272)
					continue; // malformed row

				// Extract fields
				int day = int.Parse(cols[0]);
				int hour = int.Parse(cols[1]);

				// Sunshine hours column (may be blank)
				if (!double.TryParse(cols[272], out double hourlySunshine))
					continue;

				// Detect day change
				if (currentDay == null || currentDay != day)
				{
					cumulative = 0;
					currentDay = day;
				}

				// Add this hour's sunshine to the running total
				cumulative += hourlySunshine;


				// Build timestamp (assume local time)
				var hourStamp = new DateTime(year, month, day, hour, 0, 0);
				
				LogFile.UpdateSunshineForHour(day, hour, cumulative, index);
			}
		}

		public static void LogMessage(string message)
		{
			Trace.TraceInformation(DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.fff ") + message);
		}

		public static void LogDebugMessage(string message)
		{
#if DEBUG
			//Trace.TraceInformation(DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.fff ") + message)
#endif
		}

		public static void LogConsole(string msg, ConsoleColor colour, bool newLine = true)
		{
			Console.ForegroundColor = colour;

			if (newLine)
			{
				Console.WriteLine(msg);
			}
			else
			{
				Console.Write(msg);
			}

			Console.ForegroundColor = defConsoleColour;
		}

		private static void ReadWeatherCatConfig()
		{
			if (!System.IO.File.Exists(Program.Location + "wc_config.ini"))
			{
				Program.LogMessage($"Failed to find wc_config.ini file at {Program.Location}wc_config.ini");
				Console.WriteLine($"Failed to find wc_config.ini file at {Program.Location}wc_config.ini");
				Environment.Exit(1);
			}

			Program.LogMessage("Reading wc_config.ini file");

			IniFile ini = new IniFile("wc_config.ini");

			WcDataPath = ini.GetValue("data", "path", "");
			if (WcDataPath == "")
			{
				Program.LogMessage("Failed to find data path in wc_config.ini");
				Console.WriteLine("Failed to find data path in wc_config.ini");
				Environment.Exit(1);
			}

			// Check if the data path exists
			if (!Directory.Exists(WcDataPath))
			{
				Program.LogMessage($"The data path specified in wc_config.ini does not exist - {WcDataPath}");
				Console.WriteLine($"The data path specified in wc_config.ini does not exist - {WcDataPath}");
				Environment.Exit(1);
			}

			WcConfigTemp = ini.GetValue("units", "temperature", "").ToLower();
			if (WcConfigTemp == "" || (WcConfigTemp != "c" && WcConfigTemp != "f"))
			{
				Program.LogMessage("Failed to find temperature units in wc_config.ini");
				Console.WriteLine("Failed to find temperature units in wc_config.ini");
				Environment.Exit(1);
			}

			WcConfigDew = ini.GetValue("units", "dewpoint", "").ToLower();
			if (WcConfigDew == "" || (WcConfigDew != "c" && WcConfigDew != "f"))
			{
				Program.LogMessage("Failed to find dewpoint units in wc_config.ini");
				Console.WriteLine("Failed to find dewpoint units in wc_config.ini");
				Environment.Exit(1);
			}

			WcConfigPress = ini.GetValue("units", "pressure", "").ToLower();
			if (WcConfigPress == "" || (WcConfigPress != "inhg" && WcConfigPress != "mb" && WcConfigPress != "hpa"))
			{
				Program.LogMessage("Failed to find pressure units in wc_config.ini");
				Console.WriteLine("Failed to find pressure units in wc_config.ini");
				Environment.Exit(1);
			}

			WcConfigWind = ini.GetValue("units", "wind", "");
			if (WcConfigWind == "" || (WcConfigWind != "kph" && WcConfigWind != "mps" && WcConfigWind != "mph" && WcConfigWind != "knots"))
			{
				Program.LogMessage("Failed to find wind units in wc_config.ini");
				Console.WriteLine("Failed to find wind units in wc_config.ini");
				Environment.Exit(1);
			}

			WcConfigRain = ini.GetValue("units", "rain", "");
			if (WcConfigRain == "" || (WcConfigRain != "mm" && WcConfigRain != "in"))
			{
				Program.LogMessage("Failed to find rain units in wc_config.ini");
				Console.WriteLine("Failed to find rain units in wc_config.ini");
				Environment.Exit(1);
			}
		}

		private static int ExtractYear(FileInfo fi)
		{
			return int.Parse(fi.Directory.Name);
		}

		private static int ExtractMonth(FileInfo fi)
		{
			var name = fi.Name;
			var monthPart = name.Split('_')[0];
			return int.Parse(monthPart);
		}

		public sealed class RainCounterAccumulator
		{
			public double? LastTotalRainfall { get; private set; }

			// Tune this for your station resolution (e.g., 0.2 mm per tip)
			// current has jumped by more than 20 mm/0.75 inch
			private readonly double MaxPlausibleDelta = Cumulus.Units.Rain == 0 ? 20 : 0.75;

			public void ProcessReading(double totalRainfall)
			{
				if (LastTotalRainfall is not null)
				{
					double delta = totalRainfall - LastTotalRainfall.Value;

					bool plausible = delta >= 0 && delta <= MaxPlausibleDelta;

					if (plausible)
						Program.RainCounter += delta;
				}

				LastTotalRainfall = totalRainfall;
			}
		}


		[GeneratedRegex(@"\t+")]
		private static partial Regex TabSplitRegex();
	}
}