Revize 7a998d66

Re #8691. Big refactor + config file added + working on providing methods for Parser.

Server/ServerApp/Connection/SocketListener.cs

@@ -8 +8 @@
 using System.Text;
 using System.Threading;
 
-namespace Connection
+namespace ServerApp.Connection
 {
     /// <summary>
     /// State object for reading client data asynchronously

Server/ServerApp/DataDownload/DataDownloader.cs

@@ -8 +8 @@
 using System.IO.Compression;
 using System.Net;
 
-namespace DataDownload
+namespace ServerApp.DataDownload
 {
 	/// <summary>
 	/// Enum representing all of the available data types (not all will be used in this project)
@@ -28 +28 @@
 		XML, JSON, CSV
 	}
 
+	public enum TimeSpan
+	{
+
+	}
+
 	/// <summary>
 	/// This class takes care of downloading of data. Download happens from http://openstore.zcu.cz/.
 	/// </summary>
@@ -36 +41 @@
 		/// <summary>
 		/// The root directory containing all downloaded data
 		/// </summary>
-		public string DataDirectory { get; }
+		public string RootDataDirectory { get; }
+
+		/// <summary>
+		/// For a DataType key returns full (absolute) path to a direcotry, where this type of data is stored
+		/// </summary>
+		public Dictionary<DataType, string> DataSubDirectories { get; }
 
 		/// <summary>
 		/// Flag stating whether files which already exist should be overwritten when downloaded again
@@ -55 +65 @@
 		// a shortcut to writing Path.DirectorySeparatorChar
 		private char sep = Path.DirectorySeparatorChar;
 
-		public DataDownloader()
+		public DataDownloader(string rootDataDir, string website, string namingConvention) // todo take naming conventons specifiaction into account
 		{
 			// initialize all needed variables:
 
+			DataSubDirectories = new Dictionary<DataType, string>();
+
 			Console.WriteLine(Directory.GetCurrentDirectory());
-			site = "http://openstore.zcu.cz/";
+			site = website;//"http://openstore.zcu.cz/";
 			dataStr = "OD_ZCU_";
 
-			DataDirectory = $"..{sep}..{sep}data{sep}auto";
+			RootDataDirectory = rootDataDir;//$"..{sep}..{sep}..{sep}data{sep}auto";
 			OverwriteExisting = false;
 
 			webClient = new WebClient();
@@ -89 +101 @@
 			string monthYr = monthStr + "_" + yearStr;
 			string url = site + "/" + dataStr + monthYr + "/" + dataStr + type + "_" + monthYr + "_" + format + ".zip";
 			string nameZip = dataStr + type + "_" + monthYr + "_" + format + ".zip";
-			string nameFolder = DataDirectory + sep + type + sep; //+ dataStr + type + "_" + monthYr + "_" + format;
+			string nameFolder = RootDataDirectory + sep + type + sep; //+ dataStr + type + "_" + monthYr + "_" + format;
 
 			try
 			{
-				Console.WriteLine("Downloading .zip to " + Path.GetFullPath(nameZip) + "...");
+				//Console.WriteLine("Downloading .zip to " + Path.GetFullPath(nameZip) + "...");
 
 				// Download the zip file:
 				webClient.DownloadFile(url, nameZip);
@@ -134 +146 @@
 				// dispose of the archive:
 				zipArchive.Dispose();
 
-				Console.WriteLine("Extracted to " + Path.GetFullPath(nameFolder));
-				Console.WriteLine("Deleting .zip from " + Path.GetFullPath(nameZip) + "...");
-				Console.WriteLine("Finished downloading " + nameZip);
+				//Console.WriteLine("Extracted to " + Path.GetFullPath(nameFolder));
+				//Console.WriteLine("Deleting .zip from " + Path.GetFullPath(nameZip) + "...");
+				//Console.WriteLine("Finished downloading " + nameZip);
 
 				// delete the previously downloaded zip file, files contained in it have been extracted:
 				File.Delete(nameZip); // todo check?
@@ -145 +157 @@
 			catch(System.Net.WebException we)
 			{
 				// download fails, if the specified url is invalid
-				Console.WriteLine("Download from " + url + " failed.");
-				Console.WriteLine(we.Message);
+				//Console.WriteLine("Download from " + url + " failed.");
+				//Console.WriteLine(we.Message);
 			}
 
 
@@ -160 +172 @@
 		/// Downloads selected type and time span of data in the desired format, returns a list of full paths to all successfully saved files. 
 		/// If some of the files already existed and were not overwritten, then the returned List contains paths to these files also.
 		/// </summary>
-		/// <param name="type">The type of data, e.g. the jis data</param>
-		/// <param name="format">The desired format of data files, available are CSV, XML and JSON formats</param>
-		/// <param name="startYear">The start year to start the download from, inclusive</param>
-		/// <param name="endYear">The end year to start the download from, inclusive</param>
-		/// <param name="startMonth">The start month to start the download from, inclusive</param>
-		/// <param name="endMonth">The end month to start the download from, inclusive</param>
-		/// <returns>A list of full paths to all successfully saved files</returns>
-		public List<string> DownloadData(DataType type, DataFormat format, int startYear, int endYear, int startMonth, int endMonth)
+		public List<string> DownloadData(DataType type, DataFormat format, Date startDate, Date endDate/*int startYear, int endYear, int startMonth, int endMonth*/)
 		{
+			if (startDate > endDate)
+				throw new ArgumentException("startDate must be the same as or before the endDate.");
+
+			// initialize:
 			List<string> savedFiles = new List<string>();
-			DirectoryInfo di = Directory.CreateDirectory(DataDirectory + sep + type);
+			string subDirectory = RootDataDirectory + sep + type;
+			DirectoryInfo di = Directory.CreateDirectory(subDirectory);
+
+			// create subdirectory record if it doesn't exist:
+			if (!DataSubDirectories.ContainsKey(type))
+				DataSubDirectories.Add(type, Path.GetFullPath(subDirectory));
 
-			for (int y = startYear; y <= endYear; y++)
+
+			Date currentDate = startDate;
+			bool firstLoop = true;
+			do
 			{
-				for (int m = startMonth; m <= endMonth; m++)
+				Console.WriteLine("current date: " + currentDate);
+				savedFiles.AddRange(DownloadData(type, format, (int)currentDate.Year, (int)currentDate.Month));
+				Date nextDate = currentDate.IncreaseMonthByOne();
+
+				// also try to find the 00 file for each year:
+				if(nextDate.Year > currentDate.Year || firstLoop)
 				{
-					savedFiles.AddRange(DownloadData(type, format, y, m));
+					savedFiles.AddRange(DownloadData(type, format, (int)currentDate.Year, 0));
+					if (firstLoop)
+						firstLoop = false; // so that we don't download the same thing all the time
+					
 				}
-			}
+
+				// assign the increased date to the current date:
+				currentDate = nextDate;
+
+
+			} while (currentDate != endDate);
+
+
+			
+
+			//for (int y = startYear; y <= endYear; y++)
+			//{
+			//	for (int m = startMonth; m <= endMonth; m++)
+			//	{
+			//		savedFiles.AddRange(DownloadData(type, format, y, m));
+			//	}
+			//}
 
 			return savedFiles;
 		}
 
+		public bool CheckForNewData()
+		{
+			throw new NotImplementedException();
+		}
+
+		public List<string> GetData(string subDirectory, int startYear, int endYear, int startMonth, int endMonth)
+		{
+			throw new NotImplementedException();
+		}
+
 
 		#region UNUSED
 		//public string GetDirectoryListingRegexForUrl(string url)

Server/ServerApp/DataDownload/Date.cs

@@ +1 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace ServerApp.DataDownload
+{
+	public class Date
+	{
+		public uint Month { get; }
+		public uint Year { get; }
+		public Date(uint month, uint year)
+		{
+			if (month == 0)
+				throw new ArgumentOutOfRangeException("month", "Month must be positive and not zero.");
+			this.Month = month;
+			this.Year = year;
+		}
+
+
+
+		public Date IncreaseMonthByOne()
+		{
+			uint newMonth = Month;
+			newMonth++;
+			uint newYear = Year;
+			if (newMonth > 12)
+			{
+				// newMonth must be 13
+				newMonth = 1;
+				newYear++;
+			}
+
+			return new Date(newMonth, newYear);
+		}
+
+
+		#region OVERRIDEN METHODS FOR OF THE OBJECT CLASS
+		public override bool Equals(object obj)
+		{
+			if (obj.GetType() != typeof(Date))
+				return false;
+
+			Date other = obj as Date;
+			if (other.Month == this.Month && other.Year == this.Year)
+				return true;
+			return false;
+		}
+
+		public override int GetHashCode()
+		{
+			int hashCode = -994906903;
+			hashCode = hashCode * -1521134295 + Month.GetHashCode();
+			hashCode = hashCode * -1521134295 + Year.GetHashCode();
+			return hashCode;
+		}
+
+		public override string ToString()
+		{
+			string mon = Month > 9 ? $"{Month}" : "0" + Month;
+			return $"{mon}-{Year}";
+		}
+
+		#endregion
+
+		#region OVERLOADED OPERATORS
+		public static bool operator >(Date d1, Date d2)
+		{
+			if (d1.Year > d2.Year)
+				return true;
+			else if (d1.Year < d2.Year)
+				return false;
+			else
+			{
+				// the years are equal
+				if (d1.Month > d2.Month)
+					return true;
+				else return false;
+			}
+
+		}
+
+		public static bool operator <(Date d1, Date d2)
+		{
+			return !(d1 >= d2);
+		}
+
+
+		public static bool operator >=(Date d1, Date d2)
+		{
+			if (d1.Year > d2.Year)
+				return true;
+			else if (d1.Year < d2.Year)
+				return false;
+			else
+			{
+				// the years are equal
+				if (d1.Month > d2.Month)
+					return true;
+				else if (d1.Month == d2.Month)
+					return true;
+				else return false;
+			}
+		}
+
+		public static bool operator <=(Date d1, Date d2)
+		{
+			return !(d1 > d2);
+		}
+
+		public static bool operator ==(Date d1, Date d2)
+		{
+			return d1.Equals(d2);
+		}
+
+		public static bool operator !=(Date d1, Date d2)
+		{
+			return !d1.Equals(d2);
+		}
+
+		#endregion
+	}
+}

Server/ServerApp/Program.cs

@@ -1 @@
-using Connection;
+using ServerApp.Connection;
+using ServerApp.DataDownload;
 using ServerApp.Parser.Parsers;
 using ServerApp.Predictor;
 using System;
 using System.Collections.Generic;
+using System.IO;
 
 namespace ServerApp
 {
+
+	class Config // TBD where this should go
+	{
+		public string DataWebsite { get; set; }
+		public string DownloadedFilesNaming { get; set; }
+		public string DataRootDir { get; set; }
+		public string Port { get; set; }
+	}
+
     class Program
     {
+
+
+
         static void Main(string[] args)
         {
-            DataParser p = new DataParser("data/");
 
-            //p.Parse();
+            Config config = FillConfigInfo(args);
+			if (config == null)
+			{
+				Console.ReadLine();
+				return;
+			}
+
+
+
+			//DataParser p = new DataParser("data/");
+
+			//p.Parse();
+
+
+			// test scenario - data download:
+			DataDownloader dd = new DataDownloader(config.DataRootDir, config.DataWebsite, config.DownloadedFilesNaming);
+			dd.OverwriteExisting = false;
+			List<string> savedFiles = new List<string>();
+			savedFiles.AddRange(dd.DownloadData(DataType.JIS, DataFormat.CSV, new Date(1, 2017), new Date(12, 2020)));
+			savedFiles.AddRange(dd.DownloadData(DataType.STROJE, DataFormat.CSV, new Date(1, 2017), new Date(12, 2020)));
+			savedFiles.AddRange(dd.DownloadData(DataType.POCASI, DataFormat.CSV, new Date(1, 2017), new Date(12, 2020)));
+
 
 
-            // test scenario - data download:
-            //DataDownloader dd = new DataDownloader();
-            //         List<string> savedFiles = new List<string>();
-            //         savedFiles.AddRange(dd.DownloadData(DataType.JIS, DataFormat.CSV, 2017, 2021, 0, 13));
-            //         savedFiles.AddRange(dd.DownloadData(DataType.POCASI, DataFormat.CSV, 2017, 2021, 0, 13));
-            //         savedFiles.AddRange(dd.DownloadData(DataType.STROJE, DataFormat.CSV, 2017, 2021, 0, 13));
+			Console.WriteLine("Saved files: ");
+			foreach (string s in savedFiles)
+			{
+				Console.WriteLine(s);
+			}
 
+			Console.WriteLine("subdirectories: ");
+			foreach (KeyValuePair<DataType, string> kvp in dd.DataSubDirectories)
+			{
+				Console.WriteLine("Key = {0}, Value = {1}", kvp.Key, kvp.Value);
+			}
 
-            //         Console.WriteLine("Saved files: ");
-            //         foreach(string s in savedFiles)
-            //{
-            //             Console.WriteLine(s);
-            //}
 
 
+			// date testing
+			//Date d1 = new Date(2, 2020);
+			//Date d2 = new Date(1, 2019);
 
-            // test - connection:
-            AsynchronousSocketListener asl = new AsynchronousSocketListener();
-            asl.StartListening();
-           
+			//Console.WriteLine("equals" + d1.Equals(d2));
+			//Console.WriteLine("==" + (d1 == d2));
+			//Console.WriteLine("!=" + (d1 != d2));
+			//Console.WriteLine(">" + (d1 > d2));
+			//Console.WriteLine("<" + (d1 < d2));
+			//Console.WriteLine(">=" + (d1 >= d2));
+			//Console.WriteLine("<=" + (d1 <= d2));
 
-            NaiveBayesClassifier naiveBayesClassifier = new NaiveBayesClassifier();
-            IEnumerable<ModelInput> modelInput = naiveBayesClassifier.ExtractModelInput(p.weatherList, p.jisList);
-            naiveBayesClassifier.Fit(modelInput);
-            List<ModelInput> dataList = new List<ModelInput>()
-            {
-                new ModelInput()
-                {
-                    Temp = -40,
-                }
-            };
-            var result = naiveBayesClassifier.Predict(dataList);
 
-            Console.WriteLine($"Predictions: ");
-            foreach(var item in result)
-            {
-                Console.WriteLine(item.ToString());
-            }
 
-            Console.ReadLine();
+
+			// test - connection:
+			//AsynchronousSocketListener asl = new AsynchronousSocketListener();
+			//         asl.StartListening();
+
+
+			//NaiveBayesClassifier naiveBayesClassifier = new NaiveBayesClassifier();
+			//IEnumerable<ModelInput> modelInput = naiveBayesClassifier.ExtractModelInput(p.weatherList, p.jisList);
+			//naiveBayesClassifier.Fit(modelInput);
+			//List<ModelInput> dataList = new List<ModelInput>()
+			//{
+			//    new ModelInput()
+			//    {
+			//        Temp = -40,
+			//    }
+			//};
+			//var result = naiveBayesClassifier.Predict(dataList);
+
+			//Console.WriteLine($"Predictions: ");
+			//foreach(var item in result)
+			//{
+			//    Console.WriteLine(item.ToString());
+			//}
+
+			Console.ReadLine();
         }
+
+
+        private static Config FillConfigInfo(string[] args)
+		{
+
+			Config extractedConfigInfo = new Config();
+
+			Console.WriteLine(Directory.GetCurrentDirectory());
+			
+			if (args.Length != 1)
+				Console.WriteLine("Wrong usage of parameters, pass the path to a config file."); // todo better explanation?
+
+			string fullPathConfig = Path.GetFullPath(args[0]);
+			string[] lines = null;
+			try {
+				lines = File.ReadAllLines(fullPathConfig);
+			}
+			catch(Exception ex)
+			{
+				Console.WriteLine("Could not open " + fullPathConfig);
+				return null;
+			}
+			
+			for (var i = 0; i < lines.Length; i += 1)
+			{
+				string line = lines[i];
+				Console.WriteLine(line);
+				if (line.Length == 0 || line == null || line.StartsWith("#"))
+					continue;
+
+				switch (line)
+				{
+					case "!site!":
+						extractedConfigInfo.DataWebsite = lines[++i].Trim();
+						break;
+					case "!naming_convention!":
+						extractedConfigInfo.DownloadedFilesNaming = lines[++i].Trim();
+						break;
+					case "!data_root_dir!":
+						string dirWithConfig = Path.GetDirectoryName(fullPathConfig);
+						string rootPath = dirWithConfig + Path.DirectorySeparatorChar + lines[++i].Trim();
+						extractedConfigInfo.DataRootDir = rootPath;
+						
+						break;
+					case "!port!":
+						extractedConfigInfo.Port = lines[++i].Trim();
+						break;
+					default: break;
+				}
+			}
+
+			return extractedConfigInfo;
+		}
     }
 }

Server/ServerApp/ServerApp.csproj

@@ -131 +131 @@
   <ItemGroup>
     <Compile Include="Connection\SocketListener.cs" />
     <Compile Include="DataDownload\DataDownloader.cs" />
+    <Compile Include="DataDownload\Date.cs" />
     <Compile Include="Parser\InputData\CsvDataLoader.cs" />
     <Compile Include="Parser\InputData\DataLoader.cs" />
     <Compile Include="Parser\InputData\JisInstance.cs" />
@@ -160 +161 @@
   </ItemGroup>
   <ItemGroup>
     <None Include="App.config" />
-    <None Include="data\auto\JIS\OD_ZCU_JIS_00_2019.CSV" />
-    <None Include="data\auto\JIS\OD_ZCU_JIS_01_2020.CSV" />
-    <None Include="data\auto\JIS\OD_ZCU_JIS_02_2020.CSV" />
-    <None Include="data\auto\JIS\OD_ZCU_JIS_03_2020.CSV" />
-    <None Include="data\auto\JIS\OD_ZCU_JIS_04_2020.CSV" />
-    <None Include="data\auto\JIS\OD_ZCU_JIS_05_2020.CSV" />
-    <None Include="data\auto\JIS\OD_ZCU_JIS_06_2019.CSV" />
-    <None Include="data\auto\JIS\OD_ZCU_JIS_06_2020.CSV" />
-    <None Include="data\auto\JIS\OD_ZCU_JIS_07_2019.CSV" />
-    <None Include="data\auto\JIS\OD_ZCU_JIS_07_2020.CSV" />
-    <None Include="data\auto\JIS\OD_ZCU_JIS_08_2019.CSV" />
-    <None Include="data\auto\JIS\OD_ZCU_JIS_08_2020.CSV" />
-    <None Include="data\auto\JIS\OD_ZCU_JIS_09_2019.CSV" />
-    <None Include="data\auto\JIS\OD_ZCU_JIS_09_2020.CSV" />
-    <None Include="data\auto\JIS\OD_ZCU_JIS_10_2019.CSV" />
-    <None Include="data\auto\JIS\OD_ZCU_JIS_10_2020.CSV" />
-    <None Include="data\auto\JIS\OD_ZCU_JIS_11_2019.CSV" />
-    <None Include="data\auto\JIS\OD_ZCU_JIS_11_2020.CSV" />
-    <None Include="data\auto\JIS\OD_ZCU_JIS_12_2019.CSV" />
-    <None Include="data\auto\JIS\OD_ZCU_JIS_12_2020.CSV" />
-    <None Include="data\auto\POCASI\OD_ZCU_POCASI_00_2019.CSV" />
-    <None Include="data\auto\POCASI\OD_ZCU_POCASI_01_2020.CSV" />
-    <None Include="data\auto\POCASI\OD_ZCU_POCASI_02_2020.CSV" />
-    <None Include="data\auto\POCASI\OD_ZCU_POCASI_03_2020.CSV" />
-    <None Include="data\auto\POCASI\OD_ZCU_POCASI_04_2020.CSV" />
-    <None Include="data\auto\POCASI\OD_ZCU_POCASI_05_2020.CSV" />
-    <None Include="data\auto\POCASI\OD_ZCU_POCASI_06_2019.CSV" />
-    <None Include="data\auto\POCASI\OD_ZCU_POCASI_06_2020.CSV" />
-    <None Include="data\auto\POCASI\OD_ZCU_POCASI_07_2019.CSV" />
-    <None Include="data\auto\POCASI\OD_ZCU_POCASI_07_2020.CSV" />
-    <None Include="data\auto\POCASI\OD_ZCU_POCASI_08_2019.CSV" />
-    <None Include="data\auto\POCASI\OD_ZCU_POCASI_08_2020.CSV" />
-    <None Include="data\auto\POCASI\OD_ZCU_POCASI_09_2019.CSV" />
-    <None Include="data\auto\POCASI\OD_ZCU_POCASI_09_2020.CSV" />
-    <None Include="data\auto\POCASI\OD_ZCU_POCASI_10_2019.CSV" />
-    <None Include="data\auto\POCASI\OD_ZCU_POCASI_10_2020.CSV" />
-    <None Include="data\auto\POCASI\OD_ZCU_POCASI_11_2019.CSV" />
-    <None Include="data\auto\POCASI\OD_ZCU_POCASI_11_2020.CSV" />
-    <None Include="data\auto\POCASI\OD_ZCU_POCASI_12_2019.CSV" />
-    <None Include="data\auto\POCASI\OD_ZCU_POCASI_12_2020.CSV" />
-    <None Include="data\auto\STROJE\OD_ZCU_STROJE_00_2019.CSV" />
-    <None Include="data\auto\STROJE\OD_ZCU_STROJE_01_2020.CSV" />
-    <None Include="data\auto\STROJE\OD_ZCU_STROJE_02_2020.CSV" />
-    <None Include="data\auto\STROJE\OD_ZCU_STROJE_03_2020.CSV" />
-    <None Include="data\auto\STROJE\OD_ZCU_STROJE_04_2020.CSV" />
-    <None Include="data\auto\STROJE\OD_ZCU_STROJE_05_2020.CSV" />
-    <None Include="data\auto\STROJE\OD_ZCU_STROJE_06_2019.CSV" />
-    <None Include="data\auto\STROJE\OD_ZCU_STROJE_06_2020.CSV" />
-    <None Include="data\auto\STROJE\OD_ZCU_STROJE_07_2019.CSV" />
-    <None Include="data\auto\STROJE\OD_ZCU_STROJE_07_2020.CSV" />
-    <None Include="data\auto\STROJE\OD_ZCU_STROJE_08_2019.CSV" />
-    <None Include="data\auto\STROJE\OD_ZCU_STROJE_08_2020.CSV" />
-    <None Include="data\auto\STROJE\OD_ZCU_STROJE_09_2019.CSV" />
-    <None Include="data\auto\STROJE\OD_ZCU_STROJE_09_2020.CSV" />
-    <None Include="data\auto\STROJE\OD_ZCU_STROJE_10_2019.CSV" />
-    <None Include="data\auto\STROJE\OD_ZCU_STROJE_10_2020.CSV" />
-    <None Include="data\auto\STROJE\OD_ZCU_STROJE_11_2019.CSV" />
-    <None Include="data\auto\STROJE\OD_ZCU_STROJE_11_2020.CSV" />
-    <None Include="data\auto\STROJE\OD_ZCU_STROJE_12_2019.CSV" />
-    <None Include="data\auto\STROJE\OD_ZCU_STROJE_12_2020.CSV" />
     <None Include="data\jis\OD_ZCU_JIS_06_2019.CSV" />
     <None Include="data\jis\OD_ZCU_JIS_09_2019.CSV" />
     <None Include="data\jis\OD_ZCU_JIS_10_2019.CSV" />
@@ -240 +181 @@
     <None Include="data\weather\OD_ZCU_POCASI_13_2019.CSV" />
     <None Include="packages.config" />
   </ItemGroup>
+  <ItemGroup>
+    <Folder Include="data\auto\" />
+  </ItemGroup>
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
   <Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
     <PropertyGroup>

Server/ServerApp/server_config

@@ +1 @@
+
+#
+# This is the server configuration file. See [wiki_site] for detailed information about its structure.
+# Pass this file as an argument to the server program when launching it.
+#
+
+# --- DATA DOWNLOADING CONFIGURATION ---
+
+# the main website containing all data:
+!site!
+http://openstore.zcu.cz/
+
+# the naming convention for files stored at the site to download from:
+!naming_convention!
+OD_ZCU_{type}_{month}_{year}_{format}.zip
+
+# the root directory where all data will be stored (subdirectories will be created when data is downloaded)
+# enter path relative to this file or an absolute path
+!data_root_dir!
+.\data\auto
+
+
+# --- CONNECTION CONFIGURATION ---
+
+# port configuration
+!port!
+10000