Revize 7a998d66

Re #8691. Big refactor + config file added + working on providing methods for Parser.

Server/ServerApp/DataDownload/DataDownloader.cs

@@ -8 +8 @@
 using System.IO.Compression;
 using System.Net;
 
-namespace DataDownload
+namespace ServerApp.DataDownload
 {
 	/// <summary>
 	/// Enum representing all of the available data types (not all will be used in this project)
@@ -28 +28 @@
 		XML, JSON, CSV
 	}
 
+	public enum TimeSpan
+	{
+
+	}
+
 	/// <summary>
 	/// This class takes care of downloading of data. Download happens from http://openstore.zcu.cz/.
 	/// </summary>
@@ -36 +41 @@
 		/// <summary>
 		/// The root directory containing all downloaded data
 		/// </summary>
-		public string DataDirectory { get; }
+		public string RootDataDirectory { get; }
+
+		/// <summary>
+		/// For a DataType key returns full (absolute) path to a direcotry, where this type of data is stored
+		/// </summary>
+		public Dictionary<DataType, string> DataSubDirectories { get; }
 
 		/// <summary>
 		/// Flag stating whether files which already exist should be overwritten when downloaded again
@@ -55 +65 @@
 		// a shortcut to writing Path.DirectorySeparatorChar
 		private char sep = Path.DirectorySeparatorChar;
 
-		public DataDownloader()
+		public DataDownloader(string rootDataDir, string website, string namingConvention) // todo take naming conventons specifiaction into account
 		{
 			// initialize all needed variables:
 
+			DataSubDirectories = new Dictionary<DataType, string>();
+
 			Console.WriteLine(Directory.GetCurrentDirectory());
-			site = "http://openstore.zcu.cz/";
+			site = website;//"http://openstore.zcu.cz/";
 			dataStr = "OD_ZCU_";
 
-			DataDirectory = $"..{sep}..{sep}data{sep}auto";
+			RootDataDirectory = rootDataDir;//$"..{sep}..{sep}..{sep}data{sep}auto";
 			OverwriteExisting = false;
 
 			webClient = new WebClient();
@@ -89 +101 @@
 			string monthYr = monthStr + "_" + yearStr;
 			string url = site + "/" + dataStr + monthYr + "/" + dataStr + type + "_" + monthYr + "_" + format + ".zip";
 			string nameZip = dataStr + type + "_" + monthYr + "_" + format + ".zip";
-			string nameFolder = DataDirectory + sep + type + sep; //+ dataStr + type + "_" + monthYr + "_" + format;
+			string nameFolder = RootDataDirectory + sep + type + sep; //+ dataStr + type + "_" + monthYr + "_" + format;
 
 			try
 			{
-				Console.WriteLine("Downloading .zip to " + Path.GetFullPath(nameZip) + "...");
+				//Console.WriteLine("Downloading .zip to " + Path.GetFullPath(nameZip) + "...");
 
 				// Download the zip file:
 				webClient.DownloadFile(url, nameZip);
@@ -134 +146 @@
 				// dispose of the archive:
 				zipArchive.Dispose();
 
-				Console.WriteLine("Extracted to " + Path.GetFullPath(nameFolder));
-				Console.WriteLine("Deleting .zip from " + Path.GetFullPath(nameZip) + "...");
-				Console.WriteLine("Finished downloading " + nameZip);
+				//Console.WriteLine("Extracted to " + Path.GetFullPath(nameFolder));
+				//Console.WriteLine("Deleting .zip from " + Path.GetFullPath(nameZip) + "...");
+				//Console.WriteLine("Finished downloading " + nameZip);
 
 				// delete the previously downloaded zip file, files contained in it have been extracted:
 				File.Delete(nameZip); // todo check?
@@ -145 +157 @@
 			catch(System.Net.WebException we)
 			{
 				// download fails, if the specified url is invalid
-				Console.WriteLine("Download from " + url + " failed.");
-				Console.WriteLine(we.Message);
+				//Console.WriteLine("Download from " + url + " failed.");
+				//Console.WriteLine(we.Message);
 			}
 
 
@@ -160 +172 @@
 		/// Downloads selected type and time span of data in the desired format, returns a list of full paths to all successfully saved files. 
 		/// If some of the files already existed and were not overwritten, then the returned List contains paths to these files also.
 		/// </summary>
-		/// <param name="type">The type of data, e.g. the jis data</param>
-		/// <param name="format">The desired format of data files, available are CSV, XML and JSON formats</param>
-		/// <param name="startYear">The start year to start the download from, inclusive</param>
-		/// <param name="endYear">The end year to start the download from, inclusive</param>
-		/// <param name="startMonth">The start month to start the download from, inclusive</param>
-		/// <param name="endMonth">The end month to start the download from, inclusive</param>
-		/// <returns>A list of full paths to all successfully saved files</returns>
-		public List<string> DownloadData(DataType type, DataFormat format, int startYear, int endYear, int startMonth, int endMonth)
+		public List<string> DownloadData(DataType type, DataFormat format, Date startDate, Date endDate/*int startYear, int endYear, int startMonth, int endMonth*/)
 		{
+			if (startDate > endDate)
+				throw new ArgumentException("startDate must be the same as or before the endDate.");
+
+			// initialize:
 			List<string> savedFiles = new List<string>();
-			DirectoryInfo di = Directory.CreateDirectory(DataDirectory + sep + type);
+			string subDirectory = RootDataDirectory + sep + type;
+			DirectoryInfo di = Directory.CreateDirectory(subDirectory);
+
+			// create subdirectory record if it doesn't exist:
+			if (!DataSubDirectories.ContainsKey(type))
+				DataSubDirectories.Add(type, Path.GetFullPath(subDirectory));
 
-			for (int y = startYear; y <= endYear; y++)
+
+			Date currentDate = startDate;
+			bool firstLoop = true;
+			do
 			{
-				for (int m = startMonth; m <= endMonth; m++)
+				Console.WriteLine("current date: " + currentDate);
+				savedFiles.AddRange(DownloadData(type, format, (int)currentDate.Year, (int)currentDate.Month));
+				Date nextDate = currentDate.IncreaseMonthByOne();
+
+				// also try to find the 00 file for each year:
+				if(nextDate.Year > currentDate.Year || firstLoop)
 				{
-					savedFiles.AddRange(DownloadData(type, format, y, m));
+					savedFiles.AddRange(DownloadData(type, format, (int)currentDate.Year, 0));
+					if (firstLoop)
+						firstLoop = false; // so that we don't download the same thing all the time
+					
 				}
-			}
+
+				// assign the increased date to the current date:
+				currentDate = nextDate;
+
+
+			} while (currentDate != endDate);
+
+
+			
+
+			//for (int y = startYear; y <= endYear; y++)
+			//{
+			//	for (int m = startMonth; m <= endMonth; m++)
+			//	{
+			//		savedFiles.AddRange(DownloadData(type, format, y, m));
+			//	}
+			//}
 
 			return savedFiles;
 		}
 
+		public bool CheckForNewData()
+		{
+			throw new NotImplementedException();
+		}
+
+		public List<string> GetData(string subDirectory, int startYear, int endYear, int startMonth, int endMonth)
+		{
+			throw new NotImplementedException();
+		}
+
 
 		#region UNUSED
 		//public string GetDirectoryListingRegexForUrl(string url)