aswi2021tri-musketyri-gitlab/Server/ServerApp/DataDownload/DataDownloader.cs @ a53b1de8

//
// Author: Eliska Mourycova
//

using System;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Net;

namespace ServerApp.DataDownload
{
	/// <summary>
	/// Enum representing all of the available data types (not all will be used in this project)
	/// They are in Czech for easier handling file names.
	/// TBD: They might be translated to English later.
	/// </summary>
	public enum DataType
	{
		POCASI, ENERGO, STROJE, EMAIL, OBSAZENI_MISTNOSTI, JIS, KOLOBEZKY, WIFI
	}

	/// <summary>
	/// Represent all available data formats.
	/// </summary>
	public enum DataFormat
	{
		XML, JSON, CSV
	}

	/// <summary>
	/// This class takes care of downloading of data. Download happens from http://openstore.zcu.cz/.
	/// </summary>
	public class DataDownloader
	{
		/// <summary>
		/// The root directory containing all downloaded data
		/// </summary>
		public string RootDataDirectory { get; }

		/// <summary>
		/// For a DataType key returns full (absolute) path to a direcotry, where this type of data is stored
		/// </summary>
		public Dictionary<DataType, string> DataSubDirectories { get; }

		/// <summary>
		/// Flag stating whether files which already exist should be overwritten when downloaded again
		/// </summary>
		public bool OverwriteExisting { get; set; }

		// the main site where the data can be downloaded from
		private string site;

		// the substring at the start of every file name
		private string dataStr;

		// WebClient instance used for the actual download
		private WebClient webClient;

		// a shortcut to writing Path.DirectorySeparatorChar
		private char sep = Path.DirectorySeparatorChar;

		// lists used for parsing the file names:
		private List<string> separatedFileName;
		private List<string> variablesInsertions;
		private List<char> nameTurns;

		public DataDownloader(string rootDataDir, string website, string namingConvention) // todo: take naming conventons specifiaction into account
		{
			// initialize all needed variables:

			//Console.WriteLine(Directory.GetCurrentDirectory());
			site = website;//"http://openstore.zcu.cz/";

			ParseNaming(namingConvention);
			dataStr = "OD_ZCU_";

			RootDataDirectory = rootDataDir;//$"..{sep}..{sep}..{sep}data{sep}auto";
			OverwriteExisting = false;

			DataSubDirectories = new Dictionary<DataType, string>();
			CreateSubdirectories(); // todo should we do it like this?

			webClient = new WebClient();
		}

		/// <summary>
		/// Creates subdirectories for all data types
		/// TBD if we want to do it this way
		/// </summary>
		private void CreateSubdirectories()
		{
			foreach (DataType type in (DataType[])Enum.GetValues(typeof(DataType)))
			{
				string subDirectory = RootDataDirectory + sep + type;
				DirectoryInfo di = Directory.CreateDirectory(subDirectory);

				// create subdirectory record if it doesn't exist:
				if (!DataSubDirectories.ContainsKey(type))
					DataSubDirectories.Add(type, Path.GetFullPath(subDirectory));
			}
		}

		/// <summary>
		/// Parses the naming convention to be later used for determining URLs for data download
		/// </summary>
		/// <param name="namingConvention">The configured naming convention</param>
		private void ParseNaming(string namingConvention)
		{
			separatedFileName = new List<string>();
			variablesInsertions = new List<string>();
			nameTurns = new List<char>();

			string currPart = "";
			string currVar = "";
			bool readingNormal = true;
			foreach (char c in namingConvention)
			{
				if (c == '{')
				{
					AddToNameParts(currPart);
					readingNormal = false;
					currPart = "";
				}
				else if (c == '}')
				{
					AddToVariables(currVar);
					readingNormal = true;
					currVar = "";
				}
				else
				{
					// normal char
					if (readingNormal)
						currPart += c;
					else
						currVar += c;
				}
			}

			// add the rest if there is any:
			if (readingNormal)
				AddToNameParts(currPart);
			else
				AddToVariables(currVar);

			Console.WriteLine();
		}

		// Adds to name parts
		private void AddToNameParts(string s)
		{
			if (s.Length > 0)
			{
				separatedFileName.Add(s);
				nameTurns.Add('n');
			}
				
		}

		// Adds to variable name parts
		private void AddToVariables(string s)
		{
			if (s.Length > 0)
			{
				variablesInsertions.Add(s);
				nameTurns.Add('v');
			}
				
		}

		/// <summary>
		/// Builds the name of the downloaded file. Takes naming convention into account
		/// </summary>
		/// <param name="type">The type of data</param>
		/// <param name="format">The data format</param>
		/// <param name="year">The year</param>
		/// <param name="month">The month</param>
		/// <returns></returns>
		private string BuildDownloadedName(DataType type, DataFormat format, int year, int month)
		{
			string nameZip = "";

			int partInd = 0;
			int varInd = 0;
			for(int i = 0; i < nameTurns.Count; i++)
			{
				if (nameTurns[i] == 'n')
				{
					nameZip += separatedFileName[partInd];
					partInd++;
				}
				else if(nameTurns[i] == 'v')
				{
					string add = "";
					switch (variablesInsertions[varInd])
					{
						case "type":
							add = "" + type;
							break;
						case "month": 
							add = month < 10 ? "0" + month : "" + month;
							break;
						case "year":
							add = "" + year;
							break;
						case "format":
							add = "" + format;
							break;
						default: throw new Exception("Config file error - naming conventions can only contain variables with following names: type, month, year, format");
					}
					nameZip += add;
					varInd++;
				}
			}

			return nameZip;
		}


		/// <summary>
		/// Downloads a specific archive.
		/// </summary>
		/// <param name="type">The type of data</param>
		/// <param name="format">The format of the data</param>
		/// <param name="year">The year</param>
		/// <param name="month">The month</param>
		/// <returns>A list of all extracted file names (should be only one)</returns>
		private List<string> DownloadData(DataType type, DataFormat format, int year, int month)
		{
			// the list of all files potentially relevant to the caller
			List<string> extractedFiles = new List<string>();

			// Prepare the url string to be downloaded from:
			string monthStr = month < 10 ? "0" + month : "" + month;
			string yearStr = "" + year;
			string monthYr = monthStr + "_" + yearStr;


			string nameZip = BuildDownloadedName(type, format, year, month);//dataStr + type + "_" + monthYr + "_" + format + ".zip";
			string url = site + "/" + dataStr + monthYr + "/" + nameZip;//+ dataStr + type + "_" + monthYr + "_" + format + ".zip";
			
			string nameFolder = RootDataDirectory + sep + type + sep; //+ dataStr + type + "_" + monthYr + "_" + format;

			try
			{
				//Console.WriteLine("Downloading .zip to " + Path.GetFullPath(nameZip) + "...");

				// Download the zip file:
				webClient.DownloadFile(url, nameZip);

				//ZipFile.ExtractToDirectory(nameZip, nameFolder);
				ZipArchive zipArchive = ZipFile.OpenRead(nameZip);
				// Go through all the extracted files:
				foreach (ZipArchiveEntry entry in zipArchive.Entries)
				{
					// get the relative path to the file:
					string newFileName = $"{month}-{year}.{format}";
					string extractedFile = nameFolder + newFileName; //+ entry.Name;

					// add full path to the list:
					extractedFiles.Add(Path.GetFullPath(extractedFile));

					if (OverwriteExisting)
					{
						// if overwrite is desired, execute it:
						entry.ExtractToFile(extractedFile, OverwriteExisting);
						
					}
					else
					{
						// if overwrite is not desired, check if the file exists first:
						if(File.Exists(extractedFile/*nameFolder + entry.Name*/))
						{
							continue;
						}
						else
						{
							// if it doesn't exist, save it:
							entry.ExtractToFile(extractedFile, OverwriteExisting);
						}
					}

					
				}
				// dispose of the archive:
				zipArchive.Dispose();

				//Console.WriteLine("Extracted to " + Path.GetFullPath(nameFolder));
				//Console.WriteLine("Deleting .zip from " + Path.GetFullPath(nameZip) + "...");
				//Console.WriteLine("Finished downloading " + nameZip);

				// delete the previously downloaded zip file, files contained in it have been extracted:
				File.Delete(nameZip); // todo check?

			}
			catch(System.Net.WebException we)
			{
				// download fails, if the specified url is invalid
				//Console.WriteLine("Download from " + url + " failed.");
				//Console.WriteLine(we.Message);
			}



			return extractedFiles;
		}



		/// <summary>
		/// Downloads selected type and time span of data in the desired format, returns a list of full paths to all successfully saved files. 
		/// If some of the files already existed and were not overwritten, then the returned List contains paths to these files also.
		/// </summary>
		/// <param name="type">The requested data type</param>
		/// <param name="format">The data format</param>
		/// <param name="startDate">The start date</param>
		/// <param name="endDate">The end date</param>
		/// <returns></returns>
		public List<string> DownloadData(DataType type, DataFormat format, Date startDate, Date endDate/*int startYear, int endYear, int startMonth, int endMonth*/)
		{
			if (startDate > endDate)
				throw new ArgumentException("startDate must be the same as or before the endDate.");

			// initialize:
			List<string> savedFiles = new List<string>();
			//string subDirectory = RootDataDirectory + sep + type;
			//DirectoryInfo di = Directory.CreateDirectory(subDirectory);

			//// create subdirectory record if it doesn't exist:
			//if (!DataSubDirectories.ContainsKey(type))
			//	DataSubDirectories.Add(type, Path.GetFullPath(subDirectory));


			Date currentDate = startDate;
			bool firstLoop = true;
			do
			{
				Console.WriteLine("current date: " + currentDate);
				savedFiles.AddRange(DownloadData(type, format, (int)currentDate.Year, (int)currentDate.Month));
				Date nextDate = currentDate.IncreaseMonthByOne();

				// also try to find the 00 file for each year:
				if(nextDate.Year > currentDate.Year || firstLoop)
				{
					savedFiles.AddRange(DownloadData(type, format, (int)currentDate.Year, 0));
					if (firstLoop)
						firstLoop = false; // so that we don't download the same thing all the time
					
				}

				// assign the increased date to the current date:
				currentDate = nextDate;


			} while (currentDate <= endDate);


			

			//for (int y = startYear; y <= endYear; y++)
			//{
			//	for (int m = startMonth; m <= endMonth; m++)
			//	{
			//		savedFiles.AddRange(DownloadData(type, format, y, m));
			//	}
			//}

			return savedFiles;
		}

		public bool CheckForNewData()
		{
			throw new NotImplementedException();
		}


		/// <summary>
		/// Retrieves all data files with dates falling within the specified range. If not all data for the specified range is found
		/// then returns also file/s with month 0 if exists.
		/// </summary>
		/// <param name="subDirectory">The subdirectory to search</param>
		/// <param name="startDate">The start date</param>
		/// <param name="endDate">The end date</param>
		/// <returns></returns>
		public List<string> GetData(string subDirectory, Date startDate, Date endDate)
		{
			if (startDate == null || endDate == null)
				return GetData(subDirectory);

			string[] files = Directory.GetFiles(subDirectory);
			List<string> found00Files = new List<string>();
			List<string> relevantFiles = new List<string>();
			List<Date> requestedDates = new List<Date>();

			// prepare a list of requested dates:
			Date currentDate = startDate;
			do
			{
				requestedDates.Add(currentDate);
				Date nextDate = currentDate.IncreaseMonthByOne();
				// assign the increased date to the current date:
				currentDate = nextDate;

			} while (currentDate <= endDate);



			for (int i = 0; i < files.Length; i++)
			{
				string currFileName = Path.GetFileName(files[i]);
				Console.WriteLine("curr file: " + currFileName);
				string[] splits = currFileName.Split(new char[] { '-', '.' });

				int month = int.Parse(splits[0]);
				int year = int.Parse(splits[1]);


				
				if (month == 0)
				{
					found00Files.Add(files[i]);
					continue;
				}

				Date d = new Date((uint)month, (uint)year);

				if (d >= startDate && d <= endDate)
				{
					// we want to add this
					relevantFiles.Add(files[i]);
					requestedDates.Remove(d);
				}
			}




			// 00 will only appear once for every year?
			foreach (string file00 in found00Files)
			{
				string fileName = Path.GetFileName(file00);
				string[] splits = fileName.Split(new char[] { '-', '.' });

				int month = int.Parse(splits[0]);
				int year = int.Parse(splits[1]);

				// now we have the year of one 00 file
				// dates not found in the directory now remain in requested dates
				if(requestedDates.Exists(d => d.Year == year))
				{
					// if the year of this 00 file remains in the list:
					relevantFiles.Add(file00);
				}
			}


			return relevantFiles;
		}

		/// <summary>
		/// Returns all file paths in a given directory
		/// </summary>
		/// <param name="subDirectory"></param>
		/// <returns></returns>
		private List<string> GetData(string subDirectory)
		{
			string[] files = Directory.GetFiles(subDirectory);
			List<string> relevantFiles = new List<string>(files.Length);

			for (int i = 0; i < files.Length; i++)
				relevantFiles.Add(files[i]);

			return relevantFiles;
		}


		#region UNUSED
		//public string GetDirectoryListingRegexForUrl(string url)
		//{
		//	if (url.Equals(site))
		//	{
		//		//return "\\\"([^\"]*)\\\"";
		//		return @"\bOD_ZCU_\w*\b";
		//		//return @"\A[OD_ZCU_]";
		//	}
		//	else return null;
		//}
		//public void ListDirectory()
		//{
		//	string url = site;
		//	HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
		//	using (HttpWebResponse response = (HttpWebResponse)request.GetResponse())
		//	{
		//		using (StreamReader reader = new StreamReader(response.GetResponseStream()))
		//		{
		//			string html = reader.ReadToEnd();



		//			Regex regex = new Regex(GetDirectoryListingRegexForUrl(url));
		//			MatchCollection matches = regex.Matches(html);
		//			Console.WriteLine(matches.Count);

		//			if (matches.Count > 0)
		//			{
		//				foreach (Match match in matches)
		//				{
		//					//if (match.Success)
		//					//{
		//						Console.WriteLine(match.ToString());
		//					//}
		//				}
		//			}
		//		}
		//		Console.ReadLine();
		//	}
		//}

		#endregion
	}
}