aswi2021tri-musketyri-gitlab/Server/ServerApp/DataDownload/DataDownloader.cs @ 1434de93

using System;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using static System.Net.Mime.MediaTypeNames;

namespace DataDownload
{
	public enum DataType
	{
		POCASI, ENERGO, STROJE, EMAIL, OBSAZENI_MISTNOSTI, JIS, KOLOBEZKY, WIFI
	}
	public enum DataFormat
	{
		XML, JSON, CSV
	}

	public class DataDownloader
	{
		public string DataDirectory { get; }
		public bool OverwriteExisting { get; set; }

		private string site;
		private string dataStr;
		private WebClient webClient;
		private char sep = Path.DirectorySeparatorChar;

		public DataDownloader()
		{
			Console.WriteLine(Directory.GetCurrentDirectory());
			site = "http://openstore.zcu.cz/";
			dataStr = "OD_ZCU_";

			DataDirectory = $"..{sep}..{sep}data{sep}auto";
			OverwriteExisting = false;

			webClient = new WebClient();
		}

		private List<string> DownloadData(DataType type, DataFormat format, int year, int month)
		{
			// the list of all files potentially relevant to the caller
			List<string> extractedFiles = new List<string>();

			// Prepare the url string to be downloaded from:
			string monthStr = month < 10 ? "0" + month : "" + month;
			string yearStr = "" + year;

			string monthYr = monthStr + "_" + yearStr;
			string url = site + "/" + dataStr + monthYr + "/" + dataStr + type + "_" + monthYr + "_" + format + ".zip";
			string nameZip = dataStr + type + "_" + monthYr + "_" + format + ".zip";
			string nameFolder = DataDirectory + sep + type + sep; //+ dataStr + type + "_" + monthYr + "_" + format;

			try
			{
				Console.WriteLine("Downloading .zip to " + Path.GetFullPath(nameZip) + "...");

				// Download the zip file:
				webClient.DownloadFile(url, nameZip);

				//ZipFile.ExtractToDirectory(nameZip, nameFolder);
				ZipArchive zipArchive = ZipFile.OpenRead(nameZip);
				// Go through all the extracted files:
				foreach (ZipArchiveEntry entry in zipArchive.Entries)
				{
					// get the relative path to the file:
					string extractedFile = nameFolder + entry.Name;

					// add full path to the list:
					extractedFiles.Add(Path.GetFullPath(extractedFile));

					if (OverwriteExisting)
					{
						// if overwrite is desired, execute it:
						entry.ExtractToFile(extractedFile, OverwriteExisting);
						
					}
					else
					{
						// if overwrite is not desired, check if the file exists first:
						if(File.Exists(nameFolder + entry.Name))
						{
							continue;
						}
						else
						{
							// if it doesn't exist, save it:
							entry.ExtractToFile(extractedFile, OverwriteExisting);
						}
					}

					
				}
				// dispose of the archive:
				zipArchive.Dispose();

				Console.WriteLine("Extracted to " + Path.GetFullPath(nameFolder));
				Console.WriteLine("Deleting .zip from " + Path.GetFullPath(nameZip) + "...");
				Console.WriteLine("Finished downloading " + nameZip);

				// delete the previously downloaded zip file, files contained in it have been extracted:
				File.Delete(nameZip); // todo check?

			}
			catch(System.Net.WebException we)
			{
				// download fails, if the specified url is invalid
				Console.WriteLine("Download from " + url + " failed.");
				Console.WriteLine(we.Message);
			}



			return extractedFiles;
		}



		/// <summary>
		/// Downloads selected type and time span of data in the desired format, returns a list of full paths to all successfully saved files. 
		/// If some of the files already existed and were not overwritten, then the returned List contains paths to these files also.
		/// </summary>
		/// <param name="type">The type of data, e.g. the jis data</param>
		/// <param name="format">The desired format of data files, available are CSV, XML and JSON formats</param>
		/// <param name="startYear">The start year to start the download from, inclusive</param>
		/// <param name="endYear">The end year to start the download from, inclusive</param>
		/// <param name="startMonth">The start month to start the download from, inclusive</param>
		/// <param name="endMonth">The end month to start the download from, inclusive</param>
		/// <returns>A list of full paths to all successfully saved files</returns>
		public List<string> DownloadData(DataType type, DataFormat format, int startYear, int endYear, int startMonth, int endMonth)
		{
			List<string> savedFiles = new List<string>();
			DirectoryInfo di = Directory.CreateDirectory(DataDirectory + sep + type);

			for (int y = startYear; y <= endYear; y++)
			{
				for (int m = startMonth; m <= endMonth; m++)
				{
					savedFiles.AddRange(DownloadData(type, format, y, m));
				}
			}

			return savedFiles;
		}


		#region UNUSED
		//public string GetDirectoryListingRegexForUrl(string url)
		//{
		//	if (url.Equals(site))
		//	{
		//		//return "\\\"([^\"]*)\\\"";
		//		return @"\bOD_ZCU_\w*\b";
		//		//return @"\A[OD_ZCU_]";
		//	}
		//	else return null;
		//}
		//public void ListDirectory()
		//{
		//	string url = site;
		//	HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
		//	using (HttpWebResponse response = (HttpWebResponse)request.GetResponse())
		//	{
		//		using (StreamReader reader = new StreamReader(response.GetResponseStream()))
		//		{
		//			string html = reader.ReadToEnd();



		//			Regex regex = new Regex(GetDirectoryListingRegexForUrl(url));
		//			MatchCollection matches = regex.Matches(html);
		//			Console.WriteLine(matches.Count);

		//			if (matches.Count > 0)
		//			{
		//				foreach (Match match in matches)
		//				{
		//					//if (match.Success)
		//					//{
		//						Console.WriteLine(match.ToString());
		//					//}
		//				}
		//			}
		//		}
		//		Console.ReadLine();
		//	}
		//}

		#endregion
	}
}