Revize af7609b5

Re #8193 - refactoring crawler

modules/crawler/docker_prepare_structure.py

@@ -10 +10 @@
 CRAWLER_LOGS_PATH = "CrawlerLogs/"
 
 
-def prepare_strucure_for_all_datasets():
+def prepare_strucure_for_all_datasets() -> None:
     """
     Prepares folders that are necessery but does not contain code so they are excluded from gitlab by gitignore
     """
 
-    if  not os.path.isdir(CRAWLED_DATA_PATH) :
+    if not os.path.isdir(CRAWLED_DATA_PATH):
         try:
             os.mkdir(CRAWLED_DATA_PATH)
         except os.error as e:
             print(e)
-            print("Creation of the directory %s failed" % CRAWLED_DATA_PATH)
+            print("Nelze vytvořit adresář %s" % CRAWLED_DATA_PATH)
 
-    if  not os.path.isdir(PROCESSED_DATA_PATH) :
+    if not os.path.isdir(PROCESSED_DATA_PATH):
         try:
             os.mkdir(PROCESSED_DATA_PATH)
         except os.error as e:
             print(e)
-            print("Creation of the directory %s failed" % PROCESSED_DATA_PATH)
-    
-    if  not os.path.isdir(CRAWLER_LOGS_PATH) :
+            print("Nelze vytvořit adresář %s" % PROCESSED_DATA_PATH)
+
+    if not os.path.isdir(CRAWLER_LOGS_PATH):
         try:
             os.mkdir(CRAWLER_LOGS_PATH)
         except os.error as e:
             print(e)
-            print("Creation of the directory %s failed" % PROCESSED_DATA_PATH)
-
+            print("Nelze vytvořit adresář %s" % CRAWLER_LOGS_PATH)
 
     files_in_dir = os.listdir(CONFIG_FILES_PATH)
 
@@ -44 +43 @@
         prepare_structure(name[0])
 
 
-def prepare_structure(dataset_name):
+def prepare_structure(dataset_name: str) -> None:
     """
     Create folder for every dataset in newly created folder for processed and crawled data
     """
 
-    path =  CRAWLED_DATA_PATH + dataset_name
-    if  not os.path.isdir(path) :
+    path = CRAWLED_DATA_PATH + dataset_name
+    if not os.path.isdir(path):
         os.mkdir(path)
 
-    path =  PROCESSED_DATA_PATH + dataset_name
-    if not  os.path.isdir(path):
+    path = PROCESSED_DATA_PATH + dataset_name
+    if not os.path.isdir(path):
         os.mkdir(PROCESSED_DATA_PATH + dataset_name)
 
 
-print("Inicializuji počáteční strukturu pro stažená a zpracovaná data")
-prepare_strucure_for_all_datasets()
+def main() -> None:
+    print("Inicializuji počáteční strukturu pro stažená a zpracovaná data")
+    prepare_strucure_for_all_datasets()
+
+
+if __name__ == "__main__":
+    main()