commit 20513e9fbd66bbf97c19c368530ecc5b3f30358d
Author: Martin Sebela <msebela@zcu.cz>
Date:   Thu Feb 11 21:59:31 2021 +0100

    Dokončení oborového projektu KIV/OPSWI

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
deleted file mode 100644
index 8c10b14..0000000
--- a/.gitlab-ci.yml
+++ /dev/null
@@ -1,49 +0,0 @@
-stages:
-    - build
-    - linter
-    - test
-    - deploy
-    
-build:
-    stage: build
-    tags:
-        - build
-    script:
-        # Test whether container can be build or not
-        - ./scripts/build.sh
-        - docker-compose stop crawler nginx
-        # We could also push docker image to some registry or docker hub
-
-php_linter: 
-    stage: linter
-    tags:
-        - linter
-    script:
-        - ./scripts/composer.sh require --dev squizlabs/php_codesniffer
-        - docker-compose exec -T php-fpm /bin/sh -c "/var/www/symfony/vendor/bin/php-cs-fixer fix /var/www/symfony/src --dry-run --config /var/www/symfony/.php_cs --stop-on-violation --using-cache=no"
-
-
-tests:
-    stage: test
-    tags:
-        - test
-    script:
-        - ./scripts/composer.sh require --dev symfony/phpunit-bridge
-        - ./scripts/composer.sh require --dev symfony/browser-kit symfony/css-selector         
-        - docker-compose exec -T php-fpm /bin/sh -c "export APP_ENV=test && cd ../symfony && php bin/phpunit"
-        - docker stop $(docker ps -q)
-        
-deploy_prod:
-    stage: deploy
-    tags:
-        - deploy
-    variables:
-        GIT_STRATEGY: none
-    environment:
-        name: heatmap
-        url: https://heatmap.zcu.cz
-    script:
-        - ssh root@heatmap.zcu.cz "cd ~ && rm -rf aswi2020sebela && git clone https://gitlab.kiv.zcu.cz/aswi/aswi-2020/aswi2020sebela.git && cd aswi2020sebela && ./scripts/build.sh && exit"
-    only:
-        - master
-    when: manual
diff --git a/README.md b/README.md
index 4586c02..01b0b7f 100644
--- a/README.md
+++ b/README.md
@@ -1,14 +1,31 @@
-# ZČU Heatmap
+# Heatmap.ZČU
+App was created within the school project by ZČU students. The main goal of this project is to create a simple and generic tool for vizualization of heatmap like datasets. The first implementation can be seen at https://heatmap.zcu.cz.
+## Intro webpage
+<img src="./doc/images/website-intro.png" style="width: 600px; height: auto;">
 
-- [**Nginx**](https://www.nginx.com/) - Server
-- [**Symfony**](https://symfony.com/) - PHP framework
-- [**MongoDB**](https://www.mongodb.com/) - Database
+## Heatmap webpage
+<img src="./doc/images/website-heatmap.png" style="width: 600px; height: auto;">
+
+## Responsive
+<img src="./doc/images/website-responsive.png" style="width: 200px; height: auto;">
+
+<br><hr><br>
+
+## Technologies
+
+- [**Nginx**](https://www.nginx.com/) – Webserver
+- [**Symfony**](https://symfony.com/) – PHP framework
+- [**MongoDB**](https://www.mongodb.com/) – Database
+
+<br><hr><br>
 
 ## 🔩 Development env
 ### SW needed for development
 * **[🐳 docker](https://docs.docker.com/engine/install)**
 * **[🐳 docker-compose](https://docs.docker.com/compose/install/)**
 * **[📝 Visual Studio Code](https://code.visualstudio.com/Download)**
+<br><br>
+
 ### Development setup
 * First you need to **build** and **run** all needed containers
   * run either of these scripts inside the **root of the project**:
@@ -58,4 +75,48 @@ To make changes inside crawler module click on file `python.code-workspace` whic
 * Set your breakpoints and hit <kbd>F5</kbd> or goto <kbd>Run >> Start debugging</kbd>
 * Working in both modules
 ## 🚀 Production env
-* To run production env on your local machine execute this `./scripts/build.sh`
\ No newline at end of file
+* To run production env on your local machine execute this `./scripts/build.sh`
+
+<br><hr><br>
+
+## Add new dataset to heatmap
+
+Follow these steps:
+
+1. Run script `modules/crawler/prepare_new_dataset.py` in directory `modules/crawler`.
+2. Fill items like `display-name`, `display-color`, `dataset-name`, `url`, `regex`, `update-period` within the YAML config file in directory `modules/crawler/DatasetConfigs`.
+3. Implement own dataset crawler in directory `modules/crawler/DatasetCrawler`.
+4. Implement own dataset processor in directory `modules/crawler/DatasetProcessing`.
+5. Run script `modules/crawler/force_update_dataset.py` and enter name of new dataset. After that all needed data are downloaded and processed.
+6. Fill coordinates to `devices` items in the YAML config file.
+7. After running `force_update_datasets.py` again, data will be loaded to database.
+
+<br><hr><br>
+
+## Auto updating datasets –⁠ Cron script
+
+File `modules/crawler/crone_update_script.py` performs update of all added datasets. Update period is defined in the YAML config file of each dataset (item `update-period`).
+
+Script `modules/crawler/crone_update_script.py` is executed by systems cron (`modules/crawler/crontab`) every day.
+
+<br><hr><br>
+
+## Customize appearance of the heatmap
+
+### SASS style file
+CSS color settings (backgrounds, controls, ...).
+
+`public/css/style.scss`
+
+### Heatmap locations list
+Locations with coordinates.
+
+`public/templates/heatmap-locations.html.twig`
+
+<img src="./doc/images/heatmap-locations-list.png" style="width: 250px; height: auto;">
+
+### Intro heatmap webpage
+`public/templates/index.html.twig`
+
+### Heatmap webpage
+`public/templates/heatmap.html.twig`
\ No newline at end of file
diff --git a/doc/images/1.png b/doc/images/1.png
deleted file mode 100644
index d8237e7..0000000
Binary files a/doc/images/1.png and /dev/null differ
diff --git a/doc/images/2.png b/doc/images/2.png
deleted file mode 100644
index 05fbdb0..0000000
Binary files a/doc/images/2.png and /dev/null differ
diff --git a/doc/images/3.png b/doc/images/3.png
deleted file mode 100644
index 4c63e7b..0000000
Binary files a/doc/images/3.png and /dev/null differ
diff --git a/doc/images/heatmap-locations-list.png b/doc/images/heatmap-locations-list.png
new file mode 100644
index 0000000..dad4e72
Binary files /dev/null and b/doc/images/heatmap-locations-list.png differ
diff --git a/doc/images/website-heatmap.png b/doc/images/website-heatmap.png
new file mode 100644
index 0000000..e84260d
Binary files /dev/null and b/doc/images/website-heatmap.png differ
diff --git a/doc/images/website-intro.png b/doc/images/website-intro.png
new file mode 100644
index 0000000..d6b3792
Binary files /dev/null and b/doc/images/website-intro.png differ
diff --git a/doc/images/website-responsive.png b/doc/images/website-responsive.png
new file mode 100644
index 0000000..6aab9b9
Binary files /dev/null and b/doc/images/website-responsive.png differ
diff --git a/docker-compose-prod.yml b/docker-compose-prod.yml
index 44cab37..dff2205 100644
--- a/docker-compose-prod.yml
+++ b/docker-compose-prod.yml
@@ -7,8 +7,12 @@ services:
               ports: 
                      - "443:443"
               volumes: 
-                     - /acme-1:/root/.acme.sh/heatmap.zcu.cz
-                     - /acme-2:/root/.acme.sh/www.heatmap.zcu.cz
+                     # TODO: set certificate here
+                     # https://github.com/acmesh-official/acme.sh
+                     #
+                     #- /acme-1:/root/.acme.sh/heatmap.zcu.cz
+                     #- /acme-2:/root/.acme.sh/www.heatmap.zcu.cz
+                     #
                      - /etc/certificate:/certificate
                      - ./docker/nginx/sites:/etc/nginx/sites-available
                      - /logs/nginx:/var/log/nginx
diff --git a/docker-compose.yml b/docker-compose.yml
index d5e275f..cd2d4cd 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -55,6 +55,9 @@ services:
                        - '27017-27019:27017-27019'
                environment:
                        MONGO_INITDB_DATABASE: open-data-db
+                       #
+                       # TODO: set MongoDB credentials
+                       #
                        MONGO_INITDB_ROOT_USERNAME: root
                        MONGO_INITDB_ROOT_PASSWORD: root
                volumes:
diff --git a/docker/nginx/nginx.conf b/docker/nginx/nginx.conf
index 0713374..a251084 100644
--- a/docker/nginx/nginx.conf
+++ b/docker/nginx/nginx.conf
@@ -9,7 +9,6 @@ events {
     worker_connections 1024;
 }
 
-
 http {
     include /etc/nginx/mime.types;
     default_type application/octet-stream;
diff --git a/docker/nginx/sites-dev/default.conf b/docker/nginx/sites-dev/default.conf
index bb103c1..88dfafe 100644
--- a/docker/nginx/sites-dev/default.conf
+++ b/docker/nginx/sites-dev/default.conf
@@ -2,6 +2,9 @@ server {
     listen 80 default_server;
     listen [::]:80 default_server ipv6only=on;
 
+    #
+    # TODO: set website domain
+    #
     server_name heatmap.zcu.cz www.heatmap.zcu.cz;
     root /var/www/symfony/public;
     index index.php index.html index.htm;
diff --git a/docker/nginx/sites/default.conf b/docker/nginx/sites/default.conf
index 6f36270..15357c1 100644
--- a/docker/nginx/sites/default.conf
+++ b/docker/nginx/sites/default.conf
@@ -1,6 +1,10 @@
 server {
     listen 80;
     listen [::]:80;
+
+    #
+    # TODO: set website domain
+    #
     server_name heatmap.zcu.cz www.heatmap.zcu.cz;
 
     location ~ /.well-known {
@@ -13,20 +17,39 @@ server {
 server {
     listen 443 ssl;
     listen [::]:443 ssl;
+
+    #
+    # TODO: set website domain
+    #
     server_name www.heatmap.zcu.cz;
+
+    #
+    # TODO: set certificate files
+    #
     ssl_certificate /root/.acme.sh/heatmap.zcu.cz/fullchain.cer;
     ssl_certificate_key /root/.acme.sh/heatmap.zcu.cz/heatmap.zcu.cz.key;
+
     ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
     ssl_prefer_server_ciphers on;
     ssl_ciphers 'EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH';
 
+    #
+    # TODO: set website domain
+    #
     return 301 https://heatmap.zcu.cz$request_uri;
 }
 server {
     listen 443 ssl;
     listen [::]:443 ssl;
 
+    #
+    # TODO: set website domain
+    #
     server_name heatmap.zcu.cz;
+
+    #
+    # TODO: set certificate files
+    #
     ssl_certificate /root/.acme.sh/heatmap.zcu.cz/fullchain.cer;
     ssl_certificate_key /root/.acme.sh/heatmap.zcu.cz/heatmap.zcu.cz.key;
 
diff --git a/docker/php-fpm/dev/Dockerfile b/docker/php-fpm/dev/Dockerfile
index e33b868..dad0b42 100644
--- a/docker/php-fpm/dev/Dockerfile
+++ b/docker/php-fpm/dev/Dockerfile
@@ -1,4 +1,4 @@
-#DEV ONLY DOCKERFILE
+# DEV ONLY DOCKERFILE
 FROM php:7.4-fpm-alpine
 RUN apk update; \
     apk upgrade; \
diff --git a/modules/crawler/.devcontainer/devcontainer.json b/modules/crawler/.devcontainer/devcontainer.json
index da33d12..b78c9a6 100644
--- a/modules/crawler/.devcontainer/devcontainer.json
+++ b/modules/crawler/.devcontainer/devcontainer.json
@@ -1,5 +1,4 @@
 {
-    //TODO: dodat extensionu pro yaml soubory!!!
     "forwardPorts": [
         8000,
         443,
diff --git a/modules/crawler/Utilities/Database/database_loader.py b/modules/crawler/Utilities/Database/database_loader.py
index e53a3b8..868a6a9 100644
--- a/modules/crawler/Utilities/Database/database_loader.py
+++ b/modules/crawler/Utilities/Database/database_loader.py
@@ -6,6 +6,10 @@ from typing import Dict
 import pymongo
 import re
 
+#
+# TODO: set MongoDB credentials
+#
+
 # specify mongodb connection
 MONGODB_CONNECTION = "mongodb://root:root@database"
 # mongodb account name
diff --git a/scripts/build.sh b/scripts/build.sh
index 4a5eca8..04b01c3 100755
--- a/scripts/build.sh
+++ b/scripts/build.sh
@@ -3,5 +3,6 @@ docker-compose down
 docker-compose -f docker-compose.yml -f docker-compose-prod.yml up --build -d
 docker-compose exec -T php-fpm composer install --no-interaction --working-dir=/var/www/symfony --no-dev
 docker-compose exec -T php-fpm /bin/sh -c "cd /var/www/symfony && npm install && npm link webpack && npm run build"
-#python module initiation
+
+# python module initiation
 docker-compose exec -T crawler /bin/sh -c "cd /src && python docker_prepare_structure.py && python force_update_datasets.py all"
diff --git a/scripts/pip.sh b/scripts/pip.sh
index 50eeb16..5f68797 100644
--- a/scripts/pip.sh
+++ b/scripts/pip.sh
@@ -1,3 +1,2 @@
 #!/bin/bash
-#install requirements pip.sh install -r /var/python/requirements.txt
 docker-compose exec -T crawler pip $@
\ No newline at end of file
diff --git a/scripts/renew_certificate.sh b/scripts/renew_certificate.sh
index 2c4fc11..e84b20f 100755
--- a/scripts/renew_certificate.sh
+++ b/scripts/renew_certificate.sh
@@ -3,6 +3,9 @@
 # if you want create new license run: renew_certificate new
 # if you want re-create the existing license run: renew_certificate
 
+#
+# TODO: set website domain
+#
 if [ "$1" = "new" ]; then
     docker-compose exec nginx /bin/sh -c "
     cd /root/.acme.sh; \
diff --git a/scripts/restore_db.sh b/scripts/restore_db.sh
index 2250dd5..9f89115 100755
--- a/scripts/restore_db.sh
+++ b/scripts/restore_db.sh
@@ -1,2 +1,6 @@
 #!/bin/bash
+
+#
+# TODO: set MongoDB credentials
+#
 docker-compose exec -T database mongorestore --username root --password root --authenticationDatabase admin --db open-data-db /dev-dump/data
\ No newline at end of file
diff --git a/website/.env b/website/.env
index af1b6ca..bc3d2c3 100644
--- a/website/.env
+++ b/website/.env
@@ -19,6 +19,10 @@ APP_SECRET=ca5751564bbb9a0d75d412e020790283
 #TRUSTED_PROXIES=127.0.0.0/8,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16
 #TRUSTED_HOSTS='^(localhost|example\.com)$'
 ###< symfony/framework-bundle ###
+
+#
+# TODO: set MongoDB credentials
+#
 DATABASE_CONNECTION_STRING=mongodb://root:root@database
 
 ###> sentry/sentry-symfony ###
diff --git a/website/.env.test b/website/.env.test
index 7c76a65..fc33611 100644
--- a/website/.env.test
+++ b/website/.env.test
@@ -1,3 +1,7 @@
 APP_ENV=test
 APP_SECRET=ca5751564bbb9a0d75d412e020790283
+
+#
+# TODO: set MongoDB credentials
+#
 DATABASE_CONNECTION_STRING=mongodb://root:root@database
diff --git a/website/config/services.yaml b/website/config/services.yaml
index 39e803c..dc6d114 100644
--- a/website/config/services.yaml
+++ b/website/config/services.yaml
@@ -3,6 +3,10 @@
 
 # Put parameters here that don't need to change on each machine where the app is deployed
 # https://symfony.com/doc/current/best_practices/configuration.html#application-related-configuration
+
+#
+# TODO: set MongoDB credentials
+#
 parameters:
     databaseConnectionString: 'mongodb://root:root@database'
 
diff --git a/website/phpunit.xml.dist b/website/phpunit.xml.dist
index 00a0f68..5062591 100644
--- a/website/phpunit.xml.dist
+++ b/website/phpunit.xml.dist
@@ -7,6 +7,9 @@
     <php>
         <env name="APP_ENV" value="test" />
         <env name="KERNEL_CLASS" value="App\Kernel" />
+        <!--
+            TODO: set MongoDB credentials
+        -->
         <env name="DATABASE_CONNECTION_STRING" value="mongodb://root:root@database" />
     </php>
 </phpunit>
\ No newline at end of file
diff --git a/website/public/js/index.js b/website/public/js/index.js
index 2895669..5a2a0f6 100644
--- a/website/public/js/index.js
+++ b/website/public/js/index.js
@@ -249,7 +249,6 @@ const drawHeatmap = (dataRaw) => {
     heatmapLayer = L.heatLayer(mergedPoints, { max: max, minOpacity: 0.5, radius: 35, blur: 30 }).addTo(mymap)
   }
 
-  // timto vyresen bug #8191 - TODO: znamena to, ze muzeme smazat volani updatePopup() ve funkcich, kde se nejdriv vola drawHeatmap() a pak updatePopup()?
   updatePopup()
 }
 
@@ -417,8 +416,6 @@ const showPopup = (e) => {
   info = []
   currentPageInPopup = 0
 
-  // https://wiki.openstreetmap.org/wiki/Zoom_levels
-  // Todo change to variable - it is used in heatmap init
   const stile = 40075016.686 * Math.cos(startX) / Math.pow(2, mymap.getZoom())
   const radius = 25 * stile / 256
 
diff --git a/website/templates/base.html.twig b/website/templates/base.html.twig
index 95584e3..21a1f37 100644
--- a/website/templates/base.html.twig
+++ b/website/templates/base.html.twig
@@ -4,32 +4,23 @@
     <meta charset="utf-8">
         
     <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
-    <meta name="description" content="Heatmapa kampusu Západočeské univerzity v Plzni">
-    <meta name="keywords" content="heatmap, open data, otevřená data, dataset, západočeská univerzita, zču, plzeň">
-    <meta name="author" content="Tým BHVS &ndash; Tomáš Ballák, Petr Hlaváč, Jakub Vašta, Martin Šebela">
+    <meta name="description" content="">
+    <meta name="keywords" content="">
 
-    <meta property="og:url" content="https://heatmap.zcu.cz">
+    <meta property="og:url" content="">
     <meta property="og:type" content="website">
-    <meta property="og:title" content="Heatmapa kampusu Západočeské univerzity v Plzni">
-    <meta property="og:description" content="Hodina po hodině, den po dni.">
-    <meta property="og:image" content="https://heatmap.zcu.cz/img/banner.jpg">
+    <meta property="og:title" content="">
+    <meta property="og:description" content="">
+    <meta property="og:image" content="">
 
     <link rel="shortcut icon" href="favicon.ico">
 
     <title>{% block title %}{% endblock %}</title>
-
-    <script async src="https://www.googletagmanager.com/gtag/js?id=G-CE7TFX9ZEL"></script>
-    <script>
-      window.dataLayer = window.dataLayer || [];
-      function gtag(){dataLayer.push(arguments);}
-      gtag('js', new Date());
-      gtag('config', 'G-CE7TFX9ZEL');
-    </script>
     
     {% block stylesheets %}{% endblock %}
     {% block style %}{% endblock %}
- </head>
- <body{% block bodyClass %}{% endblock %}>
+  </head>
+  <body{% block bodyClass %}{% endblock %}>
     {% block body %}{% endblock %}
     {% block javascripts %}{% endblock %}
 
