Revize 20513e9f

Dokončení oborového projektu KIV/OPSWI

.gitlab-ci.yml

@@ -1 @@
-stages:
-    - build
-    - linter
-    - test
-    - deploy
-    
-build:
-    stage: build
-    tags:
-        - build
-    script:
-        # Test whether container can be build or not
-        - ./scripts/build.sh
-        - docker-compose stop crawler nginx
-        # We could also push docker image to some registry or docker hub
-
-php_linter: 
-    stage: linter
-    tags:
-        - linter
-    script:
-        - ./scripts/composer.sh require --dev squizlabs/php_codesniffer
-        - docker-compose exec -T php-fpm /bin/sh -c "/var/www/symfony/vendor/bin/php-cs-fixer fix /var/www/symfony/src --dry-run --config /var/www/symfony/.php_cs --stop-on-violation --using-cache=no"
-
-
-tests:
-    stage: test
-    tags:
-        - test
-    script:
-        - ./scripts/composer.sh require --dev symfony/phpunit-bridge
-        - ./scripts/composer.sh require --dev symfony/browser-kit symfony/css-selector         
-        - docker-compose exec -T php-fpm /bin/sh -c "export APP_ENV=test && cd ../symfony && php bin/phpunit"
-        - docker stop $(docker ps -q)
-        
-deploy_prod:
-    stage: deploy
-    tags:
-        - deploy
-    variables:
-        GIT_STRATEGY: none
-    environment:
-        name: heatmap
-        url: https://heatmap.zcu.cz
-    script:
-        - ssh [e-mail] "cd ~ && rm -rf aswi2020sebela && git clone https://gitlab.kiv.zcu.cz/aswi/aswi-2020/aswi2020sebela.git && cd aswi2020sebela && ./scripts/build.sh && exit"
-    only:
-        - master
-    when: manual

README.md

@@ -1 @@
-# ZČU Heatmap
+# Heatmap.ZČU
+App was created within the school project by ZČU students. The main goal of this project is to create a simple and generic tool for vizualization of heatmap like datasets. The first implementation can be seen at https://heatmap.zcu.cz.
+## Intro webpage
+<img src="./doc/images/website-intro.png" style="width: 600px; height: auto;">
 
-- [**Nginx**](https://www.nginx.com/) - Server
-- [**Symfony**](https://symfony.com/) - PHP framework
-- [**MongoDB**](https://www.mongodb.com/) - Database
+## Heatmap webpage
+<img src="./doc/images/website-heatmap.png" style="width: 600px; height: auto;">
+
+## Responsive
+<img src="./doc/images/website-responsive.png" style="width: 200px; height: auto;">
+
+<br><hr><br>
+
+## Technologies
+
+- [**Nginx**](https://www.nginx.com/) – Webserver
+- [**Symfony**](https://symfony.com/) – PHP framework
+- [**MongoDB**](https://www.mongodb.com/) – Database
+
+<br><hr><br>
 
 ## 🔩 Development env
 ### SW needed for development
 * **[🐳 docker](https://docs.docker.com/engine/install)**
 * **[🐳 docker-compose](https://docs.docker.com/compose/install/)**
 * **[📝 Visual Studio Code](https://code.visualstudio.com/Download)**
+<br><br>
+
 ### Development setup
 * First you need to **build** and **run** all needed containers
   * run either of these scripts inside the **root of the project**:
@@ -58 +75 @@
 * Set your breakpoints and hit <kbd>F5</kbd> or goto <kbd>Run >> Start debugging</kbd>
 * Working in both modules
 ## 🚀 Production env
-* To run production env on your local machine execute this `./scripts/build.sh`
+* To run production env on your local machine execute this `./scripts/build.sh`
+
+<br><hr><br>
+
+## Add new dataset to heatmap
+
+Follow these steps:
+
+1. Run script `modules/crawler/prepare_new_dataset.py` in directory `modules/crawler`.
+2. Fill items like `display-name`, `display-color`, `dataset-name`, `url`, `regex`, `update-period` within the YAML config file in directory `modules/crawler/DatasetConfigs`.
+3. Implement own dataset crawler in directory `modules/crawler/DatasetCrawler`.
+4. Implement own dataset processor in directory `modules/crawler/DatasetProcessing`.
+5. Run script `modules/crawler/force_update_dataset.py` and enter name of new dataset. After that all needed data are downloaded and processed.
+6. Fill coordinates to `devices` items in the YAML config file.
+7. After running `force_update_datasets.py` again, data will be loaded to database.
+
+<br><hr><br>
+
+## Auto updating datasets –⁠ Cron script
+
+File `modules/crawler/crone_update_script.py` performs update of all added datasets. Update period is defined in the YAML config file of each dataset (item `update-period`).
+
+Script `modules/crawler/crone_update_script.py` is executed by systems cron (`modules/crawler/crontab`) every day.
+
+<br><hr><br>
+
+## Customize appearance of the heatmap
+
+### SASS style file
+CSS color settings (backgrounds, controls, ...).
+
+`public/css/style.scss`
+
+### Heatmap locations list
+Locations with coordinates.
+
+`public/templates/heatmap-locations.html.twig`
+
+<img src="./doc/images/heatmap-locations-list.png" style="width: 250px; height: auto;">
+
+### Intro heatmap webpage
+`public/templates/index.html.twig`
+
+### Heatmap webpage
+`public/templates/heatmap.html.twig`

docker-compose-prod.yml

@@ -7 +7 @@
               ports: 
                      - "443:443"
               volumes: 
-                     - /acme-1:/root/.acme.sh/heatmap.zcu.cz
-                     - /acme-2:/root/.acme.sh/www.heatmap.zcu.cz
+                     # TODO: set certificate here
+                     # https://github.com/acmesh-official/acme.sh
+                     #
+                     #- /acme-1:/root/.acme.sh/heatmap.zcu.cz
+                     #- /acme-2:/root/.acme.sh/www.heatmap.zcu.cz
+                     #
                      - /etc/certificate:/certificate
                      - ./docker/nginx/sites:/etc/nginx/sites-available
                      - /logs/nginx:/var/log/nginx

docker-compose.yml

@@ -55 +55 @@
                        - '27017-27019:27017-27019'
                environment:
                        MONGO_INITDB_DATABASE: open-data-db
+                       #
+                       # TODO: set MongoDB credentials
+                       #
                        MONGO_INITDB_ROOT_USERNAME: root
                        MONGO_INITDB_ROOT_PASSWORD: root
                volumes:

docker/nginx/nginx.conf

@@ -9 +9 @@
     worker_connections 1024;
 }
 
-
 http {
     include /etc/nginx/mime.types;
     default_type application/octet-stream;

docker/nginx/sites-dev/default.conf

@@ -2 +2 @@
     listen 80 default_server;
     listen [::]:80 default_server ipv6only=on;
 
+    #
+    # TODO: set website domain
+    #
     server_name heatmap.zcu.cz www.heatmap.zcu.cz;
     root /var/www/symfony/public;
     index index.php index.html index.htm;

docker/nginx/sites/default.conf

@@ -1 +1 @@
 server {
     listen 80;
     listen [::]:80;
+
+    #
+    # TODO: set website domain
+    #
     server_name heatmap.zcu.cz www.heatmap.zcu.cz;
 
     location ~ /.well-known {
@@ -13 +17 @@
 server {
     listen 443 ssl;
     listen [::]:443 ssl;
+
+    #
+    # TODO: set website domain
+    #
     server_name www.heatmap.zcu.cz;
+
+    #
+    # TODO: set certificate files
+    #
     ssl_certificate /root/.acme.sh/heatmap.zcu.cz/fullchain.cer;
     ssl_certificate_key /root/.acme.sh/heatmap.zcu.cz/heatmap.zcu.cz.key;
+
     ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
     ssl_prefer_server_ciphers on;
     ssl_ciphers 'EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH';
 
+    #
+    # TODO: set website domain
+    #
     return 301 https://heatmap.zcu.cz$request_uri;
 }
 server {
     listen 443 ssl;
     listen [::]:443 ssl;
 
+    #
+    # TODO: set website domain
+    #
     server_name heatmap.zcu.cz;
+
+    #
+    # TODO: set certificate files
+    #
     ssl_certificate /root/.acme.sh/heatmap.zcu.cz/fullchain.cer;
     ssl_certificate_key /root/.acme.sh/heatmap.zcu.cz/heatmap.zcu.cz.key;
 

docker/php-fpm/dev/Dockerfile

@@ -1 @@
-#DEV ONLY DOCKERFILE
+# DEV ONLY DOCKERFILE
 FROM php:7.4-fpm-alpine
 RUN apk update; \
     apk upgrade; \

modules/crawler/.devcontainer/devcontainer.json

@@ -1 +1 @@
 {
-    //TODO: dodat extensionu pro yaml soubory!!!
     "forwardPorts": [
         8000,
         443,

modules/crawler/Utilities/Database/database_loader.py

@@ -6 +6 @@
 import pymongo
 import re
 
+#
+# TODO: set MongoDB credentials
+#
+
 # specify mongodb connection
 MONGODB_CONNECTION = "mongodb://root:root@database"
 # mongodb account name

scripts/build.sh

@@ -3 +3 @@
 docker-compose -f docker-compose.yml -f docker-compose-prod.yml up --build -d
 docker-compose exec -T php-fpm composer install --no-interaction --working-dir=/var/www/symfony --no-dev
 docker-compose exec -T php-fpm /bin/sh -c "cd /var/www/symfony && npm install && npm link webpack && npm run build"
-#python module initiation
+
+# python module initiation
 docker-compose exec -T crawler /bin/sh -c "cd /src && python docker_prepare_structure.py && python force_update_datasets.py all"

scripts/pip.sh

@@ -1 +1 @@
 #!/bin/bash
-#install requirements pip.sh install -r /var/python/requirements.txt
 docker-compose exec -T crawler pip $@

scripts/renew_certificate.sh

@@ -3 +3 @@
 # if you want create new license run: renew_certificate new
 # if you want re-create the existing license run: renew_certificate
 
+#
+# TODO: set website domain
+#
 if [ "$1" = "new" ]; then
     docker-compose exec nginx /bin/sh -c "
     cd /root/.acme.sh; \

scripts/restore_db.sh

@@ -1 +1 @@
 #!/bin/bash
+
+#
+# TODO: set MongoDB credentials
+#
 docker-compose exec -T database mongorestore --username root --password root --authenticationDatabase admin --db open-data-db /dev-dump/data

website/.env

@@ -19 +19 @@
 #TRUSTED_PROXIES=127.0.0.0/8,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16
 #TRUSTED_HOSTS='^(localhost|example\.com)$'
 ###< symfony/framework-bundle ###
+
+#
+# TODO: set MongoDB credentials
+#
 DATABASE_CONNECTION_STRING=mongodb://root:root@database
 
 ###> sentry/sentry-symfony ###

website/.env.test

@@ -1 +1 @@
 APP_ENV=test
 APP_SECRET=ca5751564bbb9a0d75d412e020790283
+
+#
+# TODO: set MongoDB credentials
+#
 DATABASE_CONNECTION_STRING=mongodb://root:root@database

website/config/services.yaml

@@ -3 +3 @@
 
 # Put parameters here that don't need to change on each machine where the app is deployed
 # https://symfony.com/doc/current/best_practices/configuration.html#application-related-configuration
+
+#
+# TODO: set MongoDB credentials
+#
 parameters:
     databaseConnectionString: 'mongodb://root:root@database'
 

website/phpunit.xml.dist

@@ -7 +7 @@
     <php>
         <env name="APP_ENV" value="test" />
         <env name="KERNEL_CLASS" value="App\Kernel" />
+        <!--
+            TODO: set MongoDB credentials
+        -->
         <env name="DATABASE_CONNECTION_STRING" value="mongodb://root:root@database" />
     </php>
 </phpunit>

website/public/js/index.js

@@ -249 +249 @@
     heatmapLayer = L.heatLayer(mergedPoints, { max: max, minOpacity: 0.5, radius: 35, blur: 30 }).addTo(mymap)
   }
 
-  // timto vyresen bug #8191 - TODO: znamena to, ze muzeme smazat volani updatePopup() ve funkcich, kde se nejdriv vola drawHeatmap() a pak updatePopup()?
   updatePopup()
 }
 
@@ -417 +416 @@
   info = []
   currentPageInPopup = 0
 
-  // https://wiki.openstreetmap.org/wiki/Zoom_levels
-  // Todo change to variable - it is used in heatmap init
   const stile = 40075016.686 * Math.cos(startX) / Math.pow(2, mymap.getZoom())
   const radius = 25 * stile / 256
 

website/templates/base.html.twig

@@ -4 +4 @@
     <meta charset="utf-8">
         
     <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
-    <meta name="description" content="Heatmapa kampusu Západočeské univerzity v Plzni">
-    <meta name="keywords" content="heatmap, open data, otevřená data, dataset, západočeská univerzita, zču, plzeň">
-    <meta name="author" content="Tým BHVS &ndash; Tomáš Ballák, Petr Hlaváč, Jakub Vašta, Martin Šebela">
+    <meta name="description" content="">
+    <meta name="keywords" content="">
 
-    <meta property="og:url" content="https://heatmap.zcu.cz">
+    <meta property="og:url" content="">
     <meta property="og:type" content="website">
-    <meta property="og:title" content="Heatmapa kampusu Západočeské univerzity v Plzni">
-    <meta property="og:description" content="Hodina po hodině, den po dni.">
-    <meta property="og:image" content="https://heatmap.zcu.cz/img/banner.jpg">
+    <meta property="og:title" content="">
+    <meta property="og:description" content="">
+    <meta property="og:image" content="">
 
     <link rel="shortcut icon" href="favicon.ico">
 
     <title>{% block title %}{% endblock %}</title>
-
-    <script async src="https://www.googletagmanager.com/gtag/js?id=G-CE7TFX9ZEL"></script>
-    <script>
-      window.dataLayer = window.dataLayer || [];
-      function gtag(){dataLayer.push(arguments);}
-      gtag('js', new Date());
-      gtag('config', 'G-CE7TFX9ZEL');
-    </script>
     
     {% block stylesheets %}{% endblock %}
     {% block style %}{% endblock %}
- </head>
- <body{% block bodyClass %}{% endblock %}>
+  </head>
+  <body{% block bodyClass %}{% endblock %}>
     {% block body %}{% endblock %}
     {% block javascripts %}{% endblock %}