aswi2020sebela-gitlab/python-module/Utilities/Crawler/BasicCrawler.py @ c8f3051b

import requests
import re
from Utilities import FolderProcessor
from bs4 import BeautifulSoup


def get_all_links(url):
    # create response object
    r = requests.get(url)

    # create beautiful-soup object
    soup = BeautifulSoup(r.content, 'html5lib')
    links = []

    for link in soup.findAll('a'):
        links.append(link.get('href'))

    return links


def filter_links(links, regex):
    fitlered_links = []

    for link in links:
        if re.search(regex,link):
            fitlered_links.append(link)

    return fitlered_links


def create_absolute_links(links, archive):
    absolute_links = []

    for link in links:
        absolute_links.append(archive + link)

    return absolute_links


def remove_downloaded_links(links,dataset_name):

    downloaded_links = FolderProcessor.load_ignore_set("CrawlerLogs/" + dataset_name + "/")
    final_links = set(links) - downloaded_links

    return final_links


def download_file_from_url(url,path, dataset_name):
    r = requests.get(url, stream=True)

    url_parts = url.split("/")
    file_name = url_parts[len(url_parts)-1]

    with open(path + file_name, "wb") as file:
        for chunk in r.iter_content(chunk_size=1024):

            # writing one chunk at a time to pdf file
            if chunk:
                file.write(chunk)

    FolderProcessor.update_ignore_set("CrawlerLogs/" + dataset_name + "/", url)