import settings
import random
import requests
from proxy_webshare import proxy_webshare
from bs4 import BeautifulSoup

def url_scrape(url):
    text = None
    curr_type = None
    curr_request = None
    for i in range(3):
        try:
            header = random.choice(settings.headers_list)
            curr_request = requests.get(url, headers=header, proxies=proxy_webshare())
            break
        except:
            pass
    if curr_request:
        if curr_request.status_code == 200:

            if 'Content-Type' in curr_request.headers:  # Add this check
                curr_type = curr_request.headers['Content-Type']

            if curr_type or curr_type.split(';')[0] in settings.mime_type:
                curr_html = curr_request.text
                text = curr_html

            # Create soup object to remove HTML tags
            curr_soup = BeautifulSoup(text, 'lxml')
            # Kill all script and style elements
            for script in curr_soup(['script', 'style']):
                script.decompose()
            # Kill Wayback bars
            for div_id in ['wm-ipp-base', 'wm-ipp-print', 'donato']:
                for div in curr_soup.find_all('div', id=div_id):
                    div.decompose()

            # Body text of the html
            try:
                text = curr_soup.body.get_text()
            except:
                text = None
            return text
        else:
            return None
    else:
        return None
