dohsimpson / kubernetes-doc-pdf Goto Github PK

directories_pairs = [("https://kubernetes.io/docs/{}/".format(n.lower()), n) for n in directories]

directories_pairs = [("https://kubernetes.io/zh/docs/{}/".format(n.lower()), n) for n in directories]

C:\workspace\github\dohsimpson\kubernetes-doc-pdf>pip install pipenv
Collecting pipenv
  Downloading https://files.pythonhosted.org/packages/13/b4/3ffa55f77161cff9a5220f162670f7c5eb00df52e00939e203f601b0f579/pipenv-2018.11.26-py3-none-any.whl (5.2MB)
    100% |████████████████████████████████| 5.2MB 1.6MB/s
Requirement already satisfied: pip>=9.0.1 in c:\users\poyatm01\appdata\local\programs\python\python37-32\lib\site-packages (from pipenv) (19.0.3)
Collecting certifi (from pipenv)
  Downloading https://files.pythonhosted.org/packages/69/1b/b853c7a9d4f6a6d00749e94eb6f3a041e342a885b87340b79c1ef73e3a78/certifi-2019.6.16-py2.py3-none-any.whl (157kB)
    100% |████████████████████████████████| 163kB 1.4MB/s
Collecting virtualenv (from pipenv)
  Downloading https://files.pythonhosted.org/packages/8b/12/8d4f45b8962b03ac9efefe5ed5053f6b29334d83e438b4fe379d21c0cb8e/virtualenv-16.7.5-py2.py3-none-any.whl (3.3MB)
    100% |████████████████████████████████| 3.3MB 1.1MB/s
Requirement already satisfied: setuptools>=36.2.1 in c:\users\poyatm01\appdata\local\programs\python\python37-32\lib\site-packages (from pipenv) (40.8.0)
Collecting virtualenv-clone>=0.2.5 (from pipenv)
  Downloading https://files.pythonhosted.org/packages/ba/f8/50c2b7dbc99e05fce5e5b9d9a31f37c988c99acd4e8dedd720b7b8d4011d/virtualenv_clone-0.5.3-py2.py3-none-any.whl
Installing collected packages: certifi, virtualenv, virtualenv-clone, pipenv
Successfully installed certifi-2019.6.16 pipenv-2018.11.26 virtualenv-16.7.5 virtualenv-clone-0.5.3
You are using pip version 19.0.3, however version 19.2.3 is available.
You should consider upgrading via the 'python -m pip install --upgrade pip' command.

C:\workspace\github\dohsimpson\kubernetes-doc-pdf>pipenv install
Creating a virtualenv for this project…
Pipfile: C:\workspace\github\dohsimpson\kubernetes-doc-pdf\Pipfile
Using C:/Users/Poyatm01/AppData/Local/Programs/Python/Python37-32/python.exe (3.7.3) to create virtualenv…
[=   ] Creating virtual environment...Already using interpreter C:\Users\Poyatm01\AppData\Local\Programs\Python\Python37-32\python.exe
Using base prefix 'C:\\Users\\Poyatm01\\AppData\\Local\\Programs\\Python\\Python37-32'
New python executable in C:\Users\Poyatm01\.virtualenvs\kubernetes-doc-pdf-f5LapLRh\Scripts\python.exe
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter C:/Users/Poyatm01/AppData/Local/Programs/Python/Python37-32/python.exe

Successfully created virtual environment!
Virtualenv location: C:\Users\Poyatm01\.virtualenvs\kubernetes-doc-pdf-f5LapLRh
Installing dependencies from Pipfile.lock (c42e03)…
  ================================ 22/22 - 00:00:19
To activate this project's virtualenv, run pipenv shell.
Alternatively, run a command inside the virtualenv with pipenv run.

C:\workspace\github\dohsimpson\kubernetes-doc-pdf>pipenv shell
Launching subshell in virtual environment…
Microsoft Windows [Version 10.0.16299.1331]
(c) 2017 Microsoft Corporation. All rights reserved.

(kubernetes-doc-pdf-f5LapLRh) C:\workspace\github\dohsimpson\kubernetes-doc-pdf>python kubernetes-doc.py
Setup
downloading...
Traceback (most recent call last):
  File "kubernetes-doc.py", line 63, in <module>
    generate_directory_pdf(url, name)
  File "kubernetes-doc.py", line 39, in generate_directory_pdf
    f.write(html)
  File "C:\Users\Poyatm01\.virtualenvs\kubernetes-doc-pdf-f5LapLRh\lib\encodings\cp1252.py", line 19, in encode
    return codecs.charmap_encode(input,self.errors,encoding_table)[0]
UnicodeEncodeError: 'charmap' codec can't encode character '\u2714' in position 5290: character maps to <undefined>

import requests_html as rh
import os
import subprocess
import requests
import json
from pathlib import Path

# to change language, set the content of "lang" to iso code. Also, please check that it already exist on k8s web
lang = "es"

def generate_directory_pdf(url1, name, s=None):
    # some needed variables...
    mydir = Path(f"tmp/links_{name}")
    mydir.mkdir(parents=True, exist_ok=True)
    final_links_to_download = f"tmp/links_{name}/links_to_download.json"
    url2 = f"https://kubernetes.io/{lang}/docs/{name}"

    s = rh.HTMLSession() if not s else s
    r1 = s.get(url1)
    r2 = s.get(url2)
    html = ""
    anchors1 = r1.html.find('.td-sidebar-link')
    anchors2 = r2.html.find('.td-sidebar-link')
    links_en = [a.absolute_links.pop() for a in anchors1 if a.element.tag == 'a']
    links_es = [a.absolute_links.pop() for a in anchors2 if a.element.tag == 'a']

    links_en_uniq_a_comprobar = []
    for i in links_en:
        if i not in links_en_uniq_a_comprobar:
            links_en_uniq_a_comprobar.append(i)

    links_solo_es_uniq = []
    for i in links_es:
        if i not in links_solo_es_uniq:
            links_solo_es_uniq.append(i)

    links_es_uniq_a_comprobar = []
    links_es_uniq_a_comprobar = [link.replace("kubernetes.io/docs", "kubernetes.io/{lang}/docs") for link in links_en_uniq_a_comprobar]

    def check_url(tocheck):
        try:
            response = requests.get(tocheck, timeout=5)
            if response.status_code == 200:
                return True
            else:
                return False
        except requests.RequestException:
            return False

    checked_links_mixed = []
    for english, spanish in zip(links_en_uniq_a_comprobar, links_es_uniq_a_comprobar):
        if check_url(spanish):
            checked_links_mixed.append(spanish)
        else:
            checked_links_mixed.append(english)

    mixed_links_to_uniq = checked_links_mixed + links_solo_es_uniq
    filtered_mixed_links_for_lambda = []
    for i in mixed_links_to_uniq:
        if i not in filtered_mixed_links_for_lambda:
            filtered_mixed_links_for_lambda.append(i)


    links_post_lambda = filter(lambda href: href.startswith(url1) or href.startswith(url2), filtered_mixed_links_for_lambda)
    links_post_lambda_list = list(links_post_lambda)


    with open(final_links_to_download, 'w') as output_file:
                json.dump(links_post_lambda_list, output_file, indent=4)

    print("Downloading content from links...")
    cwd = os.getcwd()
    for l1 in links_post_lambda_list:
        r2 = s.get(l1)
        div = r2.html.find('.td-content', first=True, clean=True)
        if div:
            html += div.html
        with open("{}/{}.html".format(cwd, name), "wt") as f:
            f.write(html)

    print("generating pdf in " + name )
    subprocess.run(["{}/weasy_print.sh".format(cwd), name])


if __name__ == '__main__':
    s = rh.HTMLSession()
    directories = [\
                   "setup",
                   "concepts",
                   "tasks",
                   "tutorials",
                   "reference",
                   ]
    directories_pairs = [("https://kubernetes.io/docs/{}/".format(n.lower()), n) for n in directories]
    for url1, name in directories_pairs:
        print("Working with the content in url : " + url1)
        generate_directory_pdf(url1, name)

import requests_html as rh
import os
# import pypandoc
import subprocess
import time
import requests
import json
from pathlib import Path

def generate_directory_pdf(url1, name, s=None):
    mydir = Path(f"tmp/links_{name}")
    mydir.mkdir(parents=True, exist_ok=True)
    divss = f"tmp/links_{name}/divs.json"
    file_links_en_a_comprobar = f"tmp/links_{name}/listado_en_a_comprobar.json"
    file_links_es_a_comprobar = f"tmp/links_{name}/listado_es_a_comprobar.json"
    file_solo_es_comprobados = f"tmp/links_{name}/file_solo_es_comprobados.json"
    file_solo_en_comprobados = f"tmp/links_{name}/file_solo_en_comprobados.json"
    file_filtered_mixed_links_for_lambda = f"tmp/links_{name}/filtered_mixed_links_for_lambda.json"
    file_links_post_lambda_to_download = f"tmp/links_{name}/Final_post_lambda_links_to_download.json"
    file_links_post_lambda_LIST_to_download = f"tmp/links_{name}/Final_post_lambda_LIST_links_to_download.json"
    lang = "es"


    url2 = f"https://kubernetes.io/{lang}/docs/{name}"
    # Almacenamos en links todos las refrencias que encontramos en url:
    s = rh.HTMLSession() if not s else s
    r1 = s.get(url1)
    r2 = s.get(url2)
    html = ""
    anchors1 = r1.html.find('.td-sidebar-link')
    anchors2 = r2.html.find('.td-sidebar-link')
    links_en = [a.absolute_links.pop() for a in anchors1 if a.element.tag == 'a']
    links_es = [a.absolute_links.pop() for a in anchors2 if a.element.tag == 'a'] # todas las que ha encontrado en español

    # Uniq en el total de urls
    links_en_uniq_a_comprobar = []
    for i in links_en:
        if i not in links_en_uniq_a_comprobar:
            links_en_uniq_a_comprobar.append(i) # Limpiamos repeticiones de la misma url

    links_solo_es_uniq = []
    for i in links_es:
        if i not in links_solo_es_uniq:
            links_solo_es_uniq.append(i) # Limpiamos repeticiones de la misma url



    # Generamos una lista a_comprobar_en_to_es con todas las urls convertidas al esp:
    links_en_converted_to_es = []
    links_en_converted_to_es = [link.replace("kubernetes.io/docs", "kubernetes.io/es/docs") for link in links_en_uniq_a_comprobar]

    #Cambiamos nombre para facilitar nmbrado entre idiomas:
    links_es_uniq_a_comprobar = links_en_converted_to_es


    # Volcamos links_en_uniq_a_comprobar en fichero listado_en
    with open(file_links_en_a_comprobar, 'w') as output_file:
                json.dump(links_en_uniq_a_comprobar, output_file, indent=4)

    # Volcamos a_comprobar_en_to_es en fichero listado_es
    with open(file_links_es_a_comprobar, 'w') as output_file:
                json.dump(links_es_uniq_a_comprobar, output_file, indent=4)

    # Función que comprueba conexión del contenido de url
    def check_url(tocheck):
        try:
            response = requests.get(tocheck, timeout=5)
            if response.status_code == 200:
                return True
            else:
                return False
        except requests.RequestException:
            return False

    # Comprobamos conexión del contenido de links y de links_es
    # Se añaden a checked_links todos los que existen en español, y el resto en inglés
    checked_links_mixed = []
    links_es_comprobados = []
    links_en_comprobados = []
    for english, spanish in zip(links_en_uniq_a_comprobar, links_es_uniq_a_comprobar):
        #print(f"Se está probando las urls: {english}  y  {spanish}")
        if check_url(spanish):
            checked_links_mixed.append(spanish)
            links_es_comprobados.append(spanish)

            #print(f"Se añade la url {spanish} a Epanish y se descarta {english}")
        else:
            checked_links_mixed.append(english)
            links_en_comprobados.append(english)

    print(f"El listado de links en ESPAÑOL COMPROBADOS se puede mirar ya en el fichero file_solo_es_comprobados")
    with open(file_solo_es_comprobados, 'w') as output_file:
                json.dump(links_es_comprobados, output_file, indent=4)

    print(f"El listado de links en INGLÉS COMPROBADOS se puede mirar ya en el fichero file_solo_en_comprobados")
    with open(file_solo_en_comprobados, 'w') as output_file:
                json.dump(links_en_comprobados, output_file, indent=4)
    time.sleep(15)


    # Añadimos los links que SOLO se han encontrado en español
    mixed_links_to_uniq = checked_links_mixed + links_solo_es_uniq

    # Limpiamos posibles repeticiones entre mixed_links_to_download y links_solo_es_uniq
    filtered_mixed_links_for_lambda = []
    for i in mixed_links_to_uniq:
        if i not in filtered_mixed_links_for_lambda:
            filtered_mixed_links_for_lambda.append(i) # Limpiamos repeticiones de la misma url


# Tras esto, solo deberían quedar las urls que tienen conexión, mezcladas en dos idiomas

    #for i in links_to_download:
    #    print(f"Links comparados y añadidos a -solo español-, uno por uno (antes de lambda): {i}")

    print("Longitud de filtered_mixed_links_for_lambda:", len(filtered_mixed_links_for_lambda))
    print("final_total_links:", filtered_mixed_links_for_lambda)
    time.sleep(10)

    # Escribimos en el fichero el contenido final de las urls a descargar ANTES de LAMBDA:
    with open(file_filtered_mixed_links_for_lambda, 'w') as output_file:  # Cambié 'file' a 'output_file'
                json.dump(filtered_mixed_links_for_lambda, output_file, indent=4)
    print(f"Ya se pueden mirar los links previos a lambda en file_filtered_mixed_links_for_lambda")
    time.sleep(15)

    #yinks = filter(lambda href: href.startswith(url2), links_variable)
    links_post_lambda = filter(lambda href: href.startswith(url1) or href.startswith(url2), filtered_mixed_links_for_lambda)
    links_post_lambda_list = list(links_post_lambda)


    # Probar qué hay en list_yinks, una vez pasado el LAMBDA:
    with open(file_links_post_lambda_LIST_to_download, 'w') as output_file:  # Cambié 'file' a 'output_file'
                json.dump(links_post_lambda_list, output_file, indent=4)


    print("Ya se puede mirar el listado tras lambda en file_links_post_lambda_LIST_to_download")
    print("Longitud de links_post_lambda_list DESPUES de LAMBDA:", len(links_post_lambda_list))

    input("Presiona Enter para continuar...")

    print("downloading...")
    cwd = os.getcwd()
    for l1 in links_post_lambda_list:
        print(f"Final Links post Lambda: {l1}")
        r2 = s.get(l1)
        div = r2.html.find('.td-content', first=True, clean=True)
        print(f"Se están buscando los divs en {l1}. Esto es un div: {div}.")
        if div:
            print(f"Existe div: {div}")
            html += div.html
        with open("{}/{}.html".format(cwd, name), "wt") as f:
            f.write(html)

    print("generating pdf in " + name )
    subprocess.run(["{}/weasy_print.sh".format(cwd), name])

if __name__ == '__main__':
    s = rh.HTMLSession()
    directories = [\
                   "setup",
                   "concepts",
                   "tasks",
                   "tutorials",
                   "reference",
                   ]
    directories_pairs = [("https://kubernetes.io/docs/{}/".format(n.lower()), n) for n in directories]
    for url1, name in directories_pairs:
        print("URL: " + url1, "Directorio: " + name)
        print(name)
        generate_directory_pdf(url1, name)
        print("Generamos directorio con " + url1 + " y " + name )