modif. para uso de barras de progreso y tablas

sale modulo colorama, entra mod rich
This commit is contained in:
jp.av.dev 2022-05-20 20:22:00 -04:00
parent 814b40883f
commit ebff4c2a74
3 changed files with 353 additions and 262 deletions

0
__init__.py Normal file
View File

View File

@ -7,9 +7,10 @@ import requests
import re import re
import configparser as cfg import configparser as cfg
from os.path import isfile from os.path import isfile
from json import loads
from colorama import Fore, Back, Style
import sql_alch import sql_alch
from rich import box
from rich.console import Console
from rich.table import Table
selfpath = os.path.abspath(os.path.dirname(__file__)) selfpath = os.path.abspath(os.path.dirname(__file__))
ownip = requests.get('https://ifconfig.me/').text ownip = requests.get('https://ifconfig.me/').text
@ -18,35 +19,9 @@ parser.read(f'{selfpath}/config.cfg')
token = parser.get('iplocate','token') token = parser.get('iplocate','token')
token = token.strip("'") token = token.strip("'")
muevelog = f'{selfpath}/muevelog.sh ' muevelog = f'{selfpath}/muevelog.sh '
console = Console()
#tkn=True #tkn=True
# Colores
co_rst = Style.RESET_ALL
co_Blu = Fore.BLUE+Style.NORMAL
co_BluD = Fore.BLUE+Style.DIM
co_BluB = Fore.BLUE+Style.BRIGHT
co_Red = Fore.RED+Style.NORMAL
co_RedD = Fore.RED+Style.DIM
co_RedB = Fore.RED+Style.BRIGHT
co_Grn = Fore.GREEN+Style.NORMAL
co_GrnD = Fore.GREEN+Style.DIM
co_GrnB = Fore.GREEN+Style.BRIGHT
co_Yel = Fore.YELLOW+Style.NORMAL
co_YelD = Fore.YELLOW+Style.DIM
co_YelB = Fore.YELLOW+Style.BRIGHT
co_BluLWh = Fore.BLUE+Back.LIGHTWHITE_EX+Style.NORMAL
co_BluLWhB = Fore.BLUE+Back.LIGHTWHITE_EX+Style.BRIGHT
co_GrnMgnB = Fore.GREEN+Back.MAGENTA+Style.BRIGHT
co_RedBluD = Fore.RED+Back.BLUE+Style.DIM
co_LRedBleD = Fore.LIGHTRED_EX+Back.BLUE+Style.DIM
co_BlkMgnB = Fore.BLACK+Back.MAGENTA+Style.BRIGHT
co_BlkMgn = Fore.BLACK+Back.MAGENTA+Style.NORMAL
co_LGrLBlk = Fore.LIGHTGREEN_EX+Back.LIGHTBLACK_EX
co_RdYl = Fore.RED+Back.YELLOW+Style.NORMAL
co_RdYlB = Fore.RED+Back.YELLOW+Style.BRIGHT
co_BlkLBlkD = Fore.BLACK+Back.LIGHTBLACK_EX+Style.DIM
co_BluLBlkD = Fore.BLUE+Back.LIGHTBLACK_EX+Style.DIM
co_cuBlu = '\33[38;5;122m'
# IP validate https://stackoverflow.com/questions/319279/how-to-validate-ip-address-in-python # IP validate https://stackoverflow.com/questions/319279/how-to-validate-ip-address-in-python
ip_regx = "^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])\.){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])$" ip_regx = "^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])\.){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])$"
@ -54,97 +29,132 @@ def filtro_ip_propia(ip):
return True if ip != ownip else False return True if ip != ownip else False
def consulta_ip(ip_consulta, tkn=True):
if (re.search(ip_regx, ip_consulta)):
match tkn:
case True:
consulta = f'https://ipinfo.io/{ip_consulta}{token}'
info_ip = requests.get(consulta).text
return loads(info_ip)
case False:
consulta = f'https://ipinfo.io/{ip_consulta}'
info_ip = requests.get(consulta).text
return loads(info_ip)
case None:
resp = sql_alch.consulta_db(ip_consulta)
#print('consulta_ip :', resp)
return resp
# aqui va la consulta a base de datos
def print_ipinfo(ip, tkn=True): def print_ipinfo(ip, tkn=True):
if (re.search(ip_regx, ip)): if (re.search(ip_regx, ip)):
ip_info = consulta_ip(ip, tkn) try:
ip_info = sql_alch.consulta_ip(ip, tkn)
except Exception as ex:
print(f'Exception sql_alch.consulta_ip({ip})\n',ex)
ip_info = None
if isinstance(ip_info, dict): if isinstance(ip_info, dict):
for llave, valor in ip_info.items(): #print('es dict')
print(f'{co_YelB}{llave}\b\t{co_Blu}->{co_rst} {co_Grn}{valor}{co_rst}') print_tabla_ip(ip_info)
print(f'{co_RedB}------------------------------', end='')
print(f'--------------------------------{co_rst}')
elif isinstance(ip_info, list): elif isinstance(ip_info, list):
#print('es lista')
lista_visitas=[]
contad=0 contad=0
for tupla in ip_info: for tupla in ip_info:
visita = []
if contad < 1: if contad < 1:
for ind, val in enumerate(tupla): for ind, val in enumerate(tupla):
if ind == 0: if ind == 0:
print(f'{co_BluD} _____________________________' ip_dict = dict()
f'________________________________{co_rst}')
for dato in str(val).split(';'): for dato in str(val).split(';'):
print(f'{co_Blu}| {co_BluB}{dato.split("=")[0].ljust(12)}' ip_dict[dato.split("=")[0]] = dato.split("=")[1]
f'{co_Blu}| {co_rst}{dato.split("=")[1]}{co_rst}') print_tabla_ip(ip_dict)
print(f'{co_BluD} ________________________ __'
f'_________ __________ _____________{co_rst}')
print(f'{co_Blu}|{co_YelB} Fecha Visita {co_rst}'
f'{co_Blu}|{co_YelB}Codigo html{co_Blu}|'
f'{co_YelB} Metodo {co_rst}'
f'{co_Blu}|{co_YelB} Request {co_rst}')
print(f'{co_Blu}|------------------------|--'
f'---------|----------|-------------{co_rst}')
else: else:
# aqui modificar para representar las nuevas columnas de tabla visita visita.append(str(val).split(',')[3].split('=')[1]) #.ljust(24))
codig = str(val).split(',')[2].split('=')[1].center(11) visita.append(str(val).split(',')[2].split('=')[1]) #.center(11))
fecha = str(val).split(',')[3].split('=')[1].ljust(24)
metodo = (str(val).split(',')[4].split('=')[1]) metodo = (str(val).split(',')[4].split('=')[1])
metodo = '---'.center(10) if metodo == 'None' else metodo.center(10) metodo = '---' if metodo == 'None' else metodo #.center(10)
request = str(val).split(',')[5].split('=')[1] visita.append(metodo)
request = request[:86]+'...' if len(request) > 90 else request request = ''.join(str(val).split(',')[5].split('=')[1:])
# configurar wrap en tabla
#request = request[:86]+'...' if len(request) > 90 else request
request = '---' if request == 'None' else request request = '---' if request == 'None' else request
#if len(request) > 90: visita.append(request)
# request = request[:86]+'...'
print(f'{co_Blu}|{co_Yel}{fecha}{co_rst}'
f'{co_Blu}|{co_GrnB}{codig}'
f'{co_Blu}|{co_Grn}{metodo}{co_rst}'
f'{co_Blu}|{co_Grn}{request}{co_rst}')
else: else:
for ind, val in enumerate(tupla): for ind, val in enumerate(tupla):
if ind > 0: if ind > 0:
# aqui modificar para representar las nuevas columnas de tabla visita # aqui modificar para representar las nuevas columnas de tabla visita
codig = str(val).split(',')[2].split('=')[1].center(11) visita.append(str(val).split(',')[3].split('=')[1]) #.ljust(24)
fecha = str(val).split(',')[3].split('=')[1].ljust(24) visita.append(str(val).split(',')[2].split('=')[1]) #.center(11)
metodo = (str(val).split(',')[4].split('=')[1]) metodo = (str(val).split(',')[4].split('=')[1])
metodo = '---'.center(10) if metodo == 'None' else metodo.center(10) metodo = '---' if metodo == 'None' else metodo #.center(10)
request = str(val).split(',')[5].split('=')[1] visita.append(metodo)
request = request[:86]+'...' if len(request) > 90 else request request = ''.join(str(val).split(',')[5].split('=')[1:])
# configurar wrap en tabla
#request = request[:86]+'...' if len(request) > 90 else request
request = '---' if request == 'None' else request request = '---' if request == 'None' else request
print(f'{co_Blu}|{co_Yel}{fecha}{co_rst}' visita.append(request)
f'{co_Blu}|{co_GrnB}{codig}' lista_visitas.append(visita)
f'{co_Blu}|{co_Grn}{metodo}{co_rst}' contad+=1
f'{co_Blu}|{co_Grn}{request}{co_rst}') print_tabla_visita(lista_visitas)
contad+=1
print(f'{co_RedB}-------------------------------'
f'-------------------------------{co_rst}')
else: else:
print('otra wea: ', type(ip_info)) console.print(f'[red]Error type(ip_info) = [/red][magenta]{type(ip_info)}[/magenta][red]][/red]')
else: else:
ipr = ip.split('\n')[0] ipr = ip.split('\n')[0]
print(f'{co_Red}[{co_BlkMgn}{ipr}{co_rst}{co_Red}] no es una IP válida!{co_rst}') console.print(f'[red][[/red][magenta]{ipr}[/magenta][red]] no es una IP válida![/red]')
def print_tabla_ip(ip_info):
# color dodger_blue2
tbl_ip = Table(box = box.ROUNDED, highlight=True, border_style="dim plum1")
tbl_ip.add_column("IP", justify="left", style="bold #005fff", no_wrap=True)
tbl_ip.add_column(f"{ip_info['ip']}", justify="left", style="#00ff5f")
try:
if 'host' in ip_info:
tbl_ip.add_row("HOSTNAME", ip_info['host'])
elif 'hostname' in ip_info:
tbl_ip.add_row("HOSTNAME", ip_info['hostname'])
if 'anycast' in ip_info:
anycast = 'Si' if ip_info['anycast'] else 'No'
tbl_ip.add_row("ANYCAST", anycast)
if 'cuidad' in ip_info:
tbl_ip.add_row("CUIDAD", ip_info['cuidad'])
elif 'city' in ip_info:
tbl_ip.add_row("CUIDAD", ip_info['city'])
if 'region' in ip_info:
tbl_ip.add_row("REGION", ip_info['region'])
if 'pais' in ip_info:
tbl_ip.add_row("PAIS", ip_info['pais'])
elif 'country' in ip_info:
tbl_ip.add_row("PAIS", ip_info['country'])
if 'geoloc' in ip_info:
tbl_ip.add_row("GEOLOC", ip_info['geoloc'])
elif 'loc' in ip_info:
tbl_ip.add_row("GEOLOC", ip_info['loc'])
if 'organizacion' in ip_info:
tbl_ip.add_row("ORGANIZ.", ip_info['organizacion'])
elif 'org' in ip_info:
tbl_ip.add_row("ORGANIZ.", ip_info['org'])
if 'fecha_reg' in ip_info:
tbl_ip.add_row("FECHA REG", ip_info['fecha_reg'])
if 'tzone' in ip_info:
tbl_ip.add_row("TimeZone", ip_info['tzone'])
elif 'timezone' in ip_info:
tbl_ip.add_row("TimeZone", ip_info['timezone'])
if 'cod_post' in ip_info:
tbl_ip.add_row("COD POST", ip_info['cod_post'])
elif 'postal' in ip_info:
tbl_ip.add_row("COD POST", ip_info['postal'])
except Exception as ex:
print('Exception ipl.print_tabla_ip(): ', ex)
try:
console.print(tbl_ip)
except Exception as ex:
print('Exception print(tabla_ip): ', ex)
def print_tabla_visita(lista_visitas):
# color dodger_blue2
#tbl_v = Table(title=f"[bold][yellow]Visitas registradas:[/yellow] [green]{lista_visitas[0]}[/bold][/green]",
tbl_v = Table(box = box.ROUNDED, show_lines = False,row_styles=["dim", ""], border_style="dark_orange3")
tbl_v.add_column("Fecha visita", justify="center", style="bright_yellow", no_wrap=True)
tbl_v.add_column("Codigo", justify="center", style="bold dodger_blue2")
tbl_v.add_column("Metodo", justify="center", style="bright_magenta")
tbl_v.add_column("Request", justify="left", style="#00ff5f", overflow='fold', no_wrap=False)
for item in lista_visitas:
tbl_v.add_row(item[0], item[1], item[2], item[3])
console.print(tbl_v)
def archivo_ips(ips, tkn=True): def archivo_ips(ips, tkn=True):
with open(ips, 'r') as lista: with open(ips, 'r') as lista:
for linea in lista: for linea in lista:
if '\n' in linea: if '\n' in linea:
linea = linea.split('\n')[0] linea = linea.split('\n')[0]
print_ipinfo(linea, tkn) print_ipinfo(linea, tkn)
sys.exit(0) sys.exit(0)
@ -153,41 +163,41 @@ def main():
try: try:
match sys.argv[1]: match sys.argv[1]:
case '--sync': case '--sync':
print(f'{co_YelB}Sincronizando logs del servidor(bash script){co_rst}') console.print('[bold yellow]Sincronizando logs del servidor(bash script)[/bold yellow]')
subprocess.check_call( subprocess.check_call(
muevelog+"%s" % "--start", muevelog+"%s" % "--start",
shell=True) shell=True)
case '-c': case '-c':
print(f'{co_YelB}Cargando logs en base de datos{co_rst}') console.print('[bold yellow]Cargando logs en base de datos[/bold yellow]')
sql_alch.carga_logs() sql_alch.carga_logs()
case '-g':
console.print('[yellow]Registro de datos de ipinfo[/yellow]')
sql_alch.registro_ips()
case '-d': case '-d':
print(f'{co_YelB}Consulta base de datos:{co_rst}') console.print('[bold yellow]Consulta a base de datos:[/bold yellow]')
ip = sys.argv[2] ip = sys.argv[2]
print_ipinfo(ip, None) print_ipinfo(ip, None)
case '-D': case '-D':
print(f'{co_YelB}Consulta por archivo en base de datos:{co_rst}') console.print('[bold yellow]Consulta por archivo a base de datos:[/bold yellow]')
if isfile(sys.argv[2]): if isfile(sys.argv[2]):
archivo_ips(sys.argv[2], None) archivo_ips(sys.argv[2], None)
else: else:
print(f'{co_Red}Archivo [{co_BlkMgn}{sys.argv[2]}'+ console.print(f'[red]Archivo [[/red][magenta]{sys.argv[2]}[/magenta]'
f'{co_rst}{co_Red}] no es válido''') '[red]] no es válido![/red]')
sys.exit(0) sys.exit(0)
case '-g':
print(f'{co_YelB}Registrando datos de ipinfo{co_rst}')
sql_alch.registro_ips()
case '-f': case '-f':
if isfile(sys.argv[2]): if isfile(sys.argv[2]):
archivo_ips(sys.argv[2], False) archivo_ips(sys.argv[2], False)
else: else:
print(f'{co_Red}Archivo [{co_BlkMgn}{sys.argv[2]}'+ console.print(f'[red]Archivo [[/red][magenta]{sys.argv[2]}[/magenta]'
f'{co_rst}{co_Red}] no es válido''') '[red]] no es válido[/red]')
sys.exit(0) sys.exit(0)
case '-F': case '-F':
if isfile(sys.argv[2]): if isfile(sys.argv[2]):
archivo_ips(sys.argv[2]) archivo_ips(sys.argv[2])
else: else:
print(f'{co_Red}Archivo [{co_BlkMgn}{sys.argv[2]}'+ console.print(f'[red]Archivo [[/red][magenta]{sys.argv[2]}[/magenta]'
f'{co_rst}{co_Red}] no es válido''') '[red]] no es válido[/red]')
sys.exit(0) sys.exit(0)
case '-h': case '-h':
uso() uso()
@ -199,14 +209,16 @@ def main():
ip = sys.argv[1] ip = sys.argv[1]
print_ipinfo(ip, False) print_ipinfo(ip, False)
except IndexError: except IndexError:
print(f'{co_Red} error sys.args! {co_rst}') console.print('[red] error sys.args! [/red]')
finally: finally:
sys.exit(0) sys.exit(0)
print(f'{co_Grn}Ingresa una {co_BluB}IP {co_Grn}o {co_BluB}s '+ console.print("[green]Ingresa una[/green] [bold blue]IP [/bold blue]"
f'{co_Grn}para salir:{co_rst}') "[green]o [/green][bold blue]s [/bold blue]"
"[green]para salir:[/green]")
while True: while True:
ip = input(f'{co_BluB} -> {co_rst}') console.print("[bold blue] -> [/bold blue]", end='')
ip = input()
if ip in 'sq0SQnN': if ip in 'sq0SQnN':
exit(0) exit(0)
print_ipinfo(ip) print_ipinfo(ip)
@ -214,30 +226,30 @@ def main():
def uso(): def uso():
ayuda = f""" ayuda = f"""
{co_BluB}ipLocate{co_rst} [bold blue]ipLocate[/bold blue]
{co_cuBlu}Consulta información sobre IP(s) disponibles en ipinfo.io con o sin token. [deep_sky_blue1]Consulta información sobre IP(s) disponibles en ipinfo.io con o sin token.
Carga logs de nginx en base de datos. Consulta con ipinfo.io y registra Carga logs de nginx en base de datos. Consulta con ipinfo.io y registra
en base de datos. en base de datos.
Consultas y reportes según información en la base de datos.{co_rst} Consultas y reportes según información en la base de datos.[/deep_sky_blue1]
{co_YelB}iploc -h {co_Grn}- Muestra esta ayuda.{co_rst} [bold yellow]iploc -h[/bold yellow] [green]- Muestra esta ayuda.[/green]
[bold blue]Uso para consultas:[/bold blue]
[bold yellow]iploc[/bold yellow] [blue]<IP>[/blue] [green]- Consulta la información de <IP> disponible en ipinfo.io.[/green]
[bold yellow]iploc -t [/bold yellow][blue]<IP>[/blue] [green]- Consulta la info. de <IP> usando 'token' de ipinfo.io,
especificado en config.cfg.[/green]
[bold yellow]iploc -d [/bold yellow][blue]<IP> [/blue] [green]- Consulta la información de <IP> disponible en base de datos.[/green]
[bold yellow]iploc -f [/bold yellow][blue]<archivo> [/blue] [green]- Consulta info. de las IPs en <archivo> (ipinfo.io).[/green]
[bold yellow]iploc -F [/bold yellow][blue]<archivo> [/blue] [green]- Consulta info. de las IPs en <archivo> (token, ipinfo.io).[/green]
[bold yellow]iploc -D [/bold yellow][blue]<archivo> [/blue] [green]- Consulta info. de las IPs en <archivo> (base de datos).[/green]
{co_BluB}Uso para consultas:{co_rst} [bold blue]Operaciones base de datos:[/bold blue]
{co_YelB}iploc {co_Blu}<IP> {co_Grn}- Consulta la información de <IP> disponible en ipinfo.io.{co_rst} [bold yellow]iploc --sync [/bold yellow][green]- Sincroniza logs del servidor (bash script).[/green]
{co_YelB}iploc -t {co_Blu}<IP> {co_Grn}- Consulta la info. de <IP> usando 'token' de ipinfo.io, [bold yellow]iploc -c [/bold yellow][green]- Carga logs en base de datos.[/green]
especificado en config.cfg.{co_rst} [bold yellow]iploc -g [/bold yellow][green]- Guarda ipinfo de IPs sin registro en la BD.[/green]
{co_YelB}iploc -d {co_Blu}<IP> {co_Grn}- Consulta la información de <IP> disponible en base de datos.{co_rst}
{co_YelB}iploc -f {co_Blu}<archivo> {co_Grn}- Consulta info. de las IPs en <archivo> (ipinfo.io).{co_rst}
{co_YelB}iploc -F {co_Blu}<archivo> {co_Grn}- Consulta info. de las IPs en <archivo> (token, ipinfo.io).{co_rst}
{co_YelB}iploc -D {co_Blu}<archivo> {co_Grn}- Consulta info. de las IPs en <archivo> (base de datos).{co_rst}
{co_BluB}Operaciones base de datos:{co_rst}
{co_YelB}iploc --sync {co_Grn}- Sincroniza logs del servidor (bash script).{co_rst}
{co_YelB}iploc -c {co_Grn}- Carga logs en base de datos.{co_rst}
{co_YelB}iploc -g {co_Grn}- Guarda ipinfo de IPs sin registro en la BD.{co_rst}
""" """
print(ayuda) console.print(ayuda)
if __name__ == "__main__": if __name__ == "__main__":
main() main()

View File

@ -1,5 +1,8 @@
import os import os
import time import time
import subprocess
from iplocate import re, requests, token, filtro_ip_propia, selfpath, parser
from json import loads
from datetime import datetime from datetime import datetime
from sqlalchemy import create_engine from sqlalchemy import create_engine
from sqlalchemy.orm import relationship, sessionmaker from sqlalchemy.orm import relationship, sessionmaker
@ -8,12 +11,15 @@ from sqlalchemy import Column, Integer, String, Sequence, update
from sqlalchemy.orm.session import Session from sqlalchemy.orm.session import Session
from sqlalchemy.sql.expression import select from sqlalchemy.sql.expression import select
from sqlalchemy.sql.schema import ForeignKey from sqlalchemy.sql.schema import ForeignKey
import iplocate as ipl from rich.progress import Progress, track
from rich.console import Console
logs_dir = ipl.parser.get('bash_script', 'destino_log') ip_regx = "^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])\.){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])$"
logs_dir = parser.get('bash_script', 'destino_log')
logs_dir = logs_dir.strip("'") logs_dir = logs_dir.strip("'")
base_de_datos = f'sqlite:////{ipl.selfpath}/ipinfo.db' base_de_datos = f'sqlite:////{selfpath}/ipinfo.db'
console = Console()
Base = declarative_base() Base = declarative_base()
# Tabla registro ip info # Tabla registro ip info
@ -139,77 +145,115 @@ def ip_registrada(ip):
def carga_access_log(log): def carga_access_log(log):
if os.path.exists(log): if os.path.exists(log):
print(f'{ipl.co_Yel}Registrando [{log}]{ipl.co_rst}') nombre_log = log.split('/')[-1]
console.print(f'[bold yellow]Registrando [[/bold yellow]{nombre_log}[bold yellow]][/bold yellow]')
try: try:
with open(log, 'r') as lista: with open(log, 'r') as lista:
for linea in lista: try:
ip = linea.split(' ')[0] largo = subprocess.run(['wc', '-l', log], capture_output=True, text=True)
if ipl.filtro_ip_propia(ip): largo = int(largo.stdout.split(' ')[0])
try: for linea in track(lista, total=largo, description='[blue bold]Cargando [/blue bold]'):
ip = linea.split(' ')[0] ip = linea.split(' ')[0]
except Exception as ex: if filtro_ip_propia(ip):
ip = None try:
print('Exception split IP', ex) ip = linea.split(' ')[0]
try: except Exception as ex:
metodo = linea.split('"')[1].split(' ')[0] ip = None
if len(metodo) > 10 or len(metodo) < 2: print('Exception split IP', ex)
try:
metodo = linea.split('"')[1].split(' ')[0]
if len(metodo) > 10 or len(metodo) < 2:
metodo = '---'
except Exception as ex:
metodo = '---' metodo = '---'
except Exception as ex: try:
metodo = '---' url = linea.split('"')[1].split(' ')[1]
try: if len(url) > 254:
url = linea.split('"')[1].split(' ')[1] url = url[:252]+'...'
if len(url) > 254: except Exception as ex:
url = url[:252]+'...' url = '---'
except Exception as ex: try:
url = '---' codigo = int(linea.split('"')[2].split(' ')[1])
try: if len(str(codigo)) != 3:
codigo = int(linea.split('"')[2].split(' ')[1]) codigo = 0
if len(str(codigo)) != 3: except Exception as ex:
codigo = 0 codigo = 0
except Exception as ex: try:
codigo = 0 fecha = linea.split(' ')[3][1:]
try: fecha = fecha_access_to_epoch(fecha)
fecha = linea.split(' ')[3][1:] except Exception as ex:
fecha = fecha_access_to_epoch(fecha) fecha = None
except Exception as ex: print('Exception split Fecha:', ex)
fecha = None if ip_registrada(ip):
print('Exception split Fecha:', ex) session.add(Visita(ip=ip,
if ip_registrada(ip): cod_html=codigo,
session.add(Visita(ip=ip, fecha=fecha,
cod_html=codigo, metodo=metodo,
fecha=fecha, consulta=url,
metodo=metodo, registro=1))
consulta=url, else:
registro=1)) session.add(Visita(ip=ip,
else: cod_html=codigo,
session.add(Visita(ip=ip, fecha=fecha,
cod_html=codigo, metodo=metodo,
fecha=fecha, consulta=url))
metodo=metodo, except Exception as ex:
consulta=url)) print('Exception: ', ex)
session.commit() try:
print(f'{ipl.co_Grn}Carga completa.. borrando log{ipl.co_rst}\n') with Progress() as prog, session:
task1=prog.add_task("[yellow bold]Guardando[/yellow bold]", total=len(session.new))
session.commit()
while not prog.finished:
prog.update(task1, advance=0.1)
time.sleep(0.05)
except Exception as ex:
print('Exception Progress: ', ex)
console.print('[bold green] - Carga completa.. borrando log[/bold green]\n')
os.remove(log) os.remove(log)
return True return True
except: except:
print(f'{ipl.co_Red}Error al intentar abrir/cargar: '+ console.print(f'[red]Error al intentar abrir/cargar: [{log}[/red]]\n')
f'[{ipl.co_rst}{log}{ipl.co_Red}]\n')
return False return False
else: else:
print(f'{ipl.co_RedB}log: [{ipl.co_rst}{log}{ipl.co_RedB}]'+ console.print(f'[bold red]Log: [[/bold red]{log}[bold red]] inexistente.[/bold red]\n')
f' inexistente.{ipl.co_rst}\n') return False
return False
def carga_error_logs(log): def carga_error_logs(log):
if os.path.exists(log): if os.path.exists(log):
print(f'{ipl.co_Yel}Registrando [{log}]{ipl.co_rst}') nombre_log = log.split('/')[-1]
console.print(f'[bold yellow]Registrando [[/bold yellow]{nombre_log}[bold yellow]][/bold yellow]')
try: try:
#with Progress(), open(log, 'r') as lista:
with open(log, 'r') as lista: with open(log, 'r') as lista:
for linea in lista: try:
linea = linea.split('\n')[0] largo = subprocess.run(['wc', '-l', log], capture_output=True, text=True)
if (linea.rfind('[notice]') > 0 or linea.rfind('[crit]') > 0): largo = int(largo.stdout.split(' ')[0])
if linea.find('[crit]') > 0: for linea in track(lista, total=largo, description='[blue bold]Cargando [/blue bold]'):
linea = linea.split('\n')[0]
if (linea.rfind('[notice]') > 0 or linea.rfind('[crit]') > 0):
if linea.find('[crit]') > 0:
try:
ip = linea.split('client: ')[1].split(',')[0]
except Exception as ex:
print('Exception Ip error_log: ', ex)
ip = None
try:
fecha = ' '.join(linea.split(' ')[0:2])
except Exception:
fecha = None
try:
url = linea.split('"')[1].split(' ')[1]
if len(url) > 254:
url = url[:252]+'...'
except Exception:
url = ' '.join(linea.split(' ')[5:])
#url = '---'
try:
metodo = linea.split('"')[1].split(' ')[0]
except Exception:
metodo = '---'
else:
try: try:
ip = linea.split('client: ')[1].split(',')[0] ip = linea.split('client: ')[1].split(',')[0]
except Exception as ex: except Exception as ex:
@ -220,72 +264,67 @@ def carga_error_logs(log):
except Exception: except Exception:
fecha = None fecha = None
try: try:
url = linea.split('"')[1].split(' ')[1] metodo = linea.split('request: "')[1].split(' ')[0]
except Exception:
metodo = '---'
try:
url = linea.split('"')[1].split(' ')[0]
if len(url) > 254: if len(url) > 254:
url = url[:252]+'...' url = url[:252]+'...'
except Exception: except Exception:
url = ' '.join(linea.split(' ')[5:]) url = '---'
#url = '---' if filtro_ip_propia(ip):
try: fecha = int(fecha_error_to_epoch(fecha))
metodo = linea.split('"')[1].split(' ')[0] codigo = 0
except Exception: if ip_registrada(ip):
metodo = '---' session.add(Visita(ip=ip,
else: cod_html=codigo,
try: fecha=fecha,
ip = linea.split('client: ')[1].split(',')[0] consulta=url,
except Exception as ex: metodo=metodo,
print('Exception Ip error_log: ', ex) registro=1))
ip = None else:
try: session.add(Visita(ip=ip,
fecha = ' '.join(linea.split(' ')[0:2]) cod_html=codigo,
except Exception: fecha=fecha,
fecha = None consulta=url,
try: metodo=metodo))
metodo = linea.split('request: "')[1].split(' ')[0] except Exception as ex:
except Exception: print('Exception: ', ex)
metodo = '---' try:
try: with Progress() as prog, session:
url = linea.split('"')[1].split(' ')[0] task1=prog.add_task("[yellow bold]Guardando[/yellow bold]", total=len(session.new))
if len(url) > 254: session.commit()
url = url[:252]+'...' while not prog.finished:
except Exception: prog.update(task1, advance=0.1)
url = '---' time.sleep(0.05)
if ipl.filtro_ip_propia(ip): except Exception as ex:
fecha = int(fecha_error_to_epoch(fecha)) print('Exception Progress: ', ex)
codigo = 0 console.print(f'[bold green] - Carga completa.. borrando log[/bold green]\n')
if ip_registrada(ip):
session.add(Visita(ip=ip,
cod_html=codigo,
fecha=fecha,
consulta=url,
metodo=metodo,
registro=1))
else:
session.add(Visita(ip=ip,
cod_html=codigo,
fecha=fecha,
consulta=url,
metodo=metodo))
session.commit()
print(f'{ipl.co_Grn}Carga completa.. borrando log{ipl.co_rst}\n')
os.remove(log) os.remove(log)
return True return True
except: except:
print(f'{ipl.co_Red}Error al intentar abrir/cargar: '+ console.print(f'[red]Error al intentar abrir/cargar: [{log}[/red]]\n')
f'[{ipl.co_rst}{log}{ipl.co_Red}]\n')
return False return False
else: else:
print(f'{ipl.co_RedB}log: [{ipl.co_rst}{log}{ipl.co_RedB}]'+ console.print(f'[bold red]Log: [[/bold red]{log}[bold red]] inexistente.[/bold red]\n')
f' inexistente.{ipl.co_rst}\n')
return False return False
def carga_logs(): def carga_logs():
print(f'{ipl.co_Grn}Carga de logs en base de datos:{ipl.co_rst}\n') #print(f'[bold green]Carga de logs en base de datos:[/bold green]\n')
carga_access_log(logs_dir+'/access.log') logpath = logs_dir+'/access.log'
carga_access_log(logs_dir+'/reverse-access.log') if os.path.exists(logpath):
carga_error_logs(logs_dir+'/error.log') carga_access_log(logpath)
carga_error_logs(logs_dir+'/reverse-error.log') logpath = logs_dir+'/reverse-access.log'
if os.path.exists(logpath):
carga_access_log(logpath)
logpath = logs_dir+'/error.log'
if os.path.exists(logpath):
carga_error_logs(logpath)
logpath = logs_dir+'/reverse-error.log'
if os.path.exists(logpath):
carga_error_logs(logpath)
def carga_registro_ip(ip_info): def carga_registro_ip(ip_info):
@ -317,15 +356,33 @@ def carga_registro_ip(ip_info):
session.commit() session.commit()
except Exception as ex: except Exception as ex:
print('Exception: ', ex) print('Exception: ', ex)
# aqui progress bar registro visita
stmt = update(Visita).where(Visita.ip == ip_info['ip']).values(registro=1).\ stmt = update(Visita).where(Visita.ip == ip_info['ip']).values(registro=1).\
execution_options(synchronize_session="fetch") execution_options(synchronize_session="fetch")
#result = session.execute(stmt) #result = session.execute(stmt)
try: try:
session.execute(stmt) session.execute(stmt)
# aqui progress bar update visita (registro = 1)
session.commit() session.commit()
except Exception as ex: except Exception as ex:
print('Exception: ', ex) print('Exception: ', ex)
def consulta_ip(ip_consulta, tkn=True):
if (re.search(ip_regx, ip_consulta)):
match tkn:
case True:
consulta = f'https://ipinfo.io/{ip_consulta}{token}'
info_ip = requests.get(consulta).text
return loads(info_ip)
case False:
consulta = f'https://ipinfo.io/{ip_consulta}'
info_ip = requests.get(consulta).text
return loads(info_ip)
case None:
resp = consulta_db(ip_consulta)
return resp
def consulta_db(ip): def consulta_db(ip):
try: try:
statement = session.query(Registro, Visita).join('visitas').filter_by(ip=ip) statement = session.query(Registro, Visita).join('visitas').filter_by(ip=ip)
@ -335,25 +392,47 @@ def consulta_db(ip):
print('Exception consulta_db:\n', ex) print('Exception consulta_db:\n', ex)
def test_db():
try:
session.add(Visita(ip='dummy_ip', cod_html=000, fecha=int(time.mktime(time.localtime()))))
session.commit()
session.add(Registro(ip ='dummy_ip'))
session.commit()
except Exception as ex:
print('Exception: ', ex)
def registro_ips(): def registro_ips():
registrar = True statement = select(Visita).filter_by(registro=0)
while registrar: with Progress() as progress:
statement = select(Visita).filter_by(registro=0) total = len(session.execute(statement).scalars().all())
res = session.execute(statement).scalars().first() task1= progress.add_task("[bold blue]Cargando [/bold blue]", total=total)
if res is None: total_ant = total
print(f'{ipl.co_Grn}Registro ipinfo en DB finzalizado.{ipl.co_rst}') while not progress.finished:
registrar = False res = session.execute(statement).scalars().first()
#ip_actual= res.ip.split('\n')[0] total_act = len(session.execute(statement).scalars().all())
ip_actual= res.ip avance = total_ant - total_act
ip_info = ipl.consulta_ip(ip_actual, True) print('total update:',total,'total_act:', total_act,' Diferencia: ', avance )
carga_registro_ip(ip_info) total_ant = total_act
if res is None:
print('en res none')
progress.update (task1, advance=total)
ip_actual= res.ip
ip_info = consulta_ip(ip_actual, True)
carga_registro_ip(ip_info)
progress.update(task1, advance=avance)
console.print('\n[bold yellow]Registro en baso de datos finalizado.[/bold yellow]')
#def registro_ips(): # OLD
# registrar = True
# statement = select(Visita).filter_by(registro=0)
# while registrar:
# res = session.execute(statement).scalars().first()
# if res is None:
# console.print('[bold yellow]Registro en baso de datos finalizado.[/bold yellow]')
# registrar = False
# ip_actual= res.ip
# ip_info = consulta_ip(ip_actual, True)
# carga_registro_ip(ip_info)
#def test_db():
# try:
# session.add(Visita(ip='dummy_ip', cod_html=000, fecha=int(time.mktime(time.localtime()))))
# session.commit()
# session.add(Registro(ip ='dummy_ip'))
# session.commit()
# except Exception as ex:
# print('Exception: ', ex)