272 lines
10 KiB
Python
272 lines
10 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Импорт книг из inpx-файла lib.rus.ec в PostgreSQL.
|
|
|
|
Использование:
|
|
python parse_inpx.py <inpx.zip> <postgres_connection_string> <source>
|
|
|
|
Пример:
|
|
python parse_inpx.py \
|
|
"/mnt/hdd/share/Torrents/Downloaded/Libruks/Библиотека Либрусек для архивов 18-09-2026.inpx" \
|
|
"dbname=books user=postgres password=Karabas@Barabas host=172.20.0.4" \
|
|
"lib.rus.ec"
|
|
"""
|
|
import zipfile
|
|
import psycopg2
|
|
from psycopg2.extras import execute_values
|
|
import sys
|
|
import hashlib
|
|
|
|
|
|
def normalize_author_name(author_name):
|
|
"""Нормализует имя автора: убирает лишние запятые и пробелы"""
|
|
if not author_name:
|
|
return ''
|
|
|
|
author_name = author_name.strip()
|
|
|
|
# Убираем запятые в конце
|
|
while author_name.endswith(','):
|
|
author_name = author_name[:-1]
|
|
|
|
# Убираем двойные запятые
|
|
while ',,' in author_name:
|
|
author_name = author_name.replace(',,', ',')
|
|
|
|
# Разбиваем по запятым, убираем пустые части
|
|
parts = [p.strip() for p in author_name.split(',') if p.strip()]
|
|
|
|
return ','.join(parts)
|
|
|
|
|
|
def get_or_create_author(cur, author_name):
|
|
"""Возвращает ID автора, создавая его если нужно"""
|
|
author_name = normalize_author_name(author_name)
|
|
|
|
if not author_name:
|
|
return None
|
|
|
|
author_hash = hashlib.md5(author_name.lower().encode()).hexdigest()
|
|
|
|
cur.execute("SELECT id FROM authors WHERE name_hash = %s", (author_hash,))
|
|
row = cur.fetchone()
|
|
if row:
|
|
return row[0]
|
|
|
|
parts = author_name.split(',')
|
|
last_name = parts[0].strip() if len(parts) > 0 else ''
|
|
first_name = parts[1].strip() if len(parts) > 1 and parts[1].strip() else None
|
|
middle_name = parts[2].strip() if len(parts) > 2 and parts[2].strip() else None
|
|
first_letter = last_name[0].lower() if last_name else ''
|
|
|
|
cur.execute("""
|
|
INSERT INTO authors (full_name, last_name, first_name, middle_name,
|
|
name_hash, first_letter)
|
|
VALUES (%s, %s, %s, %s, %s, %s)
|
|
RETURNING id
|
|
""", (author_name, last_name, first_name, middle_name, author_hash, first_letter))
|
|
|
|
return cur.fetchone()[0]
|
|
|
|
|
|
def get_or_create_genre(cur, genre_code):
|
|
"""Возвращает ID жанра, создавая его если нужно"""
|
|
if not genre_code:
|
|
return None
|
|
|
|
genre_code = genre_code.strip()
|
|
|
|
cur.execute("SELECT id FROM genres WHERE code = %s", (genre_code,))
|
|
row = cur.fetchone()
|
|
if row:
|
|
return row[0]
|
|
|
|
parent_code = None
|
|
root_code = None
|
|
if '_' in genre_code:
|
|
prefix = genre_code.split('_')[0]
|
|
cur.execute("SELECT code FROM genres WHERE code = %s AND is_root = TRUE", (prefix,))
|
|
row = cur.fetchone()
|
|
if row:
|
|
parent_code = prefix
|
|
root_code = prefix
|
|
|
|
cur.execute("""
|
|
INSERT INTO genres (code, title_ru, title_en, parent_code, root_code,
|
|
is_root, is_custom)
|
|
VALUES (%s, %s, %s, %s, %s, FALSE, TRUE)
|
|
RETURNING id
|
|
""", (genre_code, genre_code, genre_code, parent_code, root_code))
|
|
|
|
return cur.fetchone()[0]
|
|
|
|
|
|
def get_or_create_series(cur, series_name):
|
|
"""Возвращает ID серии, создавая её если нужно"""
|
|
if not series_name:
|
|
return None
|
|
|
|
series_name = series_name.strip()
|
|
if not series_name:
|
|
return None
|
|
|
|
series_hash = hashlib.md5(series_name.lower().encode()).hexdigest()
|
|
|
|
cur.execute("SELECT id FROM series WHERE name_hash = %s", (series_hash,))
|
|
row = cur.fetchone()
|
|
if row:
|
|
return row[0]
|
|
|
|
first_letter = series_name[0].lower() if series_name else ''
|
|
|
|
cur.execute("""
|
|
INSERT INTO series (name, name_hash, first_letter)
|
|
VALUES (%s, %s, %s)
|
|
RETURNING id
|
|
""", (series_name, series_hash, first_letter))
|
|
|
|
return cur.fetchone()[0]
|
|
|
|
|
|
def parse_inpx_zip(zip_path, db_conn_string, source):
|
|
print(f"Connecting to database...")
|
|
conn = psycopg2.connect(db_conn_string)
|
|
cur = conn.cursor()
|
|
|
|
print(f"Clearing existing books from source: {source}...")
|
|
cur.execute("DELETE FROM books WHERE source = %s", (source,))
|
|
conn.commit()
|
|
deleted_count = cur.rowcount
|
|
if deleted_count > 0:
|
|
print(f"Deleted {deleted_count} existing books from source '{source}'")
|
|
print("Vacuuming...")
|
|
conn.autocommit = True
|
|
try:
|
|
cur.execute("VACUUM books")
|
|
print("VACUUM completed")
|
|
finally:
|
|
conn.autocommit = False
|
|
|
|
print(f"Opening archive: {zip_path}")
|
|
with zipfile.ZipFile(zip_path, 'r') as zf:
|
|
files = [(f, zf.getinfo(f).file_size) for f in zf.namelist()]
|
|
files.sort(key=lambda x: x[1], reverse=True)
|
|
target_file = files[0][0]
|
|
print(f"Processing: {target_file} ({files[0][1]} bytes)")
|
|
|
|
with zf.open(target_file, 'r') as f:
|
|
count = 0
|
|
author_cache = {}
|
|
genre_cache = {}
|
|
series_cache = {}
|
|
|
|
for line in f:
|
|
try:
|
|
decoded = line.decode('utf-8', errors='ignore')
|
|
except:
|
|
continue
|
|
|
|
parts = decoded.strip().split('\x04')
|
|
if len(parts) < 11:
|
|
continue
|
|
|
|
authors_str = parts[0].rstrip(',:') if parts[0] else ''
|
|
genre_str = parts[1] if len(parts) > 1 and parts[1] else ''
|
|
title = parts[2] if len(parts) > 2 and parts[2] else ''
|
|
series_name = parts[3].strip() if len(parts) > 3 and parts[3].strip() else None
|
|
series_number = float(parts[4]) if len(parts) > 4 and parts[4].strip() else None
|
|
|
|
internal_id = int(parts[5]) if len(parts) > 5 and parts[5].strip().isdigit() else None
|
|
file_size = int(parts[6]) if len(parts) > 6 and parts[6].strip().isdigit() else None
|
|
internal_id2 = int(parts[7]) if len(parts) > 7 and parts[7].strip().isdigit() else None
|
|
|
|
format_ = parts[9] if len(parts) > 9 and parts[9].strip() else 'fb2'
|
|
added_date = parts[10] if len(parts) > 10 and parts[10].strip() else None
|
|
language = parts[11] if len(parts) > 11 and parts[11].strip() else 'ru'
|
|
|
|
title_hash = hashlib.md5(title.lower().encode()).hexdigest() if title else None
|
|
|
|
# Серия (с кэшированием)
|
|
series_id = None
|
|
if series_name:
|
|
if series_name in series_cache:
|
|
series_id = series_cache[series_name]
|
|
else:
|
|
series_id = get_or_create_series(cur, series_name)
|
|
series_cache[series_name] = series_id
|
|
|
|
# Вставляем книгу
|
|
cur.execute("""
|
|
INSERT INTO books (title, title_hash, internal_id, file_size,
|
|
internal_id2, format, added_date, language,
|
|
source, series_id, series_number)
|
|
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
|
RETURNING id
|
|
""", (title, title_hash, internal_id, file_size, internal_id2,
|
|
format_, added_date, language, source, series_id, series_number))
|
|
book_id = cur.fetchone()[0]
|
|
|
|
# Авторы (с кэшированием)
|
|
if authors_str:
|
|
author_list = [a.strip() for a in authors_str.split(':') if a.strip()]
|
|
for position, author_name in enumerate(author_list, 1):
|
|
normalized = normalize_author_name(author_name)
|
|
if normalized in author_cache:
|
|
author_id = author_cache[normalized]
|
|
else:
|
|
author_id = get_or_create_author(cur, normalized)
|
|
author_cache[normalized] = author_id
|
|
|
|
if author_id:
|
|
cur.execute("""
|
|
INSERT INTO book_authors (book_id, author_id, position)
|
|
VALUES (%s, %s, %s)
|
|
ON CONFLICT DO NOTHING
|
|
""", (book_id, author_id, position))
|
|
|
|
# Жанры (с кэшированием)
|
|
if genre_str:
|
|
for genre_code in genre_str.split(':'):
|
|
genre_code = genre_code.strip()
|
|
if not genre_code:
|
|
continue
|
|
|
|
if genre_code in genre_cache:
|
|
genre_id = genre_cache[genre_code]
|
|
else:
|
|
genre_id = get_or_create_genre(cur, genre_code)
|
|
genre_cache[genre_code] = genre_id
|
|
|
|
if genre_id:
|
|
cur.execute("""
|
|
INSERT INTO book_genres (book_id, genre_id)
|
|
VALUES (%s, %s)
|
|
ON CONFLICT DO NOTHING
|
|
""", (book_id, genre_id))
|
|
|
|
count += 1
|
|
|
|
if count % 5000 == 0:
|
|
conn.commit()
|
|
print(f"Loaded {count} books... "
|
|
f"(authors: {len(author_cache)}, "
|
|
f"series: {len(series_cache)}, "
|
|
f"genres: {len(genre_cache)})")
|
|
|
|
conn.commit()
|
|
|
|
print("Optimizing...")
|
|
conn.commit()
|
|
cur.execute("ANALYZE books")
|
|
cur.execute("ANALYZE authors")
|
|
cur.execute("ANALYZE series")
|
|
cur.execute("ANALYZE genres")
|
|
conn.close()
|
|
print(f"Done! Total books loaded: {count}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
if len(sys.argv) != 4:
|
|
print("Usage: python parse_inpx.py <inpx.zip> <postgres_connection_string> <source>")
|
|
sys.exit(1)
|
|
parse_inpx_zip(sys.argv[1], sys.argv[2], sys.argv[3]) |