Добавлены еще часть файлов
This commit is contained in:
astakhovp committed
1 parent
5753de622e
commit
ab4953ffd1
5 files changed
+459
No files matched your search
@@ -0,0 +1,272 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Импорт книг из inpx-файла lib.rus.ec в PostgreSQL.
|
||||
|
||||
Использование:
|
||||
python parse_inpx.py <inpx.zip> <postgres_connection_string> <source>
|
||||
|
||||
Пример:
|
||||
python parse_inpx.py \
|
||||
"/mnt/hdd/share/Torrents/Downloaded/Libruks/Библиотека Либрусек для архивов 18-09-2026.inpx" \
|
||||
"dbname=books user=postgres password=Karabas@Barabas host=172.20.0.4" \
|
||||
"lib.rus.ec"
|
||||
"""
|
||||
import zipfile
|
||||
import psycopg2
|
||||
from psycopg2.extras import execute_values
|
||||
import sys
|
||||
import hashlib
|
||||
|
||||
|
||||
def normalize_author_name(author_name):
|
||||
"""Нормализует имя автора: убирает лишние запятые и пробелы"""
|
||||
if not author_name:
|
||||
return ''
|
||||
|
||||
author_name = author_name.strip()
|
||||
|
||||
# Убираем запятые в конце
|
||||
while author_name.endswith(','):
|
||||
author_name = author_name[:-1]
|
||||
|
||||
# Убираем двойные запятые
|
||||
while ',,' in author_name:
|
||||
author_name = author_name.replace(',,', ',')
|
||||
|
||||
# Разбиваем по запятым, убираем пустые части
|
||||
parts = [p.strip() for p in author_name.split(',') if p.strip()]
|
||||
|
||||
return ','.join(parts)
|
||||
|
||||
|
||||
def get_or_create_author(cur, author_name):
|
||||
"""Возвращает ID автора, создавая его если нужно"""
|
||||
author_name = normalize_author_name(author_name)
|
||||
|
||||
if not author_name:
|
||||
return None
|
||||
|
||||
author_hash = hashlib.md5(author_name.lower().encode()).hexdigest()
|
||||
|
||||
cur.execute("SELECT id FROM authors WHERE name_hash = %s", (author_hash,))
|
||||
row = cur.fetchone()
|
||||
if row:
|
||||
return row[0]
|
||||
|
||||
parts = author_name.split(',')
|
||||
last_name = parts[0].strip() if len(parts) > 0 else ''
|
||||
first_name = parts[1].strip() if len(parts) > 1 and parts[1].strip() else None
|
||||
middle_name = parts[2].strip() if len(parts) > 2 and parts[2].strip() else None
|
||||
first_letter = last_name[0].lower() if last_name else ''
|
||||
|
||||
cur.execute("""
|
||||
INSERT INTO authors (full_name, last_name, first_name, middle_name,
|
||||
name_hash, first_letter)
|
||||
VALUES (%s, %s, %s, %s, %s, %s)
|
||||
RETURNING id
|
||||
""", (author_name, last_name, first_name, middle_name, author_hash, first_letter))
|
||||
|
||||
return cur.fetchone()[0]
|
||||
|
||||
|
||||
def get_or_create_genre(cur, genre_code):
|
||||
"""Возвращает ID жанра, создавая его если нужно"""
|
||||
if not genre_code:
|
||||
return None
|
||||
|
||||
genre_code = genre_code.strip()
|
||||
|
||||
cur.execute("SELECT id FROM genres WHERE code = %s", (genre_code,))
|
||||
row = cur.fetchone()
|
||||
if row:
|
||||
return row[0]
|
||||
|
||||
parent_code = None
|
||||
root_code = None
|
||||
if '_' in genre_code:
|
||||
prefix = genre_code.split('_')[0]
|
||||
cur.execute("SELECT code FROM genres WHERE code = %s AND is_root = TRUE", (prefix,))
|
||||
row = cur.fetchone()
|
||||
if row:
|
||||
parent_code = prefix
|
||||
root_code = prefix
|
||||
|
||||
cur.execute("""
|
||||
INSERT INTO genres (code, title_ru, title_en, parent_code, root_code,
|
||||
is_root, is_custom)
|
||||
VALUES (%s, %s, %s, %s, %s, FALSE, TRUE)
|
||||
RETURNING id
|
||||
""", (genre_code, genre_code, genre_code, parent_code, root_code))
|
||||
|
||||
return cur.fetchone()[0]
|
||||
|
||||
|
||||
def get_or_create_series(cur, series_name):
|
||||
"""Возвращает ID серии, создавая её если нужно"""
|
||||
if not series_name:
|
||||
return None
|
||||
|
||||
series_name = series_name.strip()
|
||||
if not series_name:
|
||||
return None
|
||||
|
||||
series_hash = hashlib.md5(series_name.lower().encode()).hexdigest()
|
||||
|
||||
cur.execute("SELECT id FROM series WHERE name_hash = %s", (series_hash,))
|
||||
row = cur.fetchone()
|
||||
if row:
|
||||
return row[0]
|
||||
|
||||
first_letter = series_name[0].lower() if series_name else ''
|
||||
|
||||
cur.execute("""
|
||||
INSERT INTO series (name, name_hash, first_letter)
|
||||
VALUES (%s, %s, %s)
|
||||
RETURNING id
|
||||
""", (series_name, series_hash, first_letter))
|
||||
|
||||
return cur.fetchone()[0]
|
||||
|
||||
|
||||
def parse_inpx_zip(zip_path, db_conn_string, source):
|
||||
print(f"Connecting to database...")
|
||||
conn = psycopg2.connect(db_conn_string)
|
||||
cur = conn.cursor()
|
||||
|
||||
print(f"Clearing existing books from source: {source}...")
|
||||
cur.execute("DELETE FROM books WHERE source = %s", (source,))
|
||||
conn.commit()
|
||||
deleted_count = cur.rowcount
|
||||
if deleted_count > 0:
|
||||
print(f"Deleted {deleted_count} existing books from source '{source}'")
|
||||
print("Vacuuming...")
|
||||
conn.autocommit = True
|
||||
try:
|
||||
cur.execute("VACUUM books")
|
||||
print("VACUUM completed")
|
||||
finally:
|
||||
conn.autocommit = False
|
||||
|
||||
print(f"Opening archive: {zip_path}")
|
||||
with zipfile.ZipFile(zip_path, 'r') as zf:
|
||||
files = [(f, zf.getinfo(f).file_size) for f in zf.namelist()]
|
||||
files.sort(key=lambda x: x[1], reverse=True)
|
||||
target_file = files[0][0]
|
||||
print(f"Processing: {target_file} ({files[0][1]} bytes)")
|
||||
|
||||
with zf.open(target_file, 'r') as f:
|
||||
count = 0
|
||||
author_cache = {}
|
||||
genre_cache = {}
|
||||
series_cache = {}
|
||||
|
||||
for line in f:
|
||||
try:
|
||||
decoded = line.decode('utf-8', errors='ignore')
|
||||
except:
|
||||
continue
|
||||
|
||||
parts = decoded.strip().split('\x04')
|
||||
if len(parts) < 11:
|
||||
continue
|
||||
|
||||
authors_str = parts[0].rstrip(',:') if parts[0] else ''
|
||||
genre_str = parts[1] if len(parts) > 1 and parts[1] else ''
|
||||
title = parts[2] if len(parts) > 2 and parts[2] else ''
|
||||
series_name = parts[3].strip() if len(parts) > 3 and parts[3].strip() else None
|
||||
series_number = float(parts[4]) if len(parts) > 4 and parts[4].strip() else None
|
||||
|
||||
internal_id = int(parts[5]) if len(parts) > 5 and parts[5].strip().isdigit() else None
|
||||
file_size = int(parts[6]) if len(parts) > 6 and parts[6].strip().isdigit() else None
|
||||
internal_id2 = int(parts[7]) if len(parts) > 7 and parts[7].strip().isdigit() else None
|
||||
|
||||
format_ = parts[9] if len(parts) > 9 and parts[9].strip() else 'fb2'
|
||||
added_date = parts[10] if len(parts) > 10 and parts[10].strip() else None
|
||||
language = parts[11] if len(parts) > 11 and parts[11].strip() else 'ru'
|
||||
|
||||
title_hash = hashlib.md5(title.lower().encode()).hexdigest() if title else None
|
||||
|
||||
# Серия (с кэшированием)
|
||||
series_id = None
|
||||
if series_name:
|
||||
if series_name in series_cache:
|
||||
series_id = series_cache[series_name]
|
||||
else:
|
||||
series_id = get_or_create_series(cur, series_name)
|
||||
series_cache[series_name] = series_id
|
||||
|
||||
# Вставляем книгу
|
||||
cur.execute("""
|
||||
INSERT INTO books (title, title_hash, internal_id, file_size,
|
||||
internal_id2, format, added_date, language,
|
||||
source, series_id, series_number)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
||||
RETURNING id
|
||||
""", (title, title_hash, internal_id, file_size, internal_id2,
|
||||
format_, added_date, language, source, series_id, series_number))
|
||||
book_id = cur.fetchone()[0]
|
||||
|
||||
# Авторы (с кэшированием)
|
||||
if authors_str:
|
||||
author_list = [a.strip() for a in authors_str.split(':') if a.strip()]
|
||||
for position, author_name in enumerate(author_list, 1):
|
||||
normalized = normalize_author_name(author_name)
|
||||
if normalized in author_cache:
|
||||
author_id = author_cache[normalized]
|
||||
else:
|
||||
author_id = get_or_create_author(cur, normalized)
|
||||
author_cache[normalized] = author_id
|
||||
|
||||
if author_id:
|
||||
cur.execute("""
|
||||
INSERT INTO book_authors (book_id, author_id, position)
|
||||
VALUES (%s, %s, %s)
|
||||
ON CONFLICT DO NOTHING
|
||||
""", (book_id, author_id, position))
|
||||
|
||||
# Жанры (с кэшированием)
|
||||
if genre_str:
|
||||
for genre_code in genre_str.split(':'):
|
||||
genre_code = genre_code.strip()
|
||||
if not genre_code:
|
||||
continue
|
||||
|
||||
if genre_code in genre_cache:
|
||||
genre_id = genre_cache[genre_code]
|
||||
else:
|
||||
genre_id = get_or_create_genre(cur, genre_code)
|
||||
genre_cache[genre_code] = genre_id
|
||||
|
||||
if genre_id:
|
||||
cur.execute("""
|
||||
INSERT INTO book_genres (book_id, genre_id)
|
||||
VALUES (%s, %s)
|
||||
ON CONFLICT DO NOTHING
|
||||
""", (book_id, genre_id))
|
||||
|
||||
count += 1
|
||||
|
||||
if count % 5000 == 0:
|
||||
conn.commit()
|
||||
print(f"Loaded {count} books... "
|
||||
f"(authors: {len(author_cache)}, "
|
||||
f"series: {len(series_cache)}, "
|
||||
f"genres: {len(genre_cache)})")
|
||||
|
||||
conn.commit()
|
||||
|
||||
print("Optimizing...")
|
||||
conn.commit()
|
||||
cur.execute("ANALYZE books")
|
||||
cur.execute("ANALYZE authors")
|
||||
cur.execute("ANALYZE series")
|
||||
cur.execute("ANALYZE genres")
|
||||
conn.close()
|
||||
print(f"Done! Total books loaded: {count}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) != 4:
|
||||
print("Usage: python parse_inpx.py <inpx.zip> <postgres_connection_string> <source>")
|
||||
sys.exit(1)
|
||||
parse_inpx_zip(sys.argv[1], sys.argv[2], sys.argv[3])
|
||||
Reference in new issue
Block a user