diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..da01d35 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,21 @@ +FROM python:3.11-slim + +RUN pip install --no-cache-dir flask psycopg2-binary lxml + +WORKDIR /app + +# Копируем приложение +COPY app/*.py /app/ + +# Копируем скрипты импорта +COPY scripts/*.py /app/scripts/ + +# Копируем SQL-схему +COPY sql/ /app/sql/ + +# Копируем данные (genres.xml) +COPY data/genres.xml /app/data/genres.xml + +EXPOSE 8080 + +CMD ["python", "opds_server.py"] \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..7ea4511 --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,13 @@ +services: + opds: + build: . + container_name: opds_server + network_mode: host + environment: + - DB_CONN=dbname=books user=postgres password=Karabas@Barabas host=172.20.0.4 + - BOOKS_PATH=/mnt/hdd/share/Torrents/Downloaded/Libruks + - COVERS_PATH=/mnt/hdd/opds/covers + volumes: + - /mnt/hdd/share/Torrents/Downloaded/Libruks:/mnt/hdd/share/Torrents/Downloaded/Libruks:ro + - /mnt/hdd/opds/covers:/mnt/hdd/opds/covers + restart: unless-stopped \ No newline at end of file diff --git a/import_lib.sh b/import_lib.sh new file mode 100644 index 0000000..7b93dc5 --- /dev/null +++ b/import_lib.sh @@ -0,0 +1,28 @@ +cd /mnt/hdd/opds + +# 1. Создать таблицы и индексы +docker compose exec opds python -c " +import psycopg2 +conn = psycopg2.connect('dbname=books user=postgres password=Karabas@Barabas host=172.20.0.4') +cur = conn.cursor() +for f in ['/app/sql/01_create_tables.sql', '/app/sql/02_create_indexes.sql']: + with open(f) as fp: + cur.execute(fp.read()) +conn.commit() +conn.close() +print('Schema created') +" + +# 2. Загрузить жанры +docker compose exec opds python /app/scripts/parse_genres.py \ + /app/data/genres.xml \ + "dbname=books user=postgres password=Karabas@Barabas host=172.20.0.4" + +# 3. Импортировать книги +docker compose exec opds python /app/scripts/parse_inpx.py \ + "/mnt/hdd/share/Torrents/Downloaded/Libruks/Библиотека Либрусек для архивов 18-09-2026.inpx" \ + "dbname=books user=postgres password=Karabas@Barabas host=172.20.0.4" \ + "lib.rus.ec" + +# 4. Следить за прогрессом +docker compose logs opds --tail 20 \ No newline at end of file diff --git a/scripts/parse_genres.py b/scripts/parse_genres.py new file mode 100644 index 0000000..bdde1ee --- /dev/null +++ b/scripts/parse_genres.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 +""" +Загрузка справочника жанров fb2 из genres.xml в PostgreSQL. + +Использование: + python parse_genres.py +""" +import xml.etree.ElementTree as ET +import psycopg2 +from psycopg2.extras import execute_values +import sys + + +def parse_genres_xml(xml_path, db_conn_string): + print("Connecting to database...") + conn = psycopg2.connect(db_conn_string) + cur = conn.cursor() + + print("Clearing genres table...") + cur.execute("TRUNCATE TABLE genres RESTART IDENTITY CASCADE") + conn.commit() + + print(f"Parsing XML: {xml_path}") + tree = ET.parse(xml_path) + root = tree.getroot() + + genres = [] + seen_codes = set() + + for genre in root.findall('genre'): + genre_code = genre.get('value') + + title_ru = None + title_en = None + + for descr in genre.findall('root-descr'): + lang = descr.get('lang') + title = descr.get('genre-title') + if lang == 'ru': + title_ru = title + elif lang == 'en': + title_en = title + + if genre_code not in seen_codes: + genres.append({ + 'code': genre_code, + 'title_ru': title_ru, + 'title_en': title_en, + 'parent_code': None, + 'root_code': genre_code, + 'is_root': True, + 'genre_alt': [] + }) + seen_codes.add(genre_code) + + subgenres = genre.find('subgenres') + if subgenres is not None: + for subgenre in subgenres.findall('subgenre'): + sub_code = subgenre.get('value') + + sub_title_ru = None + sub_title_en = None + genre_alt = [] + + for descr in subgenre.findall('genre-descr'): + lang = descr.get('lang') + title = descr.get('title') + if lang == 'ru': + sub_title_ru = title + elif lang == 'en': + sub_title_en = title + + for alt in subgenre.findall('genre-alt'): + genre_alt.append(alt.get('value')) + + if sub_code not in seen_codes: + genres.append({ + 'code': sub_code, + 'title_ru': sub_title_ru, + 'title_en': sub_title_en, + 'parent_code': genre_code, + 'root_code': genre_code, + 'is_root': False, + 'genre_alt': genre_alt if genre_alt else None + }) + seen_codes.add(sub_code) + + print(f"Found {len(genres)} unique genres") + + batch = [] + for g in genres: + batch.append(( + g['code'], + g['title_ru'], + g['title_en'], + g['parent_code'], + g['root_code'], + g['is_root'], + False, # is_custom + g['genre_alt'] + )) + + execute_values(cur, """ + INSERT INTO genres (code, title_ru, title_en, parent_code, root_code, + is_root, is_custom, genre_alt) + VALUES %s + """, batch) + conn.commit() + + cur.execute("SELECT COUNT(*) FROM genres WHERE is_root = TRUE") + root_count = cur.fetchone()[0] + + cur.execute("SELECT COUNT(*) FROM genres WHERE is_root = FALSE") + sub_count = cur.fetchone()[0] + + print(f"Loaded: {root_count} root genres, {sub_count} subgenres") + conn.close() + print("Done!") + + +if __name__ == "__main__": + if len(sys.argv) != 3: + print("Usage: python parse_genres.py ") + sys.exit(1) + parse_genres_xml(sys.argv[1], sys.argv[2]) \ No newline at end of file diff --git a/scripts/parse_inpx.py b/scripts/parse_inpx.py new file mode 100644 index 0000000..e9581a2 --- /dev/null +++ b/scripts/parse_inpx.py @@ -0,0 +1,272 @@ +#!/usr/bin/env python3 +""" +Импорт книг из inpx-файла lib.rus.ec в PostgreSQL. + +Использование: + python parse_inpx.py + +Пример: + python parse_inpx.py \ + "/mnt/hdd/share/Torrents/Downloaded/Libruks/Библиотека Либрусек для архивов 18-09-2026.inpx" \ + "dbname=books user=postgres password=Karabas@Barabas host=172.20.0.4" \ + "lib.rus.ec" +""" +import zipfile +import psycopg2 +from psycopg2.extras import execute_values +import sys +import hashlib + + +def normalize_author_name(author_name): + """Нормализует имя автора: убирает лишние запятые и пробелы""" + if not author_name: + return '' + + author_name = author_name.strip() + + # Убираем запятые в конце + while author_name.endswith(','): + author_name = author_name[:-1] + + # Убираем двойные запятые + while ',,' in author_name: + author_name = author_name.replace(',,', ',') + + # Разбиваем по запятым, убираем пустые части + parts = [p.strip() for p in author_name.split(',') if p.strip()] + + return ','.join(parts) + + +def get_or_create_author(cur, author_name): + """Возвращает ID автора, создавая его если нужно""" + author_name = normalize_author_name(author_name) + + if not author_name: + return None + + author_hash = hashlib.md5(author_name.lower().encode()).hexdigest() + + cur.execute("SELECT id FROM authors WHERE name_hash = %s", (author_hash,)) + row = cur.fetchone() + if row: + return row[0] + + parts = author_name.split(',') + last_name = parts[0].strip() if len(parts) > 0 else '' + first_name = parts[1].strip() if len(parts) > 1 and parts[1].strip() else None + middle_name = parts[2].strip() if len(parts) > 2 and parts[2].strip() else None + first_letter = last_name[0].lower() if last_name else '' + + cur.execute(""" + INSERT INTO authors (full_name, last_name, first_name, middle_name, + name_hash, first_letter) + VALUES (%s, %s, %s, %s, %s, %s) + RETURNING id + """, (author_name, last_name, first_name, middle_name, author_hash, first_letter)) + + return cur.fetchone()[0] + + +def get_or_create_genre(cur, genre_code): + """Возвращает ID жанра, создавая его если нужно""" + if not genre_code: + return None + + genre_code = genre_code.strip() + + cur.execute("SELECT id FROM genres WHERE code = %s", (genre_code,)) + row = cur.fetchone() + if row: + return row[0] + + parent_code = None + root_code = None + if '_' in genre_code: + prefix = genre_code.split('_')[0] + cur.execute("SELECT code FROM genres WHERE code = %s AND is_root = TRUE", (prefix,)) + row = cur.fetchone() + if row: + parent_code = prefix + root_code = prefix + + cur.execute(""" + INSERT INTO genres (code, title_ru, title_en, parent_code, root_code, + is_root, is_custom) + VALUES (%s, %s, %s, %s, %s, FALSE, TRUE) + RETURNING id + """, (genre_code, genre_code, genre_code, parent_code, root_code)) + + return cur.fetchone()[0] + + +def get_or_create_series(cur, series_name): + """Возвращает ID серии, создавая её если нужно""" + if not series_name: + return None + + series_name = series_name.strip() + if not series_name: + return None + + series_hash = hashlib.md5(series_name.lower().encode()).hexdigest() + + cur.execute("SELECT id FROM series WHERE name_hash = %s", (series_hash,)) + row = cur.fetchone() + if row: + return row[0] + + first_letter = series_name[0].lower() if series_name else '' + + cur.execute(""" + INSERT INTO series (name, name_hash, first_letter) + VALUES (%s, %s, %s) + RETURNING id + """, (series_name, series_hash, first_letter)) + + return cur.fetchone()[0] + + +def parse_inpx_zip(zip_path, db_conn_string, source): + print(f"Connecting to database...") + conn = psycopg2.connect(db_conn_string) + cur = conn.cursor() + + print(f"Clearing existing books from source: {source}...") + cur.execute("DELETE FROM books WHERE source = %s", (source,)) + conn.commit() + deleted_count = cur.rowcount + if deleted_count > 0: + print(f"Deleted {deleted_count} existing books from source '{source}'") + print("Vacuuming...") + conn.autocommit = True + try: + cur.execute("VACUUM books") + print("VACUUM completed") + finally: + conn.autocommit = False + + print(f"Opening archive: {zip_path}") + with zipfile.ZipFile(zip_path, 'r') as zf: + files = [(f, zf.getinfo(f).file_size) for f in zf.namelist()] + files.sort(key=lambda x: x[1], reverse=True) + target_file = files[0][0] + print(f"Processing: {target_file} ({files[0][1]} bytes)") + + with zf.open(target_file, 'r') as f: + count = 0 + author_cache = {} + genre_cache = {} + series_cache = {} + + for line in f: + try: + decoded = line.decode('utf-8', errors='ignore') + except: + continue + + parts = decoded.strip().split('\x04') + if len(parts) < 11: + continue + + authors_str = parts[0].rstrip(',:') if parts[0] else '' + genre_str = parts[1] if len(parts) > 1 and parts[1] else '' + title = parts[2] if len(parts) > 2 and parts[2] else '' + series_name = parts[3].strip() if len(parts) > 3 and parts[3].strip() else None + series_number = float(parts[4]) if len(parts) > 4 and parts[4].strip() else None + + internal_id = int(parts[5]) if len(parts) > 5 and parts[5].strip().isdigit() else None + file_size = int(parts[6]) if len(parts) > 6 and parts[6].strip().isdigit() else None + internal_id2 = int(parts[7]) if len(parts) > 7 and parts[7].strip().isdigit() else None + + format_ = parts[9] if len(parts) > 9 and parts[9].strip() else 'fb2' + added_date = parts[10] if len(parts) > 10 and parts[10].strip() else None + language = parts[11] if len(parts) > 11 and parts[11].strip() else 'ru' + + title_hash = hashlib.md5(title.lower().encode()).hexdigest() if title else None + + # Серия (с кэшированием) + series_id = None + if series_name: + if series_name in series_cache: + series_id = series_cache[series_name] + else: + series_id = get_or_create_series(cur, series_name) + series_cache[series_name] = series_id + + # Вставляем книгу + cur.execute(""" + INSERT INTO books (title, title_hash, internal_id, file_size, + internal_id2, format, added_date, language, + source, series_id, series_number) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) + RETURNING id + """, (title, title_hash, internal_id, file_size, internal_id2, + format_, added_date, language, source, series_id, series_number)) + book_id = cur.fetchone()[0] + + # Авторы (с кэшированием) + if authors_str: + author_list = [a.strip() for a in authors_str.split(':') if a.strip()] + for position, author_name in enumerate(author_list, 1): + normalized = normalize_author_name(author_name) + if normalized in author_cache: + author_id = author_cache[normalized] + else: + author_id = get_or_create_author(cur, normalized) + author_cache[normalized] = author_id + + if author_id: + cur.execute(""" + INSERT INTO book_authors (book_id, author_id, position) + VALUES (%s, %s, %s) + ON CONFLICT DO NOTHING + """, (book_id, author_id, position)) + + # Жанры (с кэшированием) + if genre_str: + for genre_code in genre_str.split(':'): + genre_code = genre_code.strip() + if not genre_code: + continue + + if genre_code in genre_cache: + genre_id = genre_cache[genre_code] + else: + genre_id = get_or_create_genre(cur, genre_code) + genre_cache[genre_code] = genre_id + + if genre_id: + cur.execute(""" + INSERT INTO book_genres (book_id, genre_id) + VALUES (%s, %s) + ON CONFLICT DO NOTHING + """, (book_id, genre_id)) + + count += 1 + + if count % 5000 == 0: + conn.commit() + print(f"Loaded {count} books... " + f"(authors: {len(author_cache)}, " + f"series: {len(series_cache)}, " + f"genres: {len(genre_cache)})") + + conn.commit() + + print("Optimizing...") + conn.commit() + cur.execute("ANALYZE books") + cur.execute("ANALYZE authors") + cur.execute("ANALYZE series") + cur.execute("ANALYZE genres") + conn.close() + print(f"Done! Total books loaded: {count}") + + +if __name__ == "__main__": + if len(sys.argv) != 4: + print("Usage: python parse_inpx.py ") + sys.exit(1) + parse_inpx_zip(sys.argv[1], sys.argv[2], sys.argv[3]) \ No newline at end of file