Добавлены еще часть файлов

Dockerfile

@@ -0,0 +1,21 @@
+FROM python:3.11-slim
+
+RUN pip install --no-cache-dir flask psycopg2-binary lxml
+
+WORKDIR /app
+
+# Копируем приложение
+COPY app/*.py /app/
+
+# Копируем скрипты импорта
+COPY scripts/*.py /app/scripts/
+
+# Копируем SQL-схему
+COPY sql/ /app/sql/
+
+# Копируем данные (genres.xml)
+COPY data/genres.xml /app/data/genres.xml
+
+EXPOSE 8080
+
+CMD ["python", "opds_server.py"]

docker-compose.yaml

@@ -0,0 +1,13 @@
+services:
+  opds:
+    build: .
+    container_name: opds_server
+    network_mode: host
+    environment:
+      - DB_CONN=dbname=books user=postgres password=Karabas@Barabas host=172.20.0.4
+      - BOOKS_PATH=/mnt/hdd/share/Torrents/Downloaded/Libruks
+      - COVERS_PATH=/mnt/hdd/opds/covers
+    volumes:
+      - /mnt/hdd/share/Torrents/Downloaded/Libruks:/mnt/hdd/share/Torrents/Downloaded/Libruks:ro
+      - /mnt/hdd/opds/covers:/mnt/hdd/opds/covers
+    restart: unless-stopped

import_lib.sh

@@ -0,0 +1,28 @@
+cd /mnt/hdd/opds
+
+# 1. Создать таблицы и индексы
+docker compose exec opds python -c "
+import psycopg2
+conn = psycopg2.connect('dbname=books user=postgres password=Karabas@Barabas host=172.20.0.4')
+cur = conn.cursor()
+for f in ['/app/sql/01_create_tables.sql', '/app/sql/02_create_indexes.sql']:
+    with open(f) as fp:
+        cur.execute(fp.read())
+conn.commit()
+conn.close()
+print('Schema created')
+"
+
+# 2. Загрузить жанры
+docker compose exec opds python /app/scripts/parse_genres.py \
+  /app/data/genres.xml \
+  "dbname=books user=postgres password=Karabas@Barabas host=172.20.0.4"
+
+# 3. Импортировать книги
+docker compose exec opds python /app/scripts/parse_inpx.py \
+  "/mnt/hdd/share/Torrents/Downloaded/Libruks/Библиотека Либрусек для архивов 18-09-2026.inpx" \
+  "dbname=books user=postgres password=Karabas@Barabas host=172.20.0.4" \
+  "lib.rus.ec"
+
+# 4. Следить за прогрессом
+docker compose logs opds --tail 20

scripts/parse_genres.py

@@ -0,0 +1,125 @@
+#!/usr/bin/env python3
+"""
+Загрузка справочника жанров fb2 из genres.xml в PostgreSQL.
+
+Использование:
+    python parse_genres.py <genres.xml> <postgres_connection_string>
+"""
+import xml.etree.ElementTree as ET
+import psycopg2
+from psycopg2.extras import execute_values
+import sys
+
+
+def parse_genres_xml(xml_path, db_conn_string):
+    print("Connecting to database...")
+    conn = psycopg2.connect(db_conn_string)
+    cur = conn.cursor()
+
+    print("Clearing genres table...")
+    cur.execute("TRUNCATE TABLE genres RESTART IDENTITY CASCADE")
+    conn.commit()
+
+    print(f"Parsing XML: {xml_path}")
+    tree = ET.parse(xml_path)
+    root = tree.getroot()
+
+    genres = []
+    seen_codes = set()
+
+    for genre in root.findall('genre'):
+        genre_code = genre.get('value')
+
+        title_ru = None
+        title_en = None
+
+        for descr in genre.findall('root-descr'):
+            lang = descr.get('lang')
+            title = descr.get('genre-title')
+            if lang == 'ru':
+                title_ru = title
+            elif lang == 'en':
+                title_en = title
+
+        if genre_code not in seen_codes:
+            genres.append({
+                'code': genre_code,
+                'title_ru': title_ru,
+                'title_en': title_en,
+                'parent_code': None,
+                'root_code': genre_code,
+                'is_root': True,
+                'genre_alt': []
+            })
+            seen_codes.add(genre_code)
+
+        subgenres = genre.find('subgenres')
+        if subgenres is not None:
+            for subgenre in subgenres.findall('subgenre'):
+                sub_code = subgenre.get('value')
+
+                sub_title_ru = None
+                sub_title_en = None
+                genre_alt = []
+
+                for descr in subgenre.findall('genre-descr'):
+                    lang = descr.get('lang')
+                    title = descr.get('title')
+                    if lang == 'ru':
+                        sub_title_ru = title
+                    elif lang == 'en':
+                        sub_title_en = title
+
+                for alt in subgenre.findall('genre-alt'):
+                    genre_alt.append(alt.get('value'))
+
+                if sub_code not in seen_codes:
+                    genres.append({
+                        'code': sub_code,
+                        'title_ru': sub_title_ru,
+                        'title_en': sub_title_en,
+                        'parent_code': genre_code,
+                        'root_code': genre_code,
+                        'is_root': False,
+                        'genre_alt': genre_alt if genre_alt else None
+                    })
+                    seen_codes.add(sub_code)
+
+    print(f"Found {len(genres)} unique genres")
+
+    batch = []
+    for g in genres:
+        batch.append((
+            g['code'],
+            g['title_ru'],
+            g['title_en'],
+            g['parent_code'],
+            g['root_code'],
+            g['is_root'],
+            False,  # is_custom
+            g['genre_alt']
+        ))
+
+    execute_values(cur, """
+        INSERT INTO genres (code, title_ru, title_en, parent_code, root_code,
+                            is_root, is_custom, genre_alt)
+        VALUES %s
+    """, batch)
+    conn.commit()
+
+    cur.execute("SELECT COUNT(*) FROM genres WHERE is_root = TRUE")
+    root_count = cur.fetchone()[0]
+
+    cur.execute("SELECT COUNT(*) FROM genres WHERE is_root = FALSE")
+    sub_count = cur.fetchone()[0]
+
+    print(f"Loaded: {root_count} root genres, {sub_count} subgenres")
+    conn.close()
+    print("Done!")
+
+
+if __name__ == "__main__":
+    if len(sys.argv) != 3:
+        print("Usage: python parse_genres.py <genres.xml> <postgres_connection_string>")
+        sys.exit(1)
+    parse_genres_xml(sys.argv[1], sys.argv[2])

scripts/parse_inpx.py

@@ -0,0 +1,272 @@
+#!/usr/bin/env python3
+"""
+Импорт книг из inpx-файла lib.rus.ec в PostgreSQL.
+
+Использование:
+    python parse_inpx.py <inpx.zip> <postgres_connection_string> <source>
+
+Пример:
+    python parse_inpx.py \
+      "/mnt/hdd/share/Torrents/Downloaded/Libruks/Библиотека Либрусек для архивов 18-09-2026.inpx" \
+      "dbname=books user=postgres password=Karabas@Barabas host=172.20.0.4" \
+      "lib.rus.ec"
+"""
+import zipfile
+import psycopg2
+from psycopg2.extras import execute_values
+import sys
+import hashlib
+
+
+def normalize_author_name(author_name):
+    """Нормализует имя автора: убирает лишние запятые и пробелы"""
+    if not author_name:
+        return ''
+
+    author_name = author_name.strip()
+
+    # Убираем запятые в конце
+    while author_name.endswith(','):
+        author_name = author_name[:-1]
+
+    # Убираем двойные запятые
+    while ',,' in author_name:
+        author_name = author_name.replace(',,', ',')
+
+    # Разбиваем по запятым, убираем пустые части
+    parts = [p.strip() for p in author_name.split(',') if p.strip()]
+
+    return ','.join(parts)
+
+
+def get_or_create_author(cur, author_name):
+    """Возвращает ID автора, создавая его если нужно"""
+    author_name = normalize_author_name(author_name)
+
+    if not author_name:
+        return None
+
+    author_hash = hashlib.md5(author_name.lower().encode()).hexdigest()
+
+    cur.execute("SELECT id FROM authors WHERE name_hash = %s", (author_hash,))
+    row = cur.fetchone()
+    if row:
+        return row[0]
+
+    parts = author_name.split(',')
+    last_name = parts[0].strip() if len(parts) > 0 else ''
+    first_name = parts[1].strip() if len(parts) > 1 and parts[1].strip() else None
+    middle_name = parts[2].strip() if len(parts) > 2 and parts[2].strip() else None
+    first_letter = last_name[0].lower() if last_name else ''
+
+    cur.execute("""
+        INSERT INTO authors (full_name, last_name, first_name, middle_name,
+                             name_hash, first_letter)
+        VALUES (%s, %s, %s, %s, %s, %s)
+        RETURNING id
+    """, (author_name, last_name, first_name, middle_name, author_hash, first_letter))
+
+    return cur.fetchone()[0]
+
+
+def get_or_create_genre(cur, genre_code):
+    """Возвращает ID жанра, создавая его если нужно"""
+    if not genre_code:
+        return None
+
+    genre_code = genre_code.strip()
+
+    cur.execute("SELECT id FROM genres WHERE code = %s", (genre_code,))
+    row = cur.fetchone()
+    if row:
+        return row[0]
+
+    parent_code = None
+    root_code = None
+    if '_' in genre_code:
+        prefix = genre_code.split('_')[0]
+        cur.execute("SELECT code FROM genres WHERE code = %s AND is_root = TRUE", (prefix,))
+        row = cur.fetchone()
+        if row:
+            parent_code = prefix
+            root_code = prefix
+
+    cur.execute("""
+        INSERT INTO genres (code, title_ru, title_en, parent_code, root_code,
+                            is_root, is_custom)
+        VALUES (%s, %s, %s, %s, %s, FALSE, TRUE)
+        RETURNING id
+    """, (genre_code, genre_code, genre_code, parent_code, root_code))
+
+    return cur.fetchone()[0]
+
+
+def get_or_create_series(cur, series_name):
+    """Возвращает ID серии, создавая её если нужно"""
+    if not series_name:
+        return None
+
+    series_name = series_name.strip()
+    if not series_name:
+        return None
+
+    series_hash = hashlib.md5(series_name.lower().encode()).hexdigest()
+
+    cur.execute("SELECT id FROM series WHERE name_hash = %s", (series_hash,))
+    row = cur.fetchone()
+    if row:
+        return row[0]
+
+    first_letter = series_name[0].lower() if series_name else ''
+
+    cur.execute("""
+        INSERT INTO series (name, name_hash, first_letter)
+        VALUES (%s, %s, %s)
+        RETURNING id
+    """, (series_name, series_hash, first_letter))
+
+    return cur.fetchone()[0]
+
+
+def parse_inpx_zip(zip_path, db_conn_string, source):
+    print(f"Connecting to database...")
+    conn = psycopg2.connect(db_conn_string)
+    cur = conn.cursor()
+
+    print(f"Clearing existing books from source: {source}...")
+    cur.execute("DELETE FROM books WHERE source = %s", (source,))
+    conn.commit()
+    deleted_count = cur.rowcount
+    if deleted_count > 0:
+        print(f"Deleted {deleted_count} existing books from source '{source}'")
+        print("Vacuuming...")
+        conn.autocommit = True
+        try:
+            cur.execute("VACUUM books")
+            print("VACUUM completed")
+        finally:
+            conn.autocommit = False
+
+    print(f"Opening archive: {zip_path}")
+    with zipfile.ZipFile(zip_path, 'r') as zf:
+        files = [(f, zf.getinfo(f).file_size) for f in zf.namelist()]
+        files.sort(key=lambda x: x[1], reverse=True)
+        target_file = files[0][0]
+        print(f"Processing: {target_file} ({files[0][1]} bytes)")
+
+        with zf.open(target_file, 'r') as f:
+            count = 0
+            author_cache = {}
+            genre_cache = {}
+            series_cache = {}
+
+            for line in f:
+                try:
+                    decoded = line.decode('utf-8', errors='ignore')
+                except:
+                    continue
+
+                parts = decoded.strip().split('\x04')
+                if len(parts) < 11:
+                    continue
+
+                authors_str = parts[0].rstrip(',:') if parts[0] else ''
+                genre_str = parts[1] if len(parts) > 1 and parts[1] else ''
+                title = parts[2] if len(parts) > 2 and parts[2] else ''
+                series_name = parts[3].strip() if len(parts) > 3 and parts[3].strip() else None
+                series_number = float(parts[4]) if len(parts) > 4 and parts[4].strip() else None
+
+                internal_id = int(parts[5]) if len(parts) > 5 and parts[5].strip().isdigit() else None
+                file_size = int(parts[6]) if len(parts) > 6 and parts[6].strip().isdigit() else None
+                internal_id2 = int(parts[7]) if len(parts) > 7 and parts[7].strip().isdigit() else None
+
+                format_ = parts[9] if len(parts) > 9 and parts[9].strip() else 'fb2'
+                added_date = parts[10] if len(parts) > 10 and parts[10].strip() else None
+                language = parts[11] if len(parts) > 11 and parts[11].strip() else 'ru'
+
+                title_hash = hashlib.md5(title.lower().encode()).hexdigest() if title else None
+
+                # Серия (с кэшированием)
+                series_id = None
+                if series_name:
+                    if series_name in series_cache:
+                        series_id = series_cache[series_name]
+                    else:
+                        series_id = get_or_create_series(cur, series_name)
+                        series_cache[series_name] = series_id
+
+                # Вставляем книгу
+                cur.execute("""
+                    INSERT INTO books (title, title_hash, internal_id, file_size,
+                                       internal_id2, format, added_date, language,
+                                       source, series_id, series_number)
+                    VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
+                    RETURNING id
+                """, (title, title_hash, internal_id, file_size, internal_id2,
+                      format_, added_date, language, source, series_id, series_number))
+                book_id = cur.fetchone()[0]
+
+                # Авторы (с кэшированием)
+                if authors_str:
+                    author_list = [a.strip() for a in authors_str.split(':') if a.strip()]
+                    for position, author_name in enumerate(author_list, 1):
+                        normalized = normalize_author_name(author_name)
+                        if normalized in author_cache:
+                            author_id = author_cache[normalized]
+                        else:
+                            author_id = get_or_create_author(cur, normalized)
+                            author_cache[normalized] = author_id
+
+                        if author_id:
+                            cur.execute("""
+                                INSERT INTO book_authors (book_id, author_id, position)
+                                VALUES (%s, %s, %s)
+                                ON CONFLICT DO NOTHING
+                            """, (book_id, author_id, position))
+
+                # Жанры (с кэшированием)
+                if genre_str:
+                    for genre_code in genre_str.split(':'):
+                        genre_code = genre_code.strip()
+                        if not genre_code:
+                            continue
+
+                        if genre_code in genre_cache:
+                            genre_id = genre_cache[genre_code]
+                        else:
+                            genre_id = get_or_create_genre(cur, genre_code)
+                            genre_cache[genre_code] = genre_id
+
+                        if genre_id:
+                            cur.execute("""
+                                INSERT INTO book_genres (book_id, genre_id)
+                                VALUES (%s, %s)
+                                ON CONFLICT DO NOTHING
+                            """, (book_id, genre_id))
+
+                count += 1
+
+                if count % 5000 == 0:
+                    conn.commit()
+                    print(f"Loaded {count} books... "
+                          f"(authors: {len(author_cache)}, "
+                          f"series: {len(series_cache)}, "
+                          f"genres: {len(genre_cache)})")
+
+            conn.commit()
+
+    print("Optimizing...")
+    conn.commit()
+    cur.execute("ANALYZE books")
+    cur.execute("ANALYZE authors")
+    cur.execute("ANALYZE series")
+    cur.execute("ANALYZE genres")
+    conn.close()
+    print(f"Done! Total books loaded: {count}")
+
+
+if __name__ == "__main__":
+    if len(sys.argv) != 4:
+        print("Usage: python parse_inpx.py <inpx.zip> <postgres_connection_string> <source>")
+        sys.exit(1)
+    parse_inpx_zip(sys.argv[1], sys.argv[2], sys.argv[3])