diff --git a/plugins/fetch.koplugin/config.lua b/plugins/fetch.koplugin/config.lua new file mode 100644 index 0000000..2e51e21 --- /dev/null +++ b/plugins/fetch.koplugin/config.lua @@ -0,0 +1,4 @@ + return { + server_url = "http://fetch.example.com/get", + auth_token = "fill-in-with-token" + } diff --git a/plugins/fetch.koplugin/main.lua b/plugins/fetch.koplugin/main.lua new file mode 100644 index 0000000..fc721e2 --- /dev/null +++ b/plugins/fetch.koplugin/main.lua @@ -0,0 +1,309 @@ +--[[ + KOReader Fetch Plugin + + Fetches files from a remote server endpoint and saves them to a configured directory. + + Installation: + 1. Copy this file to: koreader/plugins/fetch.koplugin/main.lua + 2. Create config file at: koreader/plugins/fetch.koplugin/config.lua with: + return { + server_url = "http://example.com/get", + auth_token = "your-secret-token-here-change-me" + } + 3. Restart KOReader +]] + +local DataStorage = require("datastorage") +local Dispatcher = require("dispatcher") +local FFIUtil = require("ffi/util") +local InfoMessage = require("ui/widget/infomessage") +local PathChooser = require("ui/widget/pathchooser") +local UIManager = require("ui/uimanager") +local WidgetContainer = require("ui/widget/container/widgetcontainer") +local http = require("socket.http") +local ltn12 = require("ltn12") +local logger = require("logger") +local _ = require("gettext") +local T = FFIUtil.template +local ffi = require("ffi") +local C = ffi.C + +-- Load configuration +local function loadConfig() + local config_path = FFIUtil.joinPath( + FFIUtil.joinPath(DataStorage:getDataDir(), "plugins"), + "fetch.koplugin/config.lua" + ) + + local ok, config = pcall(dofile, config_path) + if ok and config then + return config + else + logger.warn("Fetch plugin: Could not load config file, using defaults") + return { + server_url = "http://192.168.1.100:8000/get", + auth_token = "your-secret-token-here" + } + end +end + +local Fetch = WidgetContainer:extend{ + name = "fetch", + is_doc_only = false, +} + +function Fetch:init() + -- Load server configuration + self.config = loadConfig() + + -- Load or set default home directory + self.settings = G_reader_settings:readSetting("fetch_plugin") or {} + self.home_directory = self.settings.home_directory or "/mnt/onboard/fetched" + + -- Add menu items + self.ui.menu:registerToMainMenu(self) + + logger.info("Fetch plugin initialized") +end + +function Fetch:addToMainMenu(menu_items) + menu_items.fetch = { + text = _("Fetch Files"), + sub_item_table = { + { + text = _("Fetch Now"), + callback = function() + self:fetchFiles() + end, + }, + { + text = _("Configure Home Directory"), + keep_menu_open = true, + callback = function() + self:showDirectoryDialog() + end, + }, + { + text = _("View Settings"), + keep_menu_open = true, + callback = function() + self:showSettings() + end, + }, + }, + } +end + +function Fetch:showSettings() + local text = T(_( + "Server URL: %1\n" .. + "Auth Token: %2\n" .. + "Home Directory: %3\n\n" .. + "To change server settings, edit:\n" .. + "koreader/plugins/fetch.koplugin/config.lua" + ), + self.config.server_url, + string.sub(self.config.auth_token, 1, 10) .. "...", + self.home_directory + ) + + UIManager:show(InfoMessage:new{ + text = text, + timeout = 5, + }) +end + +function Fetch:showDirectoryDialog() + local path_chooser = PathChooser:new{ + title = _("Choose Home Directory"), + path = self.home_directory, + show_files = false, + onConfirm = function(new_dir) + self.home_directory = new_dir + self.settings.home_directory = new_dir + G_reader_settings:saveSetting("fetch_plugin", self.settings) + + UIManager:show(InfoMessage:new{ + text = T(_("Home directory set to:\n%1"), new_dir), + timeout = 2, + }) + end, + } + UIManager:show(path_chooser) +end + +function Fetch:fetchFiles() + logger.info("Fetch: Starting file fetch from " .. self.config.server_url) + + UIManager:show(InfoMessage:new{ + text = _("Fetching files from server..."), + timeout = 1, + }) + + -- Ensure home directory exists + local lfs = require("libs/libkoreader-lfs") + local attributes = lfs.attributes(self.home_directory) + if not attributes or attributes.mode ~= "directory" then + UIManager:show(InfoMessage:new{ + text = T(_("Home directory does not exist:\n%1"), self.home_directory), + timeout = 3, + }) + return + end + + -- Generate temporary file path for downloaded zip + local temp_zip = FFIUtil.joinPath( + DataStorage:getDataDir(), + "fetch_temp_" .. os.time() .. ".zip" + ) + + -- Download file using socket.http + local response_body = {} + local request_body = "" + + local result, status_code, headers = http.request{ + url = self.config.server_url, + method = "POST", + headers = { + ["Authorization"] = "Bearer " .. self.config.auth_token, + ["Content-Length"] = "0", + }, + sink = ltn12.sink.table(response_body) + } + + logger.info("Fetch: HTTP result=" .. tostring(result) .. ", status=" .. tostring(status_code)) + + if not result or status_code ~= 200 then + local error_msg = _("Failed to fetch files from server") + if status_code == 401 then + error_msg = _("Authentication failed. Check your token.") + elseif status_code == 404 then + error_msg = _("No files available on server") + elseif not result then + error_msg = _("Could not connect to server") + end + + UIManager:show(InfoMessage:new{ + text = error_msg, + timeout = 3, + }) + return + end + + -- Concatenate response body + local zip_data = table.concat(response_body) + + if #zip_data == 0 then + UIManager:show(InfoMessage:new{ + text = _("No files received from server"), + timeout = 3, + }) + return + end + + logger.info("Fetch: Downloaded " .. #zip_data .. " bytes") + + -- Write zip data to temp file + local file = io.open(temp_zip, "wb") + if not file then + UIManager:show(InfoMessage:new{ + text = _("Failed to save downloaded file"), + timeout = 3, + }) + return + end + file:write(zip_data) + file:close() + + -- Extract zip file + local extract_success, file_count = self:extractZip(temp_zip, self.home_directory) + + -- Cleanup temp file + self:cleanup(temp_zip) + + if extract_success then + UIManager:show(InfoMessage:new{ + text = T(_("Successfully fetched %1 file(s) to:\n%2"), file_count, self.home_directory), + timeout = 3, + }) + logger.info("Fetch: Successfully extracted " .. file_count .. " files") + else + UIManager:show(InfoMessage:new{ + text = _("Failed to extract downloaded files"), + timeout = 3, + }) + end +end + +function Fetch:extractZip(zip_path, destination) + logger.info("Fetch: Extracting zip to " .. destination) + + -- Use system unzip command + local unzip_cmd = string.format( + 'unzip -o "%s" -d "%s" 2>&1', + zip_path, + destination + ) + + logger.dbg("Fetch: Running unzip command") + local handle = io.popen(unzip_cmd) + if not handle then + logger.err("Fetch: Failed to execute unzip command") + return false, 0 + end + + local output = handle:read("*a") + local success = handle:close() + + logger.info("Fetch: Unzip output: " .. output) + + if not success then + logger.err("Fetch: Failed to extract zip file") + return false, 0 + end + + -- Count extracted files from output + local file_count = 0 + for line in output:gmatch("[^\r\n]+") do + if line:match("inflating:") or line:match("extracting:") then + file_count = file_count + 1 + end + end + + -- If count is 0, try counting files in directory + if file_count == 0 then + local lfs = require("libs/libkoreader-lfs") + for file in lfs.dir(destination) do + if file ~= "." and file ~= ".." then + local attr = lfs.attributes(FFIUtil.joinPath(destination, file)) + if attr and attr.mode == "file" then + file_count = file_count + 1 + end + end + end + end + + return true, file_count +end + +function Fetch:cleanup(file_path) + if file_path then + os.remove(file_path) + logger.dbg("Fetch: Cleaned up temporary file") + end +end + +function Fetch:onDispatcherRegisterActions() + Dispatcher:registerAction("fetch_files", { + category = "none", + event = "FetchFiles", + title = _("Fetch files from server"), + general = true, + }) +end + +function Fetch:onFetchFiles() + self:fetchFiles() +end + +return Fetch diff --git a/server.py b/server.py new file mode 100644 index 0000000..2d0b923 --- /dev/null +++ b/server.py @@ -0,0 +1,224 @@ +#!/usr/bin/env python3 +""" +Robust HTTP file server with authentication and automatic file archiving. +Serves files from a source directory and moves them to an archive directory after serving. +""" + +import os +import shutil +import logging +import zipfile +import io +from pathlib import Path +from http.server import HTTPServer, BaseHTTPRequestHandler +from threading import Lock + +# Configuration +AUTH_TOKEN = "chai7pu5oosigh4Ahzajoocheich9hio" +SOURCE_DIR = Path("/data/books/ingest") +ARCHIVE_DIR = Path("/data/books/served") +HOST = "0.0.0.0" +PORT = 18000 + +# Setup logging +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', + handlers=[ + logging.FileHandler('file_server.log'), + logging.StreamHandler() + ] +) +logger = logging.getLogger(__name__) + +# Thread-safe file operation lock +file_lock = Lock() + + +class FileServerHandler(BaseHTTPRequestHandler): + """HTTP request handler for authenticated file serving.""" + + def log_message(self, format, *args): + """Override to use our logger instead of stderr.""" + logger.info("%s - %s" % (self.get_client_ip(), format % args)) + + def get_client_ip(self): + """Get the real client IP, considering reverse proxy headers.""" + # Check X-Forwarded-For header (set by nginx) + forwarded_for = self.headers.get('X-Forwarded-For') + if forwarded_for: + # X-Forwarded-For can be a comma-separated list, get the first one + return forwarded_for.split(',')[0].strip() + + # Check X-Real-IP header (alternative nginx header) + real_ip = self.headers.get('X-Real-IP') + if real_ip: + return real_ip.strip() + + # Fallback to direct connection IP + return self.address_string() + + def _send_error_response(self, code, message): + """Send a JSON error response.""" + self.send_response(code) + self.send_header('Content-Type', 'application/json') + self.end_headers() + self.wfile.write(f'{{"error": "{message}"}}\n'.encode()) + + def _check_auth(self): + """Verify the authentication token.""" + auth_header = self.headers.get('Authorization', '') + + # Support both "Bearer TOKEN" and just "TOKEN" formats + if auth_header.startswith('Bearer '): + token = auth_header[7:] + else: + token = auth_header + + return token == AUTH_TOKEN + + def _get_files_to_serve(self): + """Get list of files in the source directory.""" + try: + if not SOURCE_DIR.exists(): + logger.error(f"Source directory does not exist: {SOURCE_DIR}") + return [] + + files = [f for f in SOURCE_DIR.iterdir() if f.is_file()] + logger.info(f"Found {len(files)} files to serve") + return files + except Exception as e: + logger.error(f"Error reading source directory: {e}") + return [] + + def _create_zip_archive(self, files): + """Create a zip archive of the files in memory.""" + zip_buffer = io.BytesIO() + + try: + with zipfile.ZipFile(zip_buffer, 'w', zipfile.ZIP_DEFLATED) as zip_file: + for file_path in files: + try: + zip_file.write(file_path, file_path.name) + logger.info(f"Added {file_path.name} to archive") + except Exception as e: + logger.error(f"Failed to add {file_path.name} to archive: {e}") + raise + + return zip_buffer.getvalue() + except Exception as e: + logger.error(f"Error creating zip archive: {e}") + raise + + def _move_files_to_archive(self, files): + """Move served files to the archive directory.""" + # Ensure archive directory exists + ARCHIVE_DIR.mkdir(parents=True, exist_ok=True) + + moved_count = 0 + failed_files = [] + + for file_path in files: + try: + dest_path = ARCHIVE_DIR / file_path.name + + # Handle duplicate filenames + if dest_path.exists(): + base = dest_path.stem + suffix = dest_path.suffix + counter = 1 + while dest_path.exists(): + dest_path = ARCHIVE_DIR / f"{base}_{counter}{suffix}" + counter += 1 + + shutil.move(str(file_path), str(dest_path)) + logger.info(f"Moved {file_path.name} to {dest_path}") + moved_count += 1 + except Exception as e: + logger.error(f"Failed to move {file_path.name}: {e}") + failed_files.append(file_path.name) + + if failed_files: + logger.warning(f"Failed to move {len(failed_files)} files: {failed_files}") + + return moved_count, failed_files + + def do_POST(self): + """Handle POST requests to /get endpoint.""" + if self.path != '/get': + self._send_error_response(404, "Endpoint not found") + return + + # Check authentication + if not self._check_auth(): + logger.warning(f"Unauthorized access attempt from {self.get_client_ip()}") + self._send_error_response(401, "Unauthorized - Invalid token") + return + + # Use lock to prevent concurrent file operations + with file_lock: + try: + # Get files to serve + files = self._get_files_to_serve() + + if not files: + self._send_error_response(404, "No files available to serve") + return + + # Create zip archive + logger.info(f"Creating archive of {len(files)} files") + zip_data = self._create_zip_archive(files) + + # Send the zip file + self.send_response(200) + self.send_header('Content-Type', 'application/zip') + self.send_header('Content-Disposition', 'attachment; filename="files.zip"') + self.send_header('Content-Length', str(len(zip_data))) + self.end_headers() + self.wfile.write(zip_data) + + logger.info(f"Successfully sent {len(zip_data)} bytes to {self.get_client_ip()}") + + # Move files to archive directory + moved, failed = self._move_files_to_archive(files) + logger.info(f"Archived {moved}/{len(files)} files") + + except Exception as e: + logger.error(f"Error processing request: {e}", exc_info=True) + self._send_error_response(500, "Internal server error") + + def do_GET(self): + """Handle GET requests (health check).""" + if self.path == '/health': + self.send_response(200) + self.send_header('Content-Type', 'application/json') + self.end_headers() + self.wfile.write(b'{"status": "ok"}\n') + else: + self._send_error_response(404, "Endpoint not found. Use POST /get") + + +def run_server(): + """Start the HTTP server.""" + # Ensure directories exist + SOURCE_DIR.mkdir(parents=True, exist_ok=True) + ARCHIVE_DIR.mkdir(parents=True, exist_ok=True) + + logger.info(f"Starting server on {HOST}:{PORT}") + logger.info(f"Source directory: {SOURCE_DIR}") + logger.info(f"Archive directory: {ARCHIVE_DIR}") + + server = HTTPServer((HOST, PORT), FileServerHandler) + + try: + logger.info("Server is running. Press Ctrl+C to stop.") + server.serve_forever() + except KeyboardInterrupt: + logger.info("Server stopped by user") + finally: + server.server_close() + logger.info("Server closed") + + +if __name__ == '__main__': + run_server()