First working commit

plugins/fetch.koplugin/config.lua

@@ -0,0 +1,4 @@
+   return {
+       server_url = "http://fetch.example.com/get",
+       auth_token = "fill-in-with-token"
+   }

plugins/fetch.koplugin/main.lua

@@ -0,0 +1,309 @@
+--[[
+    KOReader Fetch Plugin
+    
+    Fetches files from a remote server endpoint and saves them to a configured directory.
+    
+    Installation:
+    1. Copy this file to: koreader/plugins/fetch.koplugin/main.lua
+    2. Create config file at: koreader/plugins/fetch.koplugin/config.lua with:
+       return {
+           server_url = "http://example.com/get",
+           auth_token = "your-secret-token-here-change-me"
+       }
+    3. Restart KOReader
+]]
+
+local DataStorage = require("datastorage")
+local Dispatcher = require("dispatcher")
+local FFIUtil = require("ffi/util")
+local InfoMessage = require("ui/widget/infomessage")
+local PathChooser = require("ui/widget/pathchooser")
+local UIManager = require("ui/uimanager")
+local WidgetContainer = require("ui/widget/container/widgetcontainer")
+local http = require("socket.http")
+local ltn12 = require("ltn12")
+local logger = require("logger")
+local _ = require("gettext")
+local T = FFIUtil.template
+local ffi = require("ffi")
+local C = ffi.C
+
+-- Load configuration
+local function loadConfig()
+    local config_path = FFIUtil.joinPath(
+        FFIUtil.joinPath(DataStorage:getDataDir(), "plugins"),
+        "fetch.koplugin/config.lua"
+    )
+    
+    local ok, config = pcall(dofile, config_path)
+    if ok and config then
+        return config
+    else
+        logger.warn("Fetch plugin: Could not load config file, using defaults")
+        return {
+            server_url = "http://192.168.1.100:8000/get",
+            auth_token = "your-secret-token-here"
+        }
+    end
+end
+
+local Fetch = WidgetContainer:extend{
+    name = "fetch",
+    is_doc_only = false,
+}
+
+function Fetch:init()
+    -- Load server configuration
+    self.config = loadConfig()
+    
+    -- Load or set default home directory
+    self.settings = G_reader_settings:readSetting("fetch_plugin") or {}
+    self.home_directory = self.settings.home_directory or "/mnt/onboard/fetched"
+    
+    -- Add menu items
+    self.ui.menu:registerToMainMenu(self)
+    
+    logger.info("Fetch plugin initialized")
+end
+
+function Fetch:addToMainMenu(menu_items)
+    menu_items.fetch = {
+        text = _("Fetch Files"),
+        sub_item_table = {
+            {
+                text = _("Fetch Now"),
+                callback = function()
+                    self:fetchFiles()
+                end,
+            },
+            {
+                text = _("Configure Home Directory"),
+                keep_menu_open = true,
+                callback = function()
+                    self:showDirectoryDialog()
+                end,
+            },
+            {
+                text = _("View Settings"),
+                keep_menu_open = true,
+                callback = function()
+                    self:showSettings()
+                end,
+            },
+        },
+    }
+end
+
+function Fetch:showSettings()
+    local text = T(_(
+        "Server URL: %1\n" ..
+        "Auth Token: %2\n" ..
+        "Home Directory: %3\n\n" ..
+        "To change server settings, edit:\n" ..
+        "koreader/plugins/fetch.koplugin/config.lua"
+    ),
+        self.config.server_url,
+        string.sub(self.config.auth_token, 1, 10) .. "...",
+        self.home_directory
+    )
+    
+    UIManager:show(InfoMessage:new{
+        text = text,
+        timeout = 5,
+    })
+end
+
+function Fetch:showDirectoryDialog()
+    local path_chooser = PathChooser:new{
+        title = _("Choose Home Directory"),
+        path = self.home_directory,
+        show_files = false,
+        onConfirm = function(new_dir)
+            self.home_directory = new_dir
+            self.settings.home_directory = new_dir
+            G_reader_settings:saveSetting("fetch_plugin", self.settings)
+            
+            UIManager:show(InfoMessage:new{
+                text = T(_("Home directory set to:\n%1"), new_dir),
+                timeout = 2,
+            })
+        end,
+    }
+    UIManager:show(path_chooser)
+end
+
+function Fetch:fetchFiles()
+    logger.info("Fetch: Starting file fetch from " .. self.config.server_url)
+    
+    UIManager:show(InfoMessage:new{
+        text = _("Fetching files from server..."),
+        timeout = 1,
+    })
+    
+    -- Ensure home directory exists
+    local lfs = require("libs/libkoreader-lfs")
+    local attributes = lfs.attributes(self.home_directory)
+    if not attributes or attributes.mode ~= "directory" then
+        UIManager:show(InfoMessage:new{
+            text = T(_("Home directory does not exist:\n%1"), self.home_directory),
+            timeout = 3,
+        })
+        return
+    end
+    
+    -- Generate temporary file path for downloaded zip
+    local temp_zip = FFIUtil.joinPath(
+        DataStorage:getDataDir(),
+        "fetch_temp_" .. os.time() .. ".zip"
+    )
+    
+    -- Download file using socket.http
+    local response_body = {}
+    local request_body = ""
+    
+    local result, status_code, headers = http.request{
+        url = self.config.server_url,
+        method = "POST",
+        headers = {
+            ["Authorization"] = "Bearer " .. self.config.auth_token,
+            ["Content-Length"] = "0",
+        },
+        sink = ltn12.sink.table(response_body)
+    }
+    
+    logger.info("Fetch: HTTP result=" .. tostring(result) .. ", status=" .. tostring(status_code))
+    
+    if not result or status_code ~= 200 then
+        local error_msg = _("Failed to fetch files from server")
+        if status_code == 401 then
+            error_msg = _("Authentication failed. Check your token.")
+        elseif status_code == 404 then
+            error_msg = _("No files available on server")
+        elseif not result then
+            error_msg = _("Could not connect to server")
+        end
+        
+        UIManager:show(InfoMessage:new{
+            text = error_msg,
+            timeout = 3,
+        })
+        return
+    end
+    
+    -- Concatenate response body
+    local zip_data = table.concat(response_body)
+    
+    if #zip_data == 0 then
+        UIManager:show(InfoMessage:new{
+            text = _("No files received from server"),
+            timeout = 3,
+        })
+        return
+    end
+    
+    logger.info("Fetch: Downloaded " .. #zip_data .. " bytes")
+    
+    -- Write zip data to temp file
+    local file = io.open(temp_zip, "wb")
+    if not file then
+        UIManager:show(InfoMessage:new{
+            text = _("Failed to save downloaded file"),
+            timeout = 3,
+        })
+        return
+    end
+    file:write(zip_data)
+    file:close()
+    
+    -- Extract zip file
+    local extract_success, file_count = self:extractZip(temp_zip, self.home_directory)
+    
+    -- Cleanup temp file
+    self:cleanup(temp_zip)
+    
+    if extract_success then
+        UIManager:show(InfoMessage:new{
+            text = T(_("Successfully fetched %1 file(s) to:\n%2"), file_count, self.home_directory),
+            timeout = 3,
+        })
+        logger.info("Fetch: Successfully extracted " .. file_count .. " files")
+    else
+        UIManager:show(InfoMessage:new{
+            text = _("Failed to extract downloaded files"),
+            timeout = 3,
+        })
+    end
+end
+
+function Fetch:extractZip(zip_path, destination)
+    logger.info("Fetch: Extracting zip to " .. destination)
+    
+    -- Use system unzip command
+    local unzip_cmd = string.format(
+        'unzip -o "%s" -d "%s" 2>&1',
+        zip_path,
+        destination
+    )
+    
+    logger.dbg("Fetch: Running unzip command")
+    local handle = io.popen(unzip_cmd)
+    if not handle then
+        logger.err("Fetch: Failed to execute unzip command")
+        return false, 0
+    end
+    
+    local output = handle:read("*a")
+    local success = handle:close()
+    
+    logger.info("Fetch: Unzip output: " .. output)
+    
+    if not success then
+        logger.err("Fetch: Failed to extract zip file")
+        return false, 0
+    end
+    
+    -- Count extracted files from output
+    local file_count = 0
+    for line in output:gmatch("[^\r\n]+") do
+        if line:match("inflating:") or line:match("extracting:") then
+            file_count = file_count + 1
+        end
+    end
+    
+    -- If count is 0, try counting files in directory
+    if file_count == 0 then
+        local lfs = require("libs/libkoreader-lfs")
+        for file in lfs.dir(destination) do
+            if file ~= "." and file ~= ".." then
+                local attr = lfs.attributes(FFIUtil.joinPath(destination, file))
+                if attr and attr.mode == "file" then
+                    file_count = file_count + 1
+                end
+            end
+        end
+    end
+    
+    return true, file_count
+end
+
+function Fetch:cleanup(file_path)
+    if file_path then
+        os.remove(file_path)
+        logger.dbg("Fetch: Cleaned up temporary file")
+    end
+end
+
+function Fetch:onDispatcherRegisterActions()
+    Dispatcher:registerAction("fetch_files", {
+        category = "none",
+        event = "FetchFiles",
+        title = _("Fetch files from server"),
+        general = true,
+    })
+end
+
+function Fetch:onFetchFiles()
+    self:fetchFiles()
+end
+
+return Fetch

server.py

@@ -0,0 +1,224 @@
+#!/usr/bin/env python3
+"""
+Robust HTTP file server with authentication and automatic file archiving.
+Serves files from a source directory and moves them to an archive directory after serving.
+"""
+
+import os
+import shutil
+import logging
+import zipfile
+import io
+from pathlib import Path
+from http.server import HTTPServer, BaseHTTPRequestHandler
+from threading import Lock
+
+# Configuration
+AUTH_TOKEN = "chai7pu5oosigh4Ahzajoocheich9hio"
+SOURCE_DIR = Path("/data/books/ingest")
+ARCHIVE_DIR = Path("/data/books/served")
+HOST = "0.0.0.0"
+PORT = 18000
+
+# Setup logging
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+    handlers=[
+        logging.FileHandler('file_server.log'),
+        logging.StreamHandler()
+    ]
+)
+logger = logging.getLogger(__name__)
+
+# Thread-safe file operation lock
+file_lock = Lock()
+
+
+class FileServerHandler(BaseHTTPRequestHandler):
+    """HTTP request handler for authenticated file serving."""
+    
+    def log_message(self, format, *args):
+        """Override to use our logger instead of stderr."""
+        logger.info("%s - %s" % (self.get_client_ip(), format % args))
+    
+    def get_client_ip(self):
+        """Get the real client IP, considering reverse proxy headers."""
+        # Check X-Forwarded-For header (set by nginx)
+        forwarded_for = self.headers.get('X-Forwarded-For')
+        if forwarded_for:
+            # X-Forwarded-For can be a comma-separated list, get the first one
+            return forwarded_for.split(',')[0].strip()
+        
+        # Check X-Real-IP header (alternative nginx header)
+        real_ip = self.headers.get('X-Real-IP')
+        if real_ip:
+            return real_ip.strip()
+        
+        # Fallback to direct connection IP
+        return self.address_string()
+    
+    def _send_error_response(self, code, message):
+        """Send a JSON error response."""
+        self.send_response(code)
+        self.send_header('Content-Type', 'application/json')
+        self.end_headers()
+        self.wfile.write(f'{{"error": "{message}"}}\n'.encode())
+    
+    def _check_auth(self):
+        """Verify the authentication token."""
+        auth_header = self.headers.get('Authorization', '')
+        
+        # Support both "Bearer TOKEN" and just "TOKEN" formats
+        if auth_header.startswith('Bearer '):
+            token = auth_header[7:]
+        else:
+            token = auth_header
+        
+        return token == AUTH_TOKEN
+    
+    def _get_files_to_serve(self):
+        """Get list of files in the source directory."""
+        try:
+            if not SOURCE_DIR.exists():
+                logger.error(f"Source directory does not exist: {SOURCE_DIR}")
+                return []
+            
+            files = [f for f in SOURCE_DIR.iterdir() if f.is_file()]
+            logger.info(f"Found {len(files)} files to serve")
+            return files
+        except Exception as e:
+            logger.error(f"Error reading source directory: {e}")
+            return []
+    
+    def _create_zip_archive(self, files):
+        """Create a zip archive of the files in memory."""
+        zip_buffer = io.BytesIO()
+        
+        try:
+            with zipfile.ZipFile(zip_buffer, 'w', zipfile.ZIP_DEFLATED) as zip_file:
+                for file_path in files:
+                    try:
+                        zip_file.write(file_path, file_path.name)
+                        logger.info(f"Added {file_path.name} to archive")
+                    except Exception as e:
+                        logger.error(f"Failed to add {file_path.name} to archive: {e}")
+                        raise
+            
+            return zip_buffer.getvalue()
+        except Exception as e:
+            logger.error(f"Error creating zip archive: {e}")
+            raise
+    
+    def _move_files_to_archive(self, files):
+        """Move served files to the archive directory."""
+        # Ensure archive directory exists
+        ARCHIVE_DIR.mkdir(parents=True, exist_ok=True)
+        
+        moved_count = 0
+        failed_files = []
+        
+        for file_path in files:
+            try:
+                dest_path = ARCHIVE_DIR / file_path.name
+                
+                # Handle duplicate filenames
+                if dest_path.exists():
+                    base = dest_path.stem
+                    suffix = dest_path.suffix
+                    counter = 1
+                    while dest_path.exists():
+                        dest_path = ARCHIVE_DIR / f"{base}_{counter}{suffix}"
+                        counter += 1
+                
+                shutil.move(str(file_path), str(dest_path))
+                logger.info(f"Moved {file_path.name} to {dest_path}")
+                moved_count += 1
+            except Exception as e:
+                logger.error(f"Failed to move {file_path.name}: {e}")
+                failed_files.append(file_path.name)
+        
+        if failed_files:
+            logger.warning(f"Failed to move {len(failed_files)} files: {failed_files}")
+        
+        return moved_count, failed_files
+    
+    def do_POST(self):
+        """Handle POST requests to /get endpoint."""
+        if self.path != '/get':
+            self._send_error_response(404, "Endpoint not found")
+            return
+        
+        # Check authentication
+        if not self._check_auth():
+            logger.warning(f"Unauthorized access attempt from {self.get_client_ip()}")
+            self._send_error_response(401, "Unauthorized - Invalid token")
+            return
+        
+        # Use lock to prevent concurrent file operations
+        with file_lock:
+            try:
+                # Get files to serve
+                files = self._get_files_to_serve()
+                
+                if not files:
+                    self._send_error_response(404, "No files available to serve")
+                    return
+                
+                # Create zip archive
+                logger.info(f"Creating archive of {len(files)} files")
+                zip_data = self._create_zip_archive(files)
+                
+                # Send the zip file
+                self.send_response(200)
+                self.send_header('Content-Type', 'application/zip')
+                self.send_header('Content-Disposition', 'attachment; filename="files.zip"')
+                self.send_header('Content-Length', str(len(zip_data)))
+                self.end_headers()
+                self.wfile.write(zip_data)
+                
+                logger.info(f"Successfully sent {len(zip_data)} bytes to {self.get_client_ip()}")
+                
+                # Move files to archive directory
+                moved, failed = self._move_files_to_archive(files)
+                logger.info(f"Archived {moved}/{len(files)} files")
+                
+            except Exception as e:
+                logger.error(f"Error processing request: {e}", exc_info=True)
+                self._send_error_response(500, "Internal server error")
+    
+    def do_GET(self):
+        """Handle GET requests (health check)."""
+        if self.path == '/health':
+            self.send_response(200)
+            self.send_header('Content-Type', 'application/json')
+            self.end_headers()
+            self.wfile.write(b'{"status": "ok"}\n')
+        else:
+            self._send_error_response(404, "Endpoint not found. Use POST /get")
+
+
+def run_server():
+    """Start the HTTP server."""
+    # Ensure directories exist
+    SOURCE_DIR.mkdir(parents=True, exist_ok=True)
+    ARCHIVE_DIR.mkdir(parents=True, exist_ok=True)
+    
+    logger.info(f"Starting server on {HOST}:{PORT}")
+    logger.info(f"Source directory: {SOURCE_DIR}")
+    logger.info(f"Archive directory: {ARCHIVE_DIR}")
+    
+    server = HTTPServer((HOST, PORT), FileServerHandler)
+    
+    try:
+        logger.info("Server is running. Press Ctrl+C to stop.")
+        server.serve_forever()
+    except KeyboardInterrupt:
+        logger.info("Server stopped by user")
+    finally:
+        server.server_close()
+        logger.info("Server closed")
+
+
+if __name__ == '__main__':
+    run_server()