First working commit
This commit is contained in:
parent
40e7f746e1
commit
d7e7342276
4
plugins/fetch.koplugin/config.lua
Normal file
4
plugins/fetch.koplugin/config.lua
Normal file
@ -0,0 +1,4 @@
|
||||
return {
|
||||
server_url = "http://fetch.example.com/get",
|
||||
auth_token = "fill-in-with-token"
|
||||
}
|
||||
309
plugins/fetch.koplugin/main.lua
Normal file
309
plugins/fetch.koplugin/main.lua
Normal file
@ -0,0 +1,309 @@
|
||||
--[[
|
||||
KOReader Fetch Plugin
|
||||
|
||||
Fetches files from a remote server endpoint and saves them to a configured directory.
|
||||
|
||||
Installation:
|
||||
1. Copy this file to: koreader/plugins/fetch.koplugin/main.lua
|
||||
2. Create config file at: koreader/plugins/fetch.koplugin/config.lua with:
|
||||
return {
|
||||
server_url = "http://example.com/get",
|
||||
auth_token = "your-secret-token-here-change-me"
|
||||
}
|
||||
3. Restart KOReader
|
||||
]]
|
||||
|
||||
local DataStorage = require("datastorage")
|
||||
local Dispatcher = require("dispatcher")
|
||||
local FFIUtil = require("ffi/util")
|
||||
local InfoMessage = require("ui/widget/infomessage")
|
||||
local PathChooser = require("ui/widget/pathchooser")
|
||||
local UIManager = require("ui/uimanager")
|
||||
local WidgetContainer = require("ui/widget/container/widgetcontainer")
|
||||
local http = require("socket.http")
|
||||
local ltn12 = require("ltn12")
|
||||
local logger = require("logger")
|
||||
local _ = require("gettext")
|
||||
local T = FFIUtil.template
|
||||
local ffi = require("ffi")
|
||||
local C = ffi.C
|
||||
|
||||
-- Load configuration
|
||||
local function loadConfig()
|
||||
local config_path = FFIUtil.joinPath(
|
||||
FFIUtil.joinPath(DataStorage:getDataDir(), "plugins"),
|
||||
"fetch.koplugin/config.lua"
|
||||
)
|
||||
|
||||
local ok, config = pcall(dofile, config_path)
|
||||
if ok and config then
|
||||
return config
|
||||
else
|
||||
logger.warn("Fetch plugin: Could not load config file, using defaults")
|
||||
return {
|
||||
server_url = "http://192.168.1.100:8000/get",
|
||||
auth_token = "your-secret-token-here"
|
||||
}
|
||||
end
|
||||
end
|
||||
|
||||
local Fetch = WidgetContainer:extend{
|
||||
name = "fetch",
|
||||
is_doc_only = false,
|
||||
}
|
||||
|
||||
function Fetch:init()
|
||||
-- Load server configuration
|
||||
self.config = loadConfig()
|
||||
|
||||
-- Load or set default home directory
|
||||
self.settings = G_reader_settings:readSetting("fetch_plugin") or {}
|
||||
self.home_directory = self.settings.home_directory or "/mnt/onboard/fetched"
|
||||
|
||||
-- Add menu items
|
||||
self.ui.menu:registerToMainMenu(self)
|
||||
|
||||
logger.info("Fetch plugin initialized")
|
||||
end
|
||||
|
||||
function Fetch:addToMainMenu(menu_items)
|
||||
menu_items.fetch = {
|
||||
text = _("Fetch Files"),
|
||||
sub_item_table = {
|
||||
{
|
||||
text = _("Fetch Now"),
|
||||
callback = function()
|
||||
self:fetchFiles()
|
||||
end,
|
||||
},
|
||||
{
|
||||
text = _("Configure Home Directory"),
|
||||
keep_menu_open = true,
|
||||
callback = function()
|
||||
self:showDirectoryDialog()
|
||||
end,
|
||||
},
|
||||
{
|
||||
text = _("View Settings"),
|
||||
keep_menu_open = true,
|
||||
callback = function()
|
||||
self:showSettings()
|
||||
end,
|
||||
},
|
||||
},
|
||||
}
|
||||
end
|
||||
|
||||
function Fetch:showSettings()
|
||||
local text = T(_(
|
||||
"Server URL: %1\n" ..
|
||||
"Auth Token: %2\n" ..
|
||||
"Home Directory: %3\n\n" ..
|
||||
"To change server settings, edit:\n" ..
|
||||
"koreader/plugins/fetch.koplugin/config.lua"
|
||||
),
|
||||
self.config.server_url,
|
||||
string.sub(self.config.auth_token, 1, 10) .. "...",
|
||||
self.home_directory
|
||||
)
|
||||
|
||||
UIManager:show(InfoMessage:new{
|
||||
text = text,
|
||||
timeout = 5,
|
||||
})
|
||||
end
|
||||
|
||||
function Fetch:showDirectoryDialog()
|
||||
local path_chooser = PathChooser:new{
|
||||
title = _("Choose Home Directory"),
|
||||
path = self.home_directory,
|
||||
show_files = false,
|
||||
onConfirm = function(new_dir)
|
||||
self.home_directory = new_dir
|
||||
self.settings.home_directory = new_dir
|
||||
G_reader_settings:saveSetting("fetch_plugin", self.settings)
|
||||
|
||||
UIManager:show(InfoMessage:new{
|
||||
text = T(_("Home directory set to:\n%1"), new_dir),
|
||||
timeout = 2,
|
||||
})
|
||||
end,
|
||||
}
|
||||
UIManager:show(path_chooser)
|
||||
end
|
||||
|
||||
function Fetch:fetchFiles()
|
||||
logger.info("Fetch: Starting file fetch from " .. self.config.server_url)
|
||||
|
||||
UIManager:show(InfoMessage:new{
|
||||
text = _("Fetching files from server..."),
|
||||
timeout = 1,
|
||||
})
|
||||
|
||||
-- Ensure home directory exists
|
||||
local lfs = require("libs/libkoreader-lfs")
|
||||
local attributes = lfs.attributes(self.home_directory)
|
||||
if not attributes or attributes.mode ~= "directory" then
|
||||
UIManager:show(InfoMessage:new{
|
||||
text = T(_("Home directory does not exist:\n%1"), self.home_directory),
|
||||
timeout = 3,
|
||||
})
|
||||
return
|
||||
end
|
||||
|
||||
-- Generate temporary file path for downloaded zip
|
||||
local temp_zip = FFIUtil.joinPath(
|
||||
DataStorage:getDataDir(),
|
||||
"fetch_temp_" .. os.time() .. ".zip"
|
||||
)
|
||||
|
||||
-- Download file using socket.http
|
||||
local response_body = {}
|
||||
local request_body = ""
|
||||
|
||||
local result, status_code, headers = http.request{
|
||||
url = self.config.server_url,
|
||||
method = "POST",
|
||||
headers = {
|
||||
["Authorization"] = "Bearer " .. self.config.auth_token,
|
||||
["Content-Length"] = "0",
|
||||
},
|
||||
sink = ltn12.sink.table(response_body)
|
||||
}
|
||||
|
||||
logger.info("Fetch: HTTP result=" .. tostring(result) .. ", status=" .. tostring(status_code))
|
||||
|
||||
if not result or status_code ~= 200 then
|
||||
local error_msg = _("Failed to fetch files from server")
|
||||
if status_code == 401 then
|
||||
error_msg = _("Authentication failed. Check your token.")
|
||||
elseif status_code == 404 then
|
||||
error_msg = _("No files available on server")
|
||||
elseif not result then
|
||||
error_msg = _("Could not connect to server")
|
||||
end
|
||||
|
||||
UIManager:show(InfoMessage:new{
|
||||
text = error_msg,
|
||||
timeout = 3,
|
||||
})
|
||||
return
|
||||
end
|
||||
|
||||
-- Concatenate response body
|
||||
local zip_data = table.concat(response_body)
|
||||
|
||||
if #zip_data == 0 then
|
||||
UIManager:show(InfoMessage:new{
|
||||
text = _("No files received from server"),
|
||||
timeout = 3,
|
||||
})
|
||||
return
|
||||
end
|
||||
|
||||
logger.info("Fetch: Downloaded " .. #zip_data .. " bytes")
|
||||
|
||||
-- Write zip data to temp file
|
||||
local file = io.open(temp_zip, "wb")
|
||||
if not file then
|
||||
UIManager:show(InfoMessage:new{
|
||||
text = _("Failed to save downloaded file"),
|
||||
timeout = 3,
|
||||
})
|
||||
return
|
||||
end
|
||||
file:write(zip_data)
|
||||
file:close()
|
||||
|
||||
-- Extract zip file
|
||||
local extract_success, file_count = self:extractZip(temp_zip, self.home_directory)
|
||||
|
||||
-- Cleanup temp file
|
||||
self:cleanup(temp_zip)
|
||||
|
||||
if extract_success then
|
||||
UIManager:show(InfoMessage:new{
|
||||
text = T(_("Successfully fetched %1 file(s) to:\n%2"), file_count, self.home_directory),
|
||||
timeout = 3,
|
||||
})
|
||||
logger.info("Fetch: Successfully extracted " .. file_count .. " files")
|
||||
else
|
||||
UIManager:show(InfoMessage:new{
|
||||
text = _("Failed to extract downloaded files"),
|
||||
timeout = 3,
|
||||
})
|
||||
end
|
||||
end
|
||||
|
||||
function Fetch:extractZip(zip_path, destination)
|
||||
logger.info("Fetch: Extracting zip to " .. destination)
|
||||
|
||||
-- Use system unzip command
|
||||
local unzip_cmd = string.format(
|
||||
'unzip -o "%s" -d "%s" 2>&1',
|
||||
zip_path,
|
||||
destination
|
||||
)
|
||||
|
||||
logger.dbg("Fetch: Running unzip command")
|
||||
local handle = io.popen(unzip_cmd)
|
||||
if not handle then
|
||||
logger.err("Fetch: Failed to execute unzip command")
|
||||
return false, 0
|
||||
end
|
||||
|
||||
local output = handle:read("*a")
|
||||
local success = handle:close()
|
||||
|
||||
logger.info("Fetch: Unzip output: " .. output)
|
||||
|
||||
if not success then
|
||||
logger.err("Fetch: Failed to extract zip file")
|
||||
return false, 0
|
||||
end
|
||||
|
||||
-- Count extracted files from output
|
||||
local file_count = 0
|
||||
for line in output:gmatch("[^\r\n]+") do
|
||||
if line:match("inflating:") or line:match("extracting:") then
|
||||
file_count = file_count + 1
|
||||
end
|
||||
end
|
||||
|
||||
-- If count is 0, try counting files in directory
|
||||
if file_count == 0 then
|
||||
local lfs = require("libs/libkoreader-lfs")
|
||||
for file in lfs.dir(destination) do
|
||||
if file ~= "." and file ~= ".." then
|
||||
local attr = lfs.attributes(FFIUtil.joinPath(destination, file))
|
||||
if attr and attr.mode == "file" then
|
||||
file_count = file_count + 1
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
return true, file_count
|
||||
end
|
||||
|
||||
function Fetch:cleanup(file_path)
|
||||
if file_path then
|
||||
os.remove(file_path)
|
||||
logger.dbg("Fetch: Cleaned up temporary file")
|
||||
end
|
||||
end
|
||||
|
||||
function Fetch:onDispatcherRegisterActions()
|
||||
Dispatcher:registerAction("fetch_files", {
|
||||
category = "none",
|
||||
event = "FetchFiles",
|
||||
title = _("Fetch files from server"),
|
||||
general = true,
|
||||
})
|
||||
end
|
||||
|
||||
function Fetch:onFetchFiles()
|
||||
self:fetchFiles()
|
||||
end
|
||||
|
||||
return Fetch
|
||||
224
server.py
Normal file
224
server.py
Normal file
@ -0,0 +1,224 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Robust HTTP file server with authentication and automatic file archiving.
|
||||
Serves files from a source directory and moves them to an archive directory after serving.
|
||||
"""
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import logging
|
||||
import zipfile
|
||||
import io
|
||||
from pathlib import Path
|
||||
from http.server import HTTPServer, BaseHTTPRequestHandler
|
||||
from threading import Lock
|
||||
|
||||
# Configuration
|
||||
AUTH_TOKEN = "chai7pu5oosigh4Ahzajoocheich9hio"
|
||||
SOURCE_DIR = Path("/data/books/ingest")
|
||||
ARCHIVE_DIR = Path("/data/books/served")
|
||||
HOST = "0.0.0.0"
|
||||
PORT = 18000
|
||||
|
||||
# Setup logging
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
|
||||
handlers=[
|
||||
logging.FileHandler('file_server.log'),
|
||||
logging.StreamHandler()
|
||||
]
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Thread-safe file operation lock
|
||||
file_lock = Lock()
|
||||
|
||||
|
||||
class FileServerHandler(BaseHTTPRequestHandler):
|
||||
"""HTTP request handler for authenticated file serving."""
|
||||
|
||||
def log_message(self, format, *args):
|
||||
"""Override to use our logger instead of stderr."""
|
||||
logger.info("%s - %s" % (self.get_client_ip(), format % args))
|
||||
|
||||
def get_client_ip(self):
|
||||
"""Get the real client IP, considering reverse proxy headers."""
|
||||
# Check X-Forwarded-For header (set by nginx)
|
||||
forwarded_for = self.headers.get('X-Forwarded-For')
|
||||
if forwarded_for:
|
||||
# X-Forwarded-For can be a comma-separated list, get the first one
|
||||
return forwarded_for.split(',')[0].strip()
|
||||
|
||||
# Check X-Real-IP header (alternative nginx header)
|
||||
real_ip = self.headers.get('X-Real-IP')
|
||||
if real_ip:
|
||||
return real_ip.strip()
|
||||
|
||||
# Fallback to direct connection IP
|
||||
return self.address_string()
|
||||
|
||||
def _send_error_response(self, code, message):
|
||||
"""Send a JSON error response."""
|
||||
self.send_response(code)
|
||||
self.send_header('Content-Type', 'application/json')
|
||||
self.end_headers()
|
||||
self.wfile.write(f'{{"error": "{message}"}}\n'.encode())
|
||||
|
||||
def _check_auth(self):
|
||||
"""Verify the authentication token."""
|
||||
auth_header = self.headers.get('Authorization', '')
|
||||
|
||||
# Support both "Bearer TOKEN" and just "TOKEN" formats
|
||||
if auth_header.startswith('Bearer '):
|
||||
token = auth_header[7:]
|
||||
else:
|
||||
token = auth_header
|
||||
|
||||
return token == AUTH_TOKEN
|
||||
|
||||
def _get_files_to_serve(self):
|
||||
"""Get list of files in the source directory."""
|
||||
try:
|
||||
if not SOURCE_DIR.exists():
|
||||
logger.error(f"Source directory does not exist: {SOURCE_DIR}")
|
||||
return []
|
||||
|
||||
files = [f for f in SOURCE_DIR.iterdir() if f.is_file()]
|
||||
logger.info(f"Found {len(files)} files to serve")
|
||||
return files
|
||||
except Exception as e:
|
||||
logger.error(f"Error reading source directory: {e}")
|
||||
return []
|
||||
|
||||
def _create_zip_archive(self, files):
|
||||
"""Create a zip archive of the files in memory."""
|
||||
zip_buffer = io.BytesIO()
|
||||
|
||||
try:
|
||||
with zipfile.ZipFile(zip_buffer, 'w', zipfile.ZIP_DEFLATED) as zip_file:
|
||||
for file_path in files:
|
||||
try:
|
||||
zip_file.write(file_path, file_path.name)
|
||||
logger.info(f"Added {file_path.name} to archive")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to add {file_path.name} to archive: {e}")
|
||||
raise
|
||||
|
||||
return zip_buffer.getvalue()
|
||||
except Exception as e:
|
||||
logger.error(f"Error creating zip archive: {e}")
|
||||
raise
|
||||
|
||||
def _move_files_to_archive(self, files):
|
||||
"""Move served files to the archive directory."""
|
||||
# Ensure archive directory exists
|
||||
ARCHIVE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
moved_count = 0
|
||||
failed_files = []
|
||||
|
||||
for file_path in files:
|
||||
try:
|
||||
dest_path = ARCHIVE_DIR / file_path.name
|
||||
|
||||
# Handle duplicate filenames
|
||||
if dest_path.exists():
|
||||
base = dest_path.stem
|
||||
suffix = dest_path.suffix
|
||||
counter = 1
|
||||
while dest_path.exists():
|
||||
dest_path = ARCHIVE_DIR / f"{base}_{counter}{suffix}"
|
||||
counter += 1
|
||||
|
||||
shutil.move(str(file_path), str(dest_path))
|
||||
logger.info(f"Moved {file_path.name} to {dest_path}")
|
||||
moved_count += 1
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to move {file_path.name}: {e}")
|
||||
failed_files.append(file_path.name)
|
||||
|
||||
if failed_files:
|
||||
logger.warning(f"Failed to move {len(failed_files)} files: {failed_files}")
|
||||
|
||||
return moved_count, failed_files
|
||||
|
||||
def do_POST(self):
|
||||
"""Handle POST requests to /get endpoint."""
|
||||
if self.path != '/get':
|
||||
self._send_error_response(404, "Endpoint not found")
|
||||
return
|
||||
|
||||
# Check authentication
|
||||
if not self._check_auth():
|
||||
logger.warning(f"Unauthorized access attempt from {self.get_client_ip()}")
|
||||
self._send_error_response(401, "Unauthorized - Invalid token")
|
||||
return
|
||||
|
||||
# Use lock to prevent concurrent file operations
|
||||
with file_lock:
|
||||
try:
|
||||
# Get files to serve
|
||||
files = self._get_files_to_serve()
|
||||
|
||||
if not files:
|
||||
self._send_error_response(404, "No files available to serve")
|
||||
return
|
||||
|
||||
# Create zip archive
|
||||
logger.info(f"Creating archive of {len(files)} files")
|
||||
zip_data = self._create_zip_archive(files)
|
||||
|
||||
# Send the zip file
|
||||
self.send_response(200)
|
||||
self.send_header('Content-Type', 'application/zip')
|
||||
self.send_header('Content-Disposition', 'attachment; filename="files.zip"')
|
||||
self.send_header('Content-Length', str(len(zip_data)))
|
||||
self.end_headers()
|
||||
self.wfile.write(zip_data)
|
||||
|
||||
logger.info(f"Successfully sent {len(zip_data)} bytes to {self.get_client_ip()}")
|
||||
|
||||
# Move files to archive directory
|
||||
moved, failed = self._move_files_to_archive(files)
|
||||
logger.info(f"Archived {moved}/{len(files)} files")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error processing request: {e}", exc_info=True)
|
||||
self._send_error_response(500, "Internal server error")
|
||||
|
||||
def do_GET(self):
|
||||
"""Handle GET requests (health check)."""
|
||||
if self.path == '/health':
|
||||
self.send_response(200)
|
||||
self.send_header('Content-Type', 'application/json')
|
||||
self.end_headers()
|
||||
self.wfile.write(b'{"status": "ok"}\n')
|
||||
else:
|
||||
self._send_error_response(404, "Endpoint not found. Use POST /get")
|
||||
|
||||
|
||||
def run_server():
|
||||
"""Start the HTTP server."""
|
||||
# Ensure directories exist
|
||||
SOURCE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
ARCHIVE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
logger.info(f"Starting server on {HOST}:{PORT}")
|
||||
logger.info(f"Source directory: {SOURCE_DIR}")
|
||||
logger.info(f"Archive directory: {ARCHIVE_DIR}")
|
||||
|
||||
server = HTTPServer((HOST, PORT), FileServerHandler)
|
||||
|
||||
try:
|
||||
logger.info("Server is running. Press Ctrl+C to stop.")
|
||||
server.serve_forever()
|
||||
except KeyboardInterrupt:
|
||||
logger.info("Server stopped by user")
|
||||
finally:
|
||||
server.server_close()
|
||||
logger.info("Server closed")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run_server()
|
||||
Loading…
x
Reference in New Issue
Block a user