jim800121chen
efa67d59a4
- Frontend: rewrite Home.vue to match backend POST /jobs API (remove single-stage options) - Frontend: add Monitor page (/monitor) for queue and job monitoring - Frontend: add job history with localStorage tracking (per-browser) - Frontend: fix Nginx proxy rewrite (/api -> /) and add 500MB upload limit - Backend: add MinIO storage support (STORAGE_BACKEND=minio) alongside local mode - Backend: add GET /queues/stats API for queue monitoring - Backend: fix download handler for MinIO (buffer mode for Node 18 compat) - Workers: add S3/MinIO download/upload in consumer.py with isolated temp dirs - Workers: add s3_storage.py helper with lifecycle rule (7-day TTL) - Docker: add docker-compose.yml with all services (web, scheduler, redis, workers) - Docker: ports mapped to 9500 (web) and 9501 (scheduler) - Config: add .env to .gitignore to protect secrets Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
54 lines
1.2 KiB
Python
54 lines
1.2 KiB
Python
"""
|
|
NEF Worker — Redis Stream queue consumer entry point.
|
|
|
|
Usage:
|
|
python -m services.workers.nef.worker
|
|
# or
|
|
WORKER_MODE=stub python -m services.workers.nef.worker
|
|
"""
|
|
|
|
import logging
|
|
import os
|
|
import sys
|
|
|
|
PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", ".."))
|
|
if PROJECT_ROOT not in sys.path:
|
|
sys.path.insert(0, PROJECT_ROOT)
|
|
|
|
from services.workers.consumer import WorkerConsumer
|
|
|
|
logging.basicConfig(
|
|
level=logging.INFO,
|
|
format="%(asctime)s [%(name)s] %(levelname)s %(message)s",
|
|
)
|
|
logger = logging.getLogger("nef-worker")
|
|
|
|
|
|
def get_process_fn():
|
|
mode = os.environ.get("WORKER_MODE", "real").lower()
|
|
|
|
if mode == "stub":
|
|
from services.workers.stubs import process_nef_core_stub
|
|
logger.info("Running in STUB mode")
|
|
return process_nef_core_stub
|
|
else:
|
|
from services.workers.nef.core import process_nef_core
|
|
logger.info("Running in REAL mode")
|
|
return process_nef_core
|
|
|
|
|
|
def main():
|
|
process_fn = get_process_fn()
|
|
|
|
consumer = WorkerConsumer(
|
|
stage="nef",
|
|
process_fn=process_fn,
|
|
queue_name="queue:nef",
|
|
group_name="nef-workers",
|
|
)
|
|
consumer.run()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|