Compare commits
107 Commits
v3.1.0-bet
...
v3.2.0-bet
Author | SHA1 | Date | |
---|---|---|---|
|
5e889956c7 | ||
|
5afc6ea271 | ||
|
cc368377c2 | ||
|
51764be7d4 | ||
|
fe598b4a30 | ||
|
80286e57d9 | ||
|
9f36a48d15 | ||
|
94286e8452 | ||
|
0ce2d6a998 | ||
|
9e15fd7d2e | ||
|
736348ebe8 | ||
|
ddb981cba2 | ||
|
724452fb40 | ||
|
a2ac6a6268 | ||
|
aecda25b25 | ||
|
0f22b0bb23 | ||
|
edf6d70d69 | ||
|
a2ecd2e221 | ||
|
2f022cae5d | ||
|
bb3669f5b2 | ||
|
87fd8caf2c | ||
|
e8087a7c01 | ||
|
4075a7df29 | ||
|
c081b6f764 | ||
|
ecaa6d0321 | ||
|
e7ae7833ad | ||
|
22f34f6f81 | ||
|
29efe0a10e | ||
|
965c64b468 | ||
|
ce57cda672 | ||
|
a59857079e | ||
|
9ae2a0c628 | ||
|
f2c514cd82 | ||
|
6755230c53 | ||
|
f2a0e60f20 | ||
|
6cb21e95c4 | ||
|
868add68a5 | ||
|
307148d951 | ||
|
9cb4235550 | ||
|
6014b3e961 | ||
|
32f0a891ba | ||
|
124ac46eeb | ||
|
d3e76c19bf | ||
|
dd1ce189ae | ||
|
82e43b06c6 | ||
|
aab4e6d9df | ||
|
1f097678ce | ||
|
96cb4a04ec | ||
|
b7c4b37e89 | ||
|
952cf27ecb | ||
|
765810e4e5 | ||
|
36d09a7679 | ||
|
c6d6648942 | ||
|
54ca7580b0 | ||
|
4e53d239bb | ||
|
6491894f0e | ||
|
93e05dd913 | ||
|
2efe69e50b | ||
|
761865a6a3 | ||
|
25fed8db61 | ||
|
de42e1ca47 | ||
|
138e20d36d | ||
|
433610286c | ||
|
236f94247a | ||
|
1a5c1de510 | ||
|
6fb55005d4 | ||
|
4779671323 | ||
|
e7876739e7 | ||
|
e699e30af3 | ||
|
7541a2b085 | ||
|
24aa605973 | ||
|
fd28445680 | ||
|
19b7448c3a | ||
|
53c39135af | ||
|
b9147038e6 | ||
|
9a0b7f5c55 | ||
|
2a5f9f03ae | ||
|
78f6e36fc9 | ||
|
8e7aadd183 | ||
|
794a2782ac | ||
|
383f26e777 | ||
|
2db88d3902 | ||
|
5e96633a1e | ||
|
6c80ad8b2a | ||
|
96bd39eb25 | ||
|
b4bf227603 | ||
|
90c0300d77 | ||
|
a5a22ca5c4 | ||
|
f1ef32b5d4 | ||
|
a4bc5ec015 | ||
|
2e56b7e59f | ||
|
5672d25bce | ||
|
edf20e6190 | ||
|
dc3da45c2d | ||
|
f3f792e053 | ||
|
ad689bd6d9 | ||
|
ffd5005c8b | ||
|
e9ae414a6e | ||
|
a3389b8b0d | ||
|
199dd9ffd4 | ||
|
233ea0da41 | ||
|
1134beb6a6 | ||
|
b26450d277 | ||
|
61255b5e19 | ||
|
e4bdfb8432 | ||
|
7f76d48314 | ||
|
4d101fbdeb |
1
.gitignore
vendored
@@ -30,6 +30,7 @@ apps/server/dist/*
|
||||
|
||||
#DEFAULT
|
||||
.env
|
||||
.steering
|
||||
data/
|
||||
|
||||
node_modules/
|
11
Dockerfile
@@ -82,7 +82,7 @@ RUN addgroup --system --gid ${PALMR_GID} nodejs
|
||||
RUN adduser --system --uid ${PALMR_UID} --ingroup nodejs palmr
|
||||
|
||||
# Create application directories
|
||||
RUN mkdir -p /app/palmr-app /app/web /home/palmr/.npm /home/palmr/.cache
|
||||
RUN mkdir -p /app/palmr-app /app/web /app/infra /home/palmr/.npm /home/palmr/.cache
|
||||
RUN chown -R palmr:nodejs /app /home/palmr
|
||||
|
||||
# === Copy Server Files to /app/palmr-app (separate from /app/server for bind mounts) ===
|
||||
@@ -117,10 +117,13 @@ WORKDIR /app
|
||||
# Create supervisor configuration
|
||||
RUN mkdir -p /etc/supervisor/conf.d
|
||||
|
||||
# Copy server start script
|
||||
# Copy server start script and configuration files
|
||||
COPY infra/server-start.sh /app/server-start.sh
|
||||
COPY infra/configs.json /app/infra/configs.json
|
||||
COPY infra/providers.json /app/infra/providers.json
|
||||
COPY infra/check-missing.js /app/infra/check-missing.js
|
||||
RUN chmod +x /app/server-start.sh
|
||||
RUN chown palmr:nodejs /app/server-start.sh
|
||||
RUN chown -R palmr:nodejs /app/server-start.sh /app/infra
|
||||
|
||||
# Copy supervisor configuration
|
||||
COPY infra/supervisord.conf /etc/supervisor/conf.d/supervisord.conf
|
||||
@@ -133,10 +136,12 @@ set -e
|
||||
echo "Starting Palmr Application..."
|
||||
echo "Storage Mode: \${ENABLE_S3:-false}"
|
||||
echo "Secure Site: \${SECURE_SITE:-false}"
|
||||
echo "Encryption: \${DISABLE_FILESYSTEM_ENCRYPTION:-true}"
|
||||
echo "Database: SQLite"
|
||||
|
||||
# Set global environment variables
|
||||
export DATABASE_URL="file:/app/server/prisma/palmr.db"
|
||||
export NEXT_PUBLIC_DEFAULT_LANGUAGE=\${DEFAULT_LANGUAGE:-en-US}
|
||||
|
||||
# Ensure /app/server directory exists for bind mounts
|
||||
mkdir -p /app/server/uploads /app/server/temp-uploads /app/server/prisma
|
||||
|
@@ -1,5 +0,0 @@
|
||||
|
||||
|
||||
> palmr-docs@3.1-beta lint /Users/daniel/clones/Palmr/apps/docs
|
||||
> eslint "src/**/*.+(ts|tsx)"
|
||||
|
@@ -1,6 +0,0 @@
|
||||
{
|
||||
"defaultOpen": false,
|
||||
"icon": "Key",
|
||||
"pages": ["index", "google", "discord", "github", "zitadel", "auth0", "authentik", "frontegg", "kinde-auth", "other"],
|
||||
"title": "OIDC Authentication"
|
||||
}
|
@@ -1,267 +0,0 @@
|
||||
---
|
||||
title: Quick Start (Docker)
|
||||
icon: "Rocket"
|
||||
---
|
||||
|
||||
Welcome to the fastest way to deploy <span className="font-bold">Palmr.</span> - your secure, self-hosted file sharing solution. This guide will have you up and running in minutes, whether you're new to self-hosting or an experienced developer.
|
||||
|
||||
Palmr. offers flexible deployment options to match your infrastructure needs. This guide focuses on Docker deployment with our recommended filesystem storage, perfect for most use cases.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
Ensure you have the following installed on your system:
|
||||
|
||||
- **Docker** - Container runtime ([installation guide](https://docs.docker.com/get-docker/))
|
||||
- **Docker Compose** - Multi-container orchestration ([installation guide](https://docs.docker.com/compose/install/))
|
||||
|
||||
> **Platform Support**: Palmr. is developed on macOS and extensively tested on Linux servers. While we haven't formally tested other platforms, Docker's cross-platform nature should ensure compatibility. Report any issues on our [GitHub repository](https://github.com/kyantech/Palmr/issues).
|
||||
|
||||
## Storage Options
|
||||
|
||||
Palmr. supports two storage approaches for persistent data:
|
||||
|
||||
### Named Volumes (Recommended)
|
||||
|
||||
**Best for**: Production environments, automated deployments
|
||||
|
||||
- ✅ **Managed by Docker**: No permission issues or manual path management
|
||||
- ✅ **Optimized Performance**: Docker-native storage optimization
|
||||
- ✅ **Cross-platform**: Consistent behavior across operating systems
|
||||
- ✅ **Simplified Backups**: Docker volume commands for backup/restore
|
||||
|
||||
### Bind Mounts
|
||||
|
||||
**Best for**: Development, direct file access requirements
|
||||
|
||||
- ✅ **Direct Access**: Files stored in local directory you specify
|
||||
- ✅ **Transparent Storage**: Direct filesystem access from host
|
||||
- ✅ **Custom Backup**: Use existing file system backup solutions
|
||||
- ⚠️ **Permission Considerations**: **Common Issue** - Requires UID/GID configuration (see troubleshooting below)
|
||||
|
||||
---
|
||||
|
||||
## Option 1: Named Volumes (Recommended)
|
||||
|
||||
Named volumes provide the best performance and are managed entirely by Docker.
|
||||
|
||||
### Configuration
|
||||
|
||||
Use the provided `docker-compose.yaml` for named volumes:
|
||||
|
||||
```yaml
|
||||
services:
|
||||
palmr:
|
||||
image: kyantech/palmr:latest
|
||||
container_name: palmr
|
||||
environment:
|
||||
- ENABLE_S3=false
|
||||
- ENCRYPTION_KEY=change-this-key-in-production-min-32-chars # CHANGE THIS KEY FOR SECURITY
|
||||
# - SECURE_SITE=false # Set to true if you are using a reverse proxy
|
||||
ports:
|
||||
- "5487:5487" # Web interface
|
||||
- "3333:3333" # API port (OPTIONAL EXPOSED - ONLY IF YOU WANT TO ACCESS THE API DIRECTLY)
|
||||
volumes:
|
||||
- palmr_data:/app/server # Named volume for the application data
|
||||
restart: unless-stopped # Restart the container unless it is stopped
|
||||
|
||||
volumes:
|
||||
palmr_data:
|
||||
```
|
||||
|
||||
> **Note:** If you haveing problem with uploading files, try to change the `PALMR_UID` and `PALMR_GID` to the UID and GID of the user running the container. You can find the UID and GID of the user running the container with the command `id -u` and `id -g`. in Linux systems the default user is `1000` and the default group is `1000`. For test you can add the environment variables below to the `docker-compose.yaml` file and restart the container.
|
||||
|
||||
```yaml
|
||||
environment:
|
||||
- PALMR_UID=1000 # UID for the container processes (default is 1001)
|
||||
- PALMR_GID=1000 # GID for the container processes (default is 1001)
|
||||
```
|
||||
|
||||
> **Note:** For more information about UID and GID, see our [UID/GID Configuration](/docs/3.1-beta/uid-gid-configuration) guide.
|
||||
|
||||
### Deployment
|
||||
|
||||
```bash
|
||||
docker-compose up -d
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Option 2: Bind Mounts
|
||||
|
||||
Bind mounts store data in a local directory, providing direct file system access.
|
||||
|
||||
### Configuration
|
||||
|
||||
To use bind mounts, **replace the content** of your `docker-compose.yaml` with the following configuration (you can also reference `docker-compose-bind-mount-example.yaml` as a template):
|
||||
|
||||
```yaml
|
||||
services:
|
||||
palmr:
|
||||
image: kyantech/palmr:latest
|
||||
container_name: palmr
|
||||
environment:
|
||||
- ENABLE_S3=false
|
||||
- ENCRYPTION_KEY=change-this-key-in-production-min-32-chars # CHANGE THIS KEY FOR SECURITY
|
||||
# - SECURE_SITE=false # Set to true if you are using a reverse proxy
|
||||
- PALMR_UID=1000 # UID for the container processes (default is 1001)
|
||||
- PALMR_GID=1000 # GID for the container processes (default is 1001)
|
||||
ports:
|
||||
- "5487:5487" # Web port
|
||||
- "3333:3333" # API port (OPTIONAL EXPOSED - ONLY IF YOU WANT TO ACCESS THE API DIRECTLY)
|
||||
volumes:
|
||||
# Bind mount for persistent data (uploads, database, temp files)
|
||||
- ./data:/app/server # Local directory for the application data
|
||||
restart: unless-stopped # Restart the container unless it is stopped
|
||||
```
|
||||
|
||||
### Deployment
|
||||
|
||||
```bash
|
||||
docker-compose up -d
|
||||
```
|
||||
|
||||
> **Permission Configuration**: If you encounter permission issues with bind mounts (common on NAS systems), see our [UID/GID Configuration](/docs/3.1-beta/uid-gid-configuration) guide for automatic permission handling.
|
||||
|
||||
---
|
||||
|
||||
## Environment Variables
|
||||
|
||||
Configure Palmr. behavior through environment variables:
|
||||
|
||||
| Variable | Default | Description |
|
||||
| ---------------- | ------- | ------------------------------------------------------- |
|
||||
| `ENABLE_S3` | `false` | Enable S3-compatible storage |
|
||||
| `ENCRYPTION_KEY` | - | **Required**: Minimum 32 characters for file encryption |
|
||||
| `SECURE_SITE` | `false` | Enable secure cookies for HTTPS/reverse proxy setups |
|
||||
|
||||
> **⚠️ Security Warning**: Always change the `ENCRYPTION_KEY` in production. This key encrypts your files - losing it makes files permanently inaccessible.
|
||||
|
||||
> **🔗 Reverse Proxy**: If deploying behind a reverse proxy (Traefik, Nginx, etc.), set `SECURE_SITE=true` and review our [Reverse Proxy Configuration](/docs/3.1-beta/reverse-proxy-configuration) guide for proper setup.
|
||||
|
||||
### Generate Secure Encryption Keys
|
||||
|
||||
Need a strong key for `ENCRYPTION_KEY`? Use our built-in generator to create cryptographically secure keys:
|
||||
|
||||
<KeyGenerator />
|
||||
|
||||
---
|
||||
|
||||
## Accessing Palmr.
|
||||
|
||||
Once deployed, access Palmr. through your web browser:
|
||||
|
||||
- **Local**: `http://localhost:5487`
|
||||
- **Server**: `http://YOUR_SERVER_IP:5487`
|
||||
|
||||
### API Access (Optional)
|
||||
|
||||
If you exposed port 3333 in your configuration, you can also access:
|
||||
|
||||
- **API Documentation**: `http://localhost:3333/docs` (local) or `http://YOUR_SERVER_IP:3333/docs` (server)
|
||||
- **API Endpoints**: Available at `http://localhost:3333` (local) or `http://YOUR_SERVER_IP:3333` (server)
|
||||
|
||||
> **📚 Learn More**: For complete API documentation, authentication, and integration examples, see our [API Reference](/docs/3.1-beta/api) guide.
|
||||
|
||||
> **💡 Production Tip**: For production deployments, configure HTTPS with a valid SSL certificate for enhanced security.
|
||||
|
||||
---
|
||||
|
||||
## Docker CLI Alternative
|
||||
|
||||
Prefer using Docker directly? Both storage options are supported:
|
||||
|
||||
**Named Volume:**
|
||||
|
||||
```bash
|
||||
docker run -d \
|
||||
--name palmr \
|
||||
-e ENABLE_S3=false \
|
||||
-e ENCRYPTION_KEY=your-secure-key-min-32-chars \
|
||||
-p 5487:5487 \
|
||||
-p 3333:3333 \
|
||||
-v palmr_data:/app/server \
|
||||
--restart unless-stopped \
|
||||
kyantech/palmr:latest
|
||||
```
|
||||
|
||||
**Bind Mount:**
|
||||
|
||||
```bash
|
||||
docker run -d \
|
||||
--name palmr \
|
||||
-e ENABLE_S3=false \
|
||||
-e ENCRYPTION_KEY=your-secure-key-min-32-chars \
|
||||
-p 5487:5487 \
|
||||
-p 3333:3333 \
|
||||
-v $(pwd)/data:/app/server \
|
||||
--restart unless-stopped \
|
||||
kyantech/palmr:latest
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Maintenance
|
||||
|
||||
### Updates
|
||||
|
||||
Keep Palmr. current with the latest features and security fixes:
|
||||
|
||||
```bash
|
||||
docker-compose pull
|
||||
docker-compose up -d
|
||||
```
|
||||
|
||||
### Backup & Restore
|
||||
|
||||
The backup method depends on which storage option you're using:
|
||||
|
||||
**Named Volume Backup:**
|
||||
|
||||
```bash
|
||||
docker run --rm \
|
||||
-v palmr_data:/data \
|
||||
-v $(pwd):/backup \
|
||||
alpine tar czf /backup/palmr-backup.tar.gz -C /data .
|
||||
```
|
||||
|
||||
**Named Volume Restore:**
|
||||
|
||||
```bash
|
||||
docker run --rm \
|
||||
-v palmr_data:/data \
|
||||
-v $(pwd):/backup \
|
||||
alpine tar xzf /backup/palmr-backup.tar.gz -C /data
|
||||
```
|
||||
|
||||
**Bind Mount Backup:**
|
||||
|
||||
```bash
|
||||
tar czf palmr-backup.tar.gz ./data
|
||||
```
|
||||
|
||||
**Bind Mount Restore:**
|
||||
|
||||
```bash
|
||||
tar xzf palmr-backup.tar.gz
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Next Steps
|
||||
|
||||
Your Palmr. instance is now ready! Explore additional configuration options:
|
||||
|
||||
### Advanced Configuration
|
||||
|
||||
- **[UID/GID Configuration](/docs/3.1-beta/uid-gid-configuration)** - Configure user permissions for NAS systems and custom environments
|
||||
- **[S3 Storage](/docs/3.1-beta/s3-configuration)** - Scale with Amazon S3 or compatible storage providers
|
||||
- **[Manual Installation](/docs/3.1-beta/manual-installation)** - Manual installation and custom configurations
|
||||
|
||||
### Integration & Development
|
||||
|
||||
- **[API Reference](/docs/3.1-beta/api)** - Integrate Palmr. with your applications
|
||||
- **[Architecture Guide](/docs/3.1-beta/architecture)** - Understanding Palmr. components and design
|
||||
|
||||
---
|
||||
|
||||
Need help? Visit our [GitHub Issues](https://github.com/kyantech/Palmr/issues) or community discussions.
|
@@ -30,8 +30,6 @@ services:
|
||||
```bash
|
||||
docker run -d \
|
||||
--name palmr \
|
||||
-e ENABLE_S3=false \
|
||||
-e ENCRYPTION_KEY=change-this-key-in-production-min-32-chars \
|
||||
-p 5487:5487 \
|
||||
-p 3333:3333 \
|
||||
-v palmr_data:/app/server \
|
@@ -43,6 +43,16 @@ Palmr. uses **filesystem storage** as the default storage solution, keeping thin
|
||||
- Excellent performance for local file operations
|
||||
- Optional S3-compatible storage support for cloud deployments and scalability
|
||||
|
||||
#### Performance Considerations with Encryption
|
||||
|
||||
By default, filesystem storage operates without encryption for optimal performance, providing fast uploads and downloads with minimal CPU overhead. This approach is ideal for most use cases where performance is prioritized.
|
||||
|
||||
If you need to protect sensitive files at rest, you can enable encryption by setting `DISABLE_FILESYSTEM_ENCRYPTION=false` and providing an `ENCRYPTION_KEY` in your configuration. When enabled, Palmr uses AES-256-CBC encryption, which adds CPU overhead during uploads (encryption) and downloads (decryption), particularly for large files or in resource-constrained environments like containers or low-end VMs.
|
||||
|
||||
For optimal performance with encryption enabled, ensure your hardware supports AES-NI acceleration (check with `cat /proc/cpuinfo | grep aes` on Linux).
|
||||
|
||||
As an alternative, consider using S3-compatible object storage (e.g., AWS S3 or MinIO), which can offload file storage from the local filesystem and potentially reduce local CPU overhead for encryption/decryption. See [S3 Providers](/docs/3.2-beta/s3-providers) for setup instructions.
|
||||
|
||||
### Fastify + Zod + TypeScript
|
||||
|
||||
The backend of Palmr. is powered by **Fastify**, **Zod**, and **TypeScript**, creating a robust and type-safe API layer. Fastify is a super-fast Node.js web framework optimized for performance and low overhead, designed to handle lots of concurrent requests with minimal resource usage. Zod provides runtime type validation and schema definition, ensuring all incoming data is properly validated before reaching business logic. TypeScript adds compile-time type safety throughout the entire backend codebase. This combination creates a highly reliable and maintainable backend that prevents bugs and security issues while maintaining excellent performance.
|
||||
@@ -117,7 +127,7 @@ Palmr. is designed to be flexible in how you handle file storage:
|
||||
|
||||
**Optional S3-compatible storage:**
|
||||
|
||||
- Enable S3 storage by setting `ENABLE_S3=true`, look at [S3 Providers](/docs/3.1-beta/s3-providers) for more information.
|
||||
- Enable S3 storage by setting `ENABLE_S3=true`, look at [S3 Providers](/docs/3.2-beta/s3-providers) for more information.
|
||||
- Compatible with AWS S3, MinIO, and other S3-compatible services
|
||||
- Ideal for cloud deployments and distributed setups
|
||||
- Provides additional scalability and redundancy options
|
391
apps/docs/content/docs/3.2-beta/download-memory-management.mdx
Normal file
@@ -0,0 +1,391 @@
|
||||
---
|
||||
title: Memory Management
|
||||
icon: Download
|
||||
---
|
||||
|
||||
import { Callout } from "fumadocs-ui/components/callout";
|
||||
import { Tab, Tabs } from "fumadocs-ui/components/tabs";
|
||||
|
||||
Palmr implements an intelligent memory management system that prevents crashes during large file downloads (3GB+ by default), maintaining unlimited download capacity through adaptive resource control and an automatic queue system.
|
||||
|
||||
## How It Works
|
||||
|
||||
### Automatic Resource Detection
|
||||
|
||||
The system automatically detects available container/system memory and configures appropriate limits based on available infrastructure:
|
||||
|
||||
```typescript
|
||||
const totalMemoryGB = require("os").totalmem() / 1024 ** 3;
|
||||
```
|
||||
|
||||
### System Configuration
|
||||
|
||||
The system supports two configuration approaches that you can choose based on your needs:
|
||||
|
||||
<Tabs items={["Manual Configuration", "Auto-scaling (Default)"]}>
|
||||
<Tab value="Manual Configuration">
|
||||
Manually configure all parameters for total control over the system:
|
||||
|
||||
```bash
|
||||
# Custom configuration (overrides auto-scaling)
|
||||
DOWNLOAD_MAX_CONCURRENT=8 # Maximum simultaneous downloads
|
||||
DOWNLOAD_MEMORY_THRESHOLD_MB=1536 # Memory threshold in MB
|
||||
DOWNLOAD_QUEUE_SIZE=40 # Maximum queue size
|
||||
DOWNLOAD_AUTO_SCALE=false # Disable auto-scaling
|
||||
```
|
||||
|
||||
<Callout>
|
||||
Manual configuration offers total control and predictability for specific environments where you know exactly the available resources.
|
||||
</Callout>
|
||||
|
||||
</Tab>
|
||||
<Tab value="Auto-scaling (Default)">
|
||||
Automatic configuration based on detected system memory:
|
||||
|
||||
| Available Memory | Concurrent Downloads | Memory Threshold | Queue Size | Recommended Use |
|
||||
|------------------|----------------------|-------------------|------------|--------------------|
|
||||
| ≤ 2GB | 1 | 256MB | 5 | Development |
|
||||
| 2GB - 4GB | 2 | 512MB | 10 | Small Environment |
|
||||
| 4GB - 8GB | 3 | 1GB | 15 | Standard Production|
|
||||
| 8GB - 16GB | 5 | 2GB | 25 | High Performance |
|
||||
| > 16GB | 10 | 4GB | 50 | Enterprise |
|
||||
|
||||
<Callout>
|
||||
Auto-scaling automatically adapts to different environments without manual configuration, perfect for flexible deployment.
|
||||
</Callout>
|
||||
|
||||
</Tab>
|
||||
</Tabs>
|
||||
|
||||
<Callout type="info">If environment variables are configured, they take **priority** over auto-scaling.</Callout>
|
||||
|
||||
## Download Queue System
|
||||
|
||||
### How It Works
|
||||
|
||||
The memory management system only activates for files larger than the configured minimum size (3GB by default). Smaller files bypass the queue system entirely and download immediately without memory management.
|
||||
|
||||
When a user requests a download for a large file but all slots are occupied, the system automatically queues the download instead of returning a 429 error. The queue processes downloads in FIFO order (first in, first out).
|
||||
|
||||
### Practical Example
|
||||
|
||||
Consider a system with 8GB RAM (5 concurrent downloads, queue of 25, 3GB minimum) where users want to download files of various sizes:
|
||||
|
||||
```bash
|
||||
# Small files (< 3GB): Bypass queue entirely
|
||||
[DOWNLOAD MANAGER] File document.pdf (0.05GB) below threshold (3.0GB), bypassing queue
|
||||
|
||||
# Large files 1-5: Start immediately
|
||||
[DOWNLOAD MANAGER] Immediate start: 1734567890-abc123def
|
||||
[DOWNLOAD MANAGER] Starting video1.mp4 (5.2GB)
|
||||
|
||||
# Large files 6-10: Automatically queued
|
||||
[DOWNLOAD MANAGER] Queued: 1734567891-def456ghi (Position: 1/25)
|
||||
[DOWNLOAD MANAGER] Queued file: video2.mp4 (8.1GB)
|
||||
|
||||
# When download 1 finishes: download 6 starts automatically
|
||||
[DOWNLOAD MANAGER] Processing queue: 1734567891-def456ghi (4 remaining)
|
||||
[DOWNLOAD MANAGER] Starting queued file: video2.mp4 (8.1GB)
|
||||
```
|
||||
|
||||
### System Benefits
|
||||
|
||||
**User Experience**
|
||||
|
||||
- Users don't receive errors, they simply wait in queue
|
||||
- Downloads start automatically when slots become available
|
||||
- Transparent operation without client changes
|
||||
- Fair processing order with FIFO queue
|
||||
|
||||
**Technical Features**
|
||||
|
||||
- Limited buffers (64KB per stream) for controlled memory usage
|
||||
- Automatic backpressure control with pipeline streams
|
||||
- Adaptive memory throttling based on usage patterns
|
||||
- Forced garbage collection after large downloads
|
||||
- Smart timeout handling (30 minutes for queued downloads)
|
||||
- Automatic cleanup of orphaned downloads every 30 seconds
|
||||
|
||||
## Container Compatibility
|
||||
|
||||
The system works with Docker, Kubernetes, and any containerized environment:
|
||||
|
||||
<Tabs items={["Docker", "Kubernetes", "Docker Compose"]}>
|
||||
<Tab value="Docker">
|
||||
|
||||
```bash
|
||||
# Example: Container with 8GB
|
||||
docker run -m 8g palmr/server
|
||||
# Result: 5 concurrent downloads, queue of 25, threshold 2GB
|
||||
```
|
||||
|
||||
</Tab>
|
||||
<Tab value="Kubernetes">
|
||||
|
||||
```yaml
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: palmr-server
|
||||
resources:
|
||||
limits:
|
||||
memory: "4Gi" # Detects 4GB
|
||||
cpu: "2"
|
||||
requests:
|
||||
memory: "2Gi"
|
||||
cpu: "1"
|
||||
# Result: 3 concurrent downloads, queue of 15, threshold 1GB
|
||||
```
|
||||
|
||||
</Tab>
|
||||
<Tab value="Docker Compose">
|
||||
|
||||
```yaml
|
||||
services:
|
||||
palmr-server:
|
||||
image: palmr/server
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 16G # Detects 16GB
|
||||
# Result: 10 concurrent downloads, queue of 50, threshold 4GB
|
||||
```
|
||||
|
||||
</Tab>
|
||||
</Tabs>
|
||||
|
||||
## Configuration
|
||||
|
||||
### Environment Variables
|
||||
|
||||
Configure the download memory management system using these environment variables:
|
||||
|
||||
| Variable | Default | Description |
|
||||
| ------------------------------ | ---------- | ----------------------------------------------------- |
|
||||
| `DOWNLOAD_MAX_CONCURRENT` | auto-scale | Maximum number of simultaneous downloads |
|
||||
| `DOWNLOAD_MEMORY_THRESHOLD_MB` | auto-scale | Memory limit in MB before throttling |
|
||||
| `DOWNLOAD_QUEUE_SIZE` | auto-scale | Maximum download queue size |
|
||||
| `DOWNLOAD_AUTO_SCALE` | `true` | Enable/disable auto-scaling based on system memory |
|
||||
| `DOWNLOAD_MIN_FILE_SIZE_GB` | `3.0` | Minimum file size in GB to activate memory management |
|
||||
|
||||
### Configuration Examples by Scenario
|
||||
|
||||
<Tabs items={["Home Server", "Enterprise", "High Performance", "Conservative"]}>
|
||||
<Tab value="Home Server">
|
||||
Configuration optimized for personal use or small groups (4GB RAM):
|
||||
|
||||
```bash
|
||||
DOWNLOAD_MAX_CONCURRENT=2
|
||||
DOWNLOAD_MEMORY_THRESHOLD_MB=1024
|
||||
DOWNLOAD_QUEUE_SIZE=8
|
||||
DOWNLOAD_MIN_FILE_SIZE_GB=2.0
|
||||
DOWNLOAD_AUTO_SCALE=false
|
||||
```
|
||||
|
||||
</Tab>
|
||||
<Tab value="Enterprise">
|
||||
Configuration for corporate environments with multiple users (16GB RAM):
|
||||
|
||||
```bash
|
||||
DOWNLOAD_MAX_CONCURRENT=12
|
||||
DOWNLOAD_MEMORY_THRESHOLD_MB=4096
|
||||
DOWNLOAD_QUEUE_SIZE=60
|
||||
DOWNLOAD_MIN_FILE_SIZE_GB=5.0
|
||||
DOWNLOAD_AUTO_SCALE=false
|
||||
```
|
||||
|
||||
</Tab>
|
||||
<Tab value="High Performance">
|
||||
Configuration for maximum performance and throughput (32GB RAM):
|
||||
|
||||
```bash
|
||||
DOWNLOAD_MAX_CONCURRENT=20
|
||||
DOWNLOAD_MEMORY_THRESHOLD_MB=8192
|
||||
DOWNLOAD_QUEUE_SIZE=100
|
||||
DOWNLOAD_MIN_FILE_SIZE_GB=10.0
|
||||
DOWNLOAD_AUTO_SCALE=false
|
||||
```
|
||||
|
||||
</Tab>
|
||||
<Tab value="Conservative">
|
||||
For environments with limited or shared resources:
|
||||
|
||||
```bash
|
||||
DOWNLOAD_MAX_CONCURRENT=3
|
||||
DOWNLOAD_MEMORY_THRESHOLD_MB=1024
|
||||
DOWNLOAD_QUEUE_SIZE=15
|
||||
DOWNLOAD_MIN_FILE_SIZE_GB=1.0
|
||||
DOWNLOAD_AUTO_SCALE=false
|
||||
```
|
||||
|
||||
</Tab>
|
||||
</Tabs>
|
||||
|
||||
### Additional Configuration
|
||||
|
||||
For optimal performance with large downloads, consider these additional settings:
|
||||
|
||||
```bash
|
||||
# Force garbage collection (recommended for large downloads)
|
||||
NODE_OPTIONS="--expose-gc"
|
||||
|
||||
# Adjust timeout for very large downloads
|
||||
KEEP_ALIVE_TIMEOUT=300000
|
||||
REQUEST_TIMEOUT=0
|
||||
```
|
||||
|
||||
## Monitoring and Logs
|
||||
|
||||
### System Logs
|
||||
|
||||
The system provides detailed logs to track operation:
|
||||
|
||||
```bash
|
||||
[DOWNLOAD MANAGER] System Memory: 8.0GB, Max Concurrent: 5, Memory Threshold: 2048MB, Queue Size: 25
|
||||
[DOWNLOAD] Requesting slot for 1734567890-abc123def: video.mp4 (15.2GB)
|
||||
[DOWNLOAD MANAGER] Queued: 1734567890-abc123def (Position: 3/25)
|
||||
[DOWNLOAD MANAGER] Processing queue: 1734567890-abc123def (2 remaining)
|
||||
[DOWNLOAD] Starting 1734567890-abc123def: video.mp4 (15.2GB)
|
||||
[MEMORY THROTTLE] video.mp4 - Pausing stream due to high memory usage: 1843MB
|
||||
[DOWNLOAD] Applying throttling: 100ms delay for 1734567890-abc123def
|
||||
```
|
||||
|
||||
### Configuration Validation
|
||||
|
||||
The system automatically validates configurations at startup and provides warnings or errors:
|
||||
|
||||
**Warnings**
|
||||
|
||||
- `DOWNLOAD_MAX_CONCURRENT > 50`: May cause performance issues
|
||||
- `DOWNLOAD_MEMORY_THRESHOLD_MB < 128MB`: Downloads may be throttled frequently
|
||||
- `DOWNLOAD_MEMORY_THRESHOLD_MB > 16GB`: System may run out of memory
|
||||
- `DOWNLOAD_QUEUE_SIZE > 1000`: May consume significant memory
|
||||
- `DOWNLOAD_QUEUE_SIZE < DOWNLOAD_MAX_CONCURRENT`: Queue smaller than concurrent downloads
|
||||
|
||||
**Errors**
|
||||
|
||||
- `DOWNLOAD_MAX_CONCURRENT < 1`: Invalid value
|
||||
- `DOWNLOAD_QUEUE_SIZE < 1`: Invalid value
|
||||
|
||||
## Queue Management APIs
|
||||
|
||||
The system provides REST APIs to monitor and manage the download queue:
|
||||
|
||||
### Get Queue Status
|
||||
|
||||
```http
|
||||
GET /api/filesystem/download-queue/status
|
||||
```
|
||||
|
||||
**Response:**
|
||||
|
||||
```json
|
||||
{
|
||||
"data": {
|
||||
"queueLength": 3,
|
||||
"maxQueueSize": 25,
|
||||
"activeDownloads": 5,
|
||||
"maxConcurrent": 5,
|
||||
"queuedDownloads": [
|
||||
{
|
||||
"downloadId": "1734567890-abc123def",
|
||||
"position": 1,
|
||||
"waitTime": 45000,
|
||||
"fileName": "video.mp4",
|
||||
"fileSize": 16106127360
|
||||
}
|
||||
]
|
||||
},
|
||||
"status": "success"
|
||||
}
|
||||
```
|
||||
|
||||
### Cancel Download
|
||||
|
||||
```http
|
||||
DELETE /api/filesystem/download-queue/{downloadId}
|
||||
```
|
||||
|
||||
**Response:**
|
||||
|
||||
```json
|
||||
{
|
||||
"downloadId": "1734567890-abc123def",
|
||||
"message": "Download cancelled successfully"
|
||||
}
|
||||
```
|
||||
|
||||
### Clear Queue (Admin)
|
||||
|
||||
```http
|
||||
DELETE /api/filesystem/download-queue
|
||||
```
|
||||
|
||||
**Response:**
|
||||
|
||||
```json
|
||||
{
|
||||
"clearedCount": 8,
|
||||
"message": "Download queue cleared successfully"
|
||||
}
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Common Issues
|
||||
|
||||
**Downloads failing with "Download queue is full"**
|
||||
|
||||
_Cause:_ Too many simultaneous downloads with a full queue
|
||||
|
||||
_Solutions:_
|
||||
|
||||
- Wait for some downloads to finish
|
||||
- Check for orphaned downloads in queue
|
||||
- Consider increasing container resources
|
||||
- Use API to clear queue if necessary
|
||||
|
||||
**Downloads stay too long in queue**
|
||||
|
||||
_Cause:_ Active downloads are slow or stuck
|
||||
|
||||
_Solutions:_
|
||||
|
||||
- Check logs for orphaned downloads
|
||||
- Use API to cancel specific downloads
|
||||
- Check client network connections
|
||||
- Monitor memory throttling
|
||||
|
||||
**Very slow downloads**
|
||||
|
||||
_Cause:_ Active throttling due to high memory usage
|
||||
|
||||
_Solutions:_
|
||||
|
||||
- Check other processes consuming memory
|
||||
- Consider increasing container resources
|
||||
- Monitor throttling logs
|
||||
- Check number of simultaneous downloads
|
||||
|
||||
## Summary
|
||||
|
||||
This system enables unlimited downloads (including 50TB+ files) without compromising system stability through:
|
||||
|
||||
**Key Features**
|
||||
|
||||
- Auto-configuration based on available resources
|
||||
- Automatic FIFO queue system for pending downloads
|
||||
- Adaptive control of simultaneous downloads
|
||||
- Intelligent throttling when needed
|
||||
|
||||
**System Benefits**
|
||||
|
||||
- Management APIs to monitor and control queue
|
||||
- Automatic cleanup of resources and orphaned downloads
|
||||
- Full compatibility with Docker/Kubernetes
|
||||
- Perfect user experience with no 429 errors
|
||||
|
||||
The system maintains high performance for small/medium files while preventing crashes with gigantic files, offering a seamless experience where users never see 429 errors, they simply wait in queue until their download starts automatically.
|
@@ -5,7 +5,7 @@ icon: Cog
|
||||
|
||||
Hey there! Looking to run **Palmr.** your way, with complete control over every piece of the stack? This manual installation guide is for you. No Docker, no pre-built containers just the raw source code to tweak, customize, and deploy as you see fit.
|
||||
|
||||
> **Prefer a quicker setup?** If this hands-on approach feels like overkill, check out our [**Quick Start (Docker)**](/docs/3.1-beta/quick-start) guide for a fast, containerized deployment. This manual path is tailored for developers who want to dive deep, modify the codebase, or integrate custom services.
|
||||
> **Prefer a quicker setup?** If this hands-on approach feels like overkill, check out our [**Quick Start (Docker)**](/docs/3.2-beta/quick-start) guide for a fast, containerized deployment. This manual path is tailored for developers who want to dive deep, modify the codebase, or integrate custom services.
|
||||
|
||||
Here's what you'll do at a glance:
|
||||
|
||||
@@ -232,10 +232,10 @@ pnpm serve
|
||||
|
||||
Palmr. is now up and running locally . Here are some suggested next steps:
|
||||
|
||||
- **Manage Users**: Dive into the [Users Management](/docs/3.1-beta/manage-users) guide.
|
||||
- **Manage Users**: Dive into the [Users Management](/docs/3.2-beta/manage-users) guide.
|
||||
- **Switch to Object Storage**: Update `.env` variables to use an S3-compatible bucket (see Quick Notes above).
|
||||
- **Secure Your Instance**: Put Palmr. behind a reverse proxy like **Nginx** or **Caddy** and enable HTTPS.
|
||||
- **Learn the Internals**: Explore how everything connects in the [Architecture](/docs/3.1-beta/architecture) overview.
|
||||
- **Learn the Internals**: Explore how everything connects in the [Architecture](/docs/3.2-beta/architecture) overview.
|
||||
|
||||
Jump into whichever area fits your needs our docs are designed for exploration in any order.
|
||||
|
@@ -14,6 +14,7 @@
|
||||
"available-languages",
|
||||
"uid-gid-configuration",
|
||||
"reverse-proxy-configuration",
|
||||
"download-memory-management",
|
||||
"password-reset-without-smtp",
|
||||
"oidc-authentication",
|
||||
"troubleshooting",
|
||||
@@ -29,5 +30,5 @@
|
||||
"gh-sponsor"
|
||||
],
|
||||
"root": true,
|
||||
"title": "v3.1-beta"
|
||||
}
|
||||
"title": "v3.2-beta"
|
||||
}
|
@@ -360,7 +360,7 @@ With Auth0 authentication configured, you might want to:
|
||||
- **Review security settings**: Ensure your authentication setup meets your security requirements
|
||||
- **Monitor usage**: Keep track of authentication patterns and user activity
|
||||
|
||||
For more information about OIDC authentication in Palmr, see the [OIDC Authentication overview](/docs/3.1-beta/oidc-authentication).
|
||||
For more information about OIDC authentication in Palmr, see the [OIDC Authentication overview](/docs/3.2-beta/oidc-authentication).
|
||||
|
||||
## Useful resources
|
||||
|
@@ -374,7 +374,7 @@ With Authentik authentication configured, you might want to:
|
||||
- **Review security settings**: Ensure your authentication setup meets your security requirements
|
||||
- **Monitor usage**: Keep track of authentication patterns and user activity
|
||||
|
||||
For more information about OIDC authentication in Palmr, see the [OIDC Authentication overview](/docs/3.1-beta/oidc-authentication).
|
||||
For more information about OIDC authentication in Palmr, see the [OIDC Authentication overview](/docs/3.2-beta/oidc-authentication).
|
||||
|
||||
## Useful resources
|
||||
|
@@ -332,7 +332,7 @@ With Discord authentication configured, you might want to:
|
||||
- **Review security settings**: Ensure your authentication setup meets your security requirements
|
||||
- **Monitor usage**: Keep track of authentication patterns and user activity
|
||||
|
||||
For more information about OIDC authentication in Palmr, see the [OIDC Authentication overview](/docs/3.1-beta/oidc-authentication).
|
||||
For more information about OIDC authentication in Palmr, see the [OIDC Authentication overview](/docs/3.2-beta/oidc-authentication).
|
||||
|
||||
## Useful resources
|
||||
|
@@ -314,7 +314,7 @@ With Frontegg authentication configured, you might want to:
|
||||
- **Review security settings**: Ensure your authentication setup meets your security requirements
|
||||
- **Monitor usage**: Keep track of authentication patterns and user activity
|
||||
|
||||
For more information about OIDC authentication in Palmr, see the [OIDC Authentication overview](/docs/3.1-beta/oidc-authentication).
|
||||
For more information about OIDC authentication in Palmr, see the [OIDC Authentication overview](/docs/3.2-beta/oidc-authentication).
|
||||
|
||||
## Useful resources
|
||||
|
@@ -295,7 +295,7 @@ With GitHub authentication configured, you might want to:
|
||||
- **Review security settings**: Ensure your authentication setup meets your security requirements
|
||||
- **Monitor usage**: Keep track of authentication patterns and user activity
|
||||
|
||||
For more information about OIDC authentication in Palmr, see the [OIDC Authentication overview](/docs/3.1-beta/oidc-authentication).
|
||||
For more information about OIDC authentication in Palmr, see the [OIDC Authentication overview](/docs/3.2-beta/oidc-authentication).
|
||||
|
||||
## Useful resources
|
||||
|
@@ -325,7 +325,7 @@ With Google authentication configured, you might want to:
|
||||
- **Review security settings**: Ensure your authentication setup meets your security requirements
|
||||
- **Monitor usage**: Keep track of authentication patterns and user activity
|
||||
|
||||
For more information about OIDC authentication in Palmr, see the [OIDC Authentication overview](/docs/3.1-beta/oidc-authentication).
|
||||
For more information about OIDC authentication in Palmr, see the [OIDC Authentication overview](/docs/3.2-beta/oidc-authentication).
|
||||
|
||||
## Useful resources
|
||||
|
@@ -38,14 +38,14 @@ Before configuring OIDC authentication, ensure you have:
|
||||
|
||||
Palmr's OIDC implementation is compatible with any OpenID Connect compliant provider, including as official providers:
|
||||
|
||||
- **[Google](/docs/3.1-beta/oidc-authentication/google)**
|
||||
- **[Discord](/docs/3.1-beta/oidc-authentication/discord)**
|
||||
- **[Github](/docs/3.1-beta/oidc-authentication/github)**
|
||||
- **[Zitadel](/docs/3.1-beta/oidc-authentication/zitadel)**
|
||||
- **[Auth0](/docs/3.1-beta/oidc-authentication/auth0)**
|
||||
- **[Authentik](/docs/3.1-beta/oidc-authentication/authentik)**
|
||||
- **[Frontegg](/docs/3.1-beta/oidc-authentication/frontegg)**
|
||||
- **[Kinde Auth](/docs/3.1-beta/oidc-authentication/kinde-auth)**
|
||||
- **[Google](/docs/3.2-beta/oidc-authentication/google)**
|
||||
- **[Discord](/docs/3.2-beta/oidc-authentication/discord)**
|
||||
- **[Github](/docs/3.2-beta/oidc-authentication/github)**
|
||||
- **[Zitadel](/docs/3.2-beta/oidc-authentication/zitadel)**
|
||||
- **[Auth0](/docs/3.2-beta/oidc-authentication/auth0)**
|
||||
- **[Authentik](/docs/3.2-beta/oidc-authentication/authentik)**
|
||||
- **[Frontegg](/docs/3.2-beta/oidc-authentication/frontegg)**
|
||||
- **[Kinde Auth](/docs/3.2-beta/oidc-authentication/kinde-auth)**
|
||||
|
||||
Although these are the official providers (internally tested with 100% success), you can connect any OIDC provider by providing your credentials and connection URL. We've developed a practical way to integrate virtually all OIDC providers available in the market. In this documentation, you can consult how to configure each of the official providers, as well as include other providers not listed as official. Just below, you will find instructions on how to access the OIDC provider configuration. For specific details about configuring each provider, select the desired option in the sidebar, in the "OIDC Authentication" section.
|
||||
|
@@ -359,7 +359,7 @@ With Kinde Auth authentication configured, you might want to:
|
||||
- **Review security settings**: Ensure your authentication setup meets your security requirements
|
||||
- **Monitor usage**: Keep track of authentication patterns and user activity
|
||||
|
||||
For more information about OIDC authentication in Palmr, see the [OIDC Authentication overview](/docs/3.1-beta/oidc-authentication).
|
||||
For more information about OIDC authentication in Palmr, see the [OIDC Authentication overview](/docs/3.2-beta/oidc-authentication).
|
||||
|
||||
## Useful resources
|
||||
|
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"defaultOpen": false,
|
||||
"icon": "Key",
|
||||
"pages": [
|
||||
"index",
|
||||
"google",
|
||||
"discord",
|
||||
"github",
|
||||
"zitadel",
|
||||
"auth0",
|
||||
"authentik",
|
||||
"frontegg",
|
||||
"kinde-auth",
|
||||
"pocket-id",
|
||||
"other"
|
||||
],
|
||||
"title": "OIDC Authentication"
|
||||
}
|
@@ -0,0 +1,279 @@
|
||||
---
|
||||
title: Pocket ID
|
||||
icon: IdCardLanyard
|
||||
---
|
||||
|
||||
import { ZoomableImage } from "@/components/ui/zoomable-image";
|
||||
|
||||
Pocket ID is one of Palmr's officially supported OIDC providers, offering a robust and flexible identity management solution. This integration allows users to sign in to Palmr using Pocket ID's authentication system, making it perfect for organizations that need a self-hosted identity provider with OIDC support.
|
||||
|
||||
<ZoomableImage src="/assets/v3/oidc/pocket-id/sign-in-with-pocket-id.png" alt="Sign in with Pocket ID" />
|
||||
|
||||
## Why use Pocket ID authentication?
|
||||
|
||||
Pocket ID authentication provides several advantages for organizations seeking a self-hosted identity solution:
|
||||
|
||||
- **Self-hosted control** - Full control over your authentication infrastructure and data
|
||||
- **OIDC compliance** - Standard OpenID Connect implementation for seamless integration
|
||||
- **Flexible deployment** - Deploy on any infrastructure that suits your needs
|
||||
- **Automatic discovery** - Supports OIDC discovery for streamlined configuration
|
||||
- **Simple configuration** - Intuitive setup process with minimal complexity
|
||||
- **Data sovereignty** - Keep all authentication data within your infrastructure
|
||||
- **Cost-effective** - No per-user pricing, perfect for growing organizations
|
||||
|
||||
---
|
||||
|
||||
## Prerequisites
|
||||
|
||||
Before configuring Pocket ID authentication, ensure you have:
|
||||
|
||||
- **Pocket ID instance** - A running Pocket ID server accessible via HTTPS
|
||||
- **Admin privileges in Palmr** - Required to configure OIDC settings
|
||||
- **Domain configuration** - For production deployments with custom domains
|
||||
|
||||
> **Note:** Pocket ID is pre-configured as an official provider in Palmr, which means the technical configuration is handled automatically. You only need to provide your OAuth credentials.
|
||||
|
||||
---
|
||||
|
||||
## Setting up Pocket ID Application
|
||||
|
||||
### Creating a Pocket ID application
|
||||
|
||||
To get started with Pocket ID authentication, you'll need to create an application in your Pocket ID admin interface.
|
||||
|
||||
1. **Navigate to Pocket ID Admin**: Go to your Pocket ID instance URL (e.g., `https://your-pocket-id.domain.com`)
|
||||
|
||||
<ZoomableImage src="/assets/v3/oidc/pocket-id/pocket-id-console.png" alt="Pocket ID Console" />
|
||||
|
||||
2. **Navigate to OIDC Clients**: Click **"OIDC Clients"** in the applications in the left sidebar, you will be redirected to the OIDC Clients page
|
||||
|
||||
<ZoomableImage src="/assets/v3/oidc/pocket-id/oidc-clients.png" alt="OIDC Clients" />
|
||||
|
||||
3. **Create a new OIDC Client**: Click **"Add OIDC Client"** button in the OIDC Clients page
|
||||
|
||||
<ZoomableImage src="/assets/v3/oidc/pocket-id/create-oidc-client-button.png" alt="Create OIDC Client Button" />
|
||||
|
||||
Configure the following settings:
|
||||
|
||||
- **Name**: "Palmr File Sharing" (or your preferred name)
|
||||
- **Public Client**: "Diasabled"
|
||||
- **PKCE**: "Disabled"
|
||||
- **Logo**: "Upload a logo image"
|
||||
|
||||
<ZoomableImage src="/assets/v3/oidc/pocket-id/create-oidc-client.png" alt="Create OIDC Client" />
|
||||
|
||||
### Configuring application URLs
|
||||
|
||||
You'll need to configure several URLs in your Pocket ID application settings. Here's what to add for each environment:
|
||||
|
||||
### Redirect URIs
|
||||
|
||||
| Environment | URL |
|
||||
| ----------- | ------------------------------------------------------------------ |
|
||||
| Production | `https://yourdomain.com/api/auth/providers/pocketid/callback` |
|
||||
| Development | `http://localhost:3000/api/auth/providers/pocketid/callback` |
|
||||
| Custom Port | `https://yourdomain.com:5487/api/auth/providers/pocketid/callback` |
|
||||
|
||||
### Post Logout Redirect URIs
|
||||
|
||||
| Environment | URL |
|
||||
| ----------- | ----------------------------- |
|
||||
| Production | `https://yourdomain.com` |
|
||||
| Development | `http://localhost:3000` |
|
||||
| Custom Port | `https://yourdomain.com:5487` |
|
||||
|
||||
> **Note:** Replace `yourdomain.com` with your actual domain name in all production and custom port URLs.
|
||||
> **Note:** You can add multiple redirect URIs for different environments (development, staging, production).
|
||||
|
||||
<ZoomableImage src="/assets/v3/oidc/pocket-id/config-urls.png" alt="Pocket ID Application URLs Configuration" />
|
||||
|
||||
### Getting OAuth credentials
|
||||
|
||||
After creating your application, you'll receive your OAuth credentials:
|
||||
|
||||
<ZoomableImage
|
||||
src="/assets/v3/oidc/pocket-id/credentials.png"
|
||||
alt="Pocket ID OAuth Credentials"
|
||||
legend="The client ID and client secret shown in the image are examples only (fake credentials). You must use your own credentials from Pocket ID."
|
||||
/>
|
||||
|
||||
Save these credentials securely - you'll need them to configure Palmr:
|
||||
|
||||
- Client ID
|
||||
- Client Secret
|
||||
- Provider URL (your Pocket ID instance URL)
|
||||
|
||||
---
|
||||
|
||||
## Configuring Palmr
|
||||
|
||||
### Accessing OIDC settings
|
||||
|
||||
To configure Pocket ID authentication in Palmr:
|
||||
|
||||
1. **Login as administrator**: Sign in to Palmr with an admin account
|
||||
2. **Access settings**: Click your profile picture in the header and select **Settings**
|
||||
3. **Navigate to authentication**: Find and click on the **Authentication Providers** section
|
||||
|
||||
<ZoomableImage src="/assets/v3/oidc/auth-providers.png" alt="Palmr Authentication Providers" />
|
||||
|
||||
### Enabling Pocket ID provider
|
||||
|
||||
1. **Locate Pocket ID**: Find Pocket ID in the list of available providers
|
||||
2. **Enable the provider**: Toggle the status to **Enabled**
|
||||
|
||||
<ZoomableImage src="/assets/v3/oidc/pocket-id/enabled-pocket-id.png" alt="Palmr Pocket ID Provider Enabled" />
|
||||
|
||||
3. **Configure credentials**:
|
||||
- **Provider URL**: Your Pocket ID server URL (e.g., `https://auth.yourdomain.com`)
|
||||
- **Client ID**: Paste the Client ID from your Pocket ID application
|
||||
- **Client Secret**: Paste the Client Secret from your Pocket ID application
|
||||
|
||||
<ZoomableImage
|
||||
src="/assets/v3/oidc/pocket-id/edit-pocket-id.png"
|
||||
alt="Edit Pocket ID Provider"
|
||||
legend="This is a fake application, you have to use your own credentials."
|
||||
/>
|
||||
|
||||
### Advanced configuration options
|
||||
|
||||
Configure additional settings to customize the authentication behavior:
|
||||
|
||||
**Auto Registration**: Enable to automatically create user accounts when someone authenticates for the first time.
|
||||
|
||||
**Sort Order**: Control where the Pocket ID login button appears relative to other authentication providers.
|
||||
|
||||
**Icon**: Choose a custom icon for the Pocket ID login button (default is `Key`).
|
||||
|
||||
<ZoomableImage src="/assets/v3/oidc/pocket-id/pocket-id-icon.png" alt="Pocket ID Icon" />
|
||||
|
||||
---
|
||||
|
||||
## Account linking
|
||||
|
||||
By default, if a user is already registered in Palmr with their Pocket ID email, they will be automatically linked to their Palmr account.
|
||||
|
||||
> **Note:** You can't disable account linking. If you want to unlink a user from their Pocket ID account, you need to delete the user from Palmr.
|
||||
|
||||
---
|
||||
|
||||
## Technical configuration
|
||||
|
||||
Pocket ID's technical configuration is handled automatically through OIDC discovery, but understanding the setup can help with troubleshooting:
|
||||
|
||||
```yaml
|
||||
Provider Type: OAuth 2.0 with OIDC Discovery
|
||||
Issuer URL: https://your-pocket-id.domain.com
|
||||
Authorization Endpoint: /authorize
|
||||
Token Endpoint: /api/oidc/token
|
||||
UserInfo Endpoint: /api/oidc/userinfo
|
||||
Scopes: openid profile email
|
||||
```
|
||||
|
||||
### Field mappings
|
||||
|
||||
Palmr automatically maps Pocket ID user information to local user accounts:
|
||||
|
||||
- **User ID**: Maps from Pocket ID's `sub` field
|
||||
- **Email**: Maps from Pocket ID's `email` field
|
||||
- **Name**: Maps from Pocket ID's `name` field, falls back to `preferred_username`
|
||||
- **First Name**: Maps from Pocket ID's `given_name` field
|
||||
- **Last Name**: Maps from Pocket ID's `family_name` field
|
||||
- **Avatar**: Maps from Pocket ID's `picture` field
|
||||
|
||||
---
|
||||
|
||||
## Testing the configuration
|
||||
|
||||
### Verifying the setup
|
||||
|
||||
After configuring Pocket ID authentication, test the integration:
|
||||
|
||||
1. **Check login page**: Verify the "Sign in with Pocket ID" button appears
|
||||
2. **Test authentication flow**: Click the button and complete authentication
|
||||
3. **Verify user creation**: Confirm new user account creation (if auto-registration is enabled)
|
||||
|
||||
### Login flow verification
|
||||
|
||||
The complete authentication process should work as follows:
|
||||
|
||||
1. User clicks "Sign in with Pocket ID"
|
||||
2. User is redirected to Pocket ID login page
|
||||
3. User authenticates with their credentials
|
||||
4. Pocket ID redirects back to Palmr
|
||||
5. Palmr creates or updates the user account
|
||||
6. User gains access to Palmr
|
||||
|
||||
---
|
||||
|
||||
## Troubleshooting common issues
|
||||
|
||||
### Redirect URI mismatch
|
||||
|
||||
**Error**: `invalid_redirect_uri`
|
||||
|
||||
**Solution**:
|
||||
|
||||
1. Verify the exact callback URL in your Pocket ID application
|
||||
2. Check for protocol mismatches (http vs https)
|
||||
3. Ensure no trailing slashes unless specified
|
||||
4. Add development URLs if testing locally
|
||||
|
||||
### Authentication failures
|
||||
|
||||
**Error**: `access_denied` or `unauthorized_client`
|
||||
|
||||
**Solution**:
|
||||
|
||||
1. Verify Client ID and Secret are correct
|
||||
2. Check if the application is enabled in Pocket ID
|
||||
3. Ensure required scopes are configured
|
||||
4. Verify the user has necessary permissions
|
||||
|
||||
### Discovery endpoint issues
|
||||
|
||||
**Error**: Cannot fetch OIDC configuration
|
||||
|
||||
**Solution**:
|
||||
|
||||
1. Verify your Pocket ID server is accessible
|
||||
2. Check if the discovery endpoint (`/.well-known/openid-configuration`) is available
|
||||
3. Ensure SSL certificates are valid
|
||||
4. Check network connectivity and firewall rules
|
||||
|
||||
---
|
||||
|
||||
## Security best practices
|
||||
|
||||
### Credential management
|
||||
|
||||
- **Secure storage**: Keep Client Secret secure and never commit to version control
|
||||
- **Regular rotation**: Periodically rotate Client Secret
|
||||
- **Environment variables**: Store credentials in environment variables
|
||||
- **Access monitoring**: Regular review of authentication logs
|
||||
|
||||
### Production considerations
|
||||
|
||||
- **HTTPS required**: Always use HTTPS in production
|
||||
- **Valid certificates**: Ensure SSL certificates are valid
|
||||
- **Regular updates**: Keep Pocket ID server updated
|
||||
- **Backup strategy**: Regular backups of Pocket ID configuration
|
||||
|
||||
---
|
||||
|
||||
## Next steps
|
||||
|
||||
After configuring Pocket ID authentication:
|
||||
|
||||
- **Monitor usage**: Track authentication patterns
|
||||
- **Configure MFA**: Set up multi-factor authentication if needed
|
||||
- **User management**: Review auto-registration settings
|
||||
- **Backup verification**: Test backup and restore procedures
|
||||
|
||||
For more information about OIDC authentication in Palmr, see the [OIDC Authentication overview](/docs/3.2-beta/oidc-authentication).
|
||||
|
||||
## Useful resources
|
||||
|
||||
- [Pocket ID Documentation](https://docs.pocket-id.org)
|
||||
- [OIDC Specification](https://openid.net/specs/openid-connect-core-1_0.html)
|
||||
- [Palmr OIDC Overview](/docs/3.2-beta/oidc-authentication)
|
@@ -413,7 +413,7 @@ With Zitadel authentication configured, you might want to:
|
||||
- **Review security settings**: Ensure your authentication setup meets your security requirements
|
||||
- **Monitor usage**: Keep track of authentication patterns and user activity
|
||||
|
||||
For more information about OIDC authentication in Palmr, see the [OIDC Authentication overview](/docs/3.1-beta/oidc-authentication).
|
||||
For more information about OIDC authentication in Palmr, see the [OIDC Authentication overview](/docs/3.2-beta/oidc-authentication).
|
||||
|
||||
## Useful resources
|
||||
|
@@ -47,12 +47,12 @@ docker exec -it <container_name_or_id> /bin/sh
|
||||
|
||||
Replace `<container_name_or_id>` with the name or ID of your Palmr container. This command opens an interactive shell session inside the container, allowing you to execute commands directly.
|
||||
|
||||
### 3. Navigate to the server directory
|
||||
### 3. Navigate to the application directory
|
||||
|
||||
Once inside the container, navigate to the server directory where the reset script is located:
|
||||
Once inside the container, navigate to the application directory where the reset script is located:
|
||||
|
||||
```bash
|
||||
cd /app/server
|
||||
cd /app/palmr-app
|
||||
```
|
||||
|
||||
This directory contains the necessary scripts and configurations for managing Palmr's backend operations.
|
||||
@@ -135,11 +135,11 @@ If you encounter issues while running the script, refer to the following solutio
|
||||
- Confirm that the `prisma/palmr.db` file exists and has the correct permissions.
|
||||
- Verify that the container has access to the database volume.
|
||||
|
||||
- **Error: "Script must be run from server directory"**
|
||||
This error appears if you are not in the correct directory. Navigate to the server directory with:
|
||||
- **Error: "Script must be run from application directory"**
|
||||
This error appears if you are not in the correct directory. Navigate to the application directory with:
|
||||
|
||||
```bash
|
||||
cd /app/server
|
||||
cd /app/palmr-app
|
||||
```
|
||||
|
||||
- **Error: "User not found"**
|
378
apps/docs/content/docs/3.2-beta/quick-start.mdx
Normal file
@@ -0,0 +1,378 @@
|
||||
---
|
||||
title: Quick Start (Docker)
|
||||
icon: "Rocket"
|
||||
---
|
||||
|
||||
import { Callout } from "fumadocs-ui/components/callout";
|
||||
import { Tab, Tabs } from "fumadocs-ui/components/tabs";
|
||||
|
||||
import { Card, CardGrid } from "@/components/ui/card";
|
||||
|
||||
Welcome to the fastest way to deploy <span className="font-bold">Palmr.</span> - your secure, self-hosted file sharing solution. This guide will have you up and running in minutes, whether you're new to self-hosting or an experienced developer.
|
||||
|
||||
Palmr. offers flexible deployment options to match your infrastructure needs. This guide focuses on Docker deployment with our recommended filesystem storage, perfect for most use cases.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
Before you begin, make sure you have:
|
||||
|
||||
- **Docker** - Container runtime ([installation guide](https://docs.docker.com/get-docker/))
|
||||
- **Docker Compose** - Multi-container orchestration ([installation guide](https://docs.docker.com/compose/install/))
|
||||
- **2GB+ available disk space** for the application and your files
|
||||
- **Port 5487** available for the web interface
|
||||
- **Port 3333** available for API access (optional)
|
||||
|
||||
<Callout>
|
||||
**Platform Support**: Palmr. is developed on macOS and extensively tested on Linux servers. While we haven't formally
|
||||
tested other platforms, Docker's cross-platform nature should ensure compatibility. Report any issues on our [GitHub
|
||||
repository](https://github.com/kyantech/Palmr/issues).
|
||||
</Callout>
|
||||
|
||||
## Storage Options
|
||||
|
||||
Palmr. supports two storage approaches for persistent data:
|
||||
|
||||
- **Named Volumes (Recommended)** - Docker-managed storage with optimal performance and no permission issues
|
||||
- **Bind Mounts** - Direct host filesystem access, ideal for development and direct file management
|
||||
|
||||
## Deployment Options
|
||||
|
||||
Choose your storage method based on your needs:
|
||||
|
||||
<Tabs items={['Named Volumes (Recommended)', 'Bind Mounts']}>
|
||||
<Tab value="Named Volumes (Recommended)">
|
||||
Docker-managed storage that provides the best balance of performance, security, and ease of use:
|
||||
|
||||
- **No Permission Issues**: Docker handles all permission management automatically
|
||||
- **Performance**: Optimized for container workloads with better I/O performance
|
||||
- **Production Ready**: Recommended for production deployments
|
||||
|
||||
### Configuration
|
||||
|
||||
Create a `docker-compose.yml` file:
|
||||
|
||||
```yaml
|
||||
services:
|
||||
palmr:
|
||||
image: kyantech/palmr:latest
|
||||
container_name: palmr
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "5487:5487" # Web interface
|
||||
# - "3333:3333" # API (optional)
|
||||
environment:
|
||||
# Optional: Uncomment and configure as needed (if you don`t use, you can remove)
|
||||
# - ENABLE_S3=true # Set to true to enable S3-compatible storage
|
||||
# - DISABLE_FILESYSTEM_ENCRYPTION=true # Set to false to enable file encryption
|
||||
# - ENCRYPTION_KEY=your-secure-key-min-32-chars # Required only if encryption is enabled
|
||||
# - PALMR_UID=1000 # UID for the container processes (default is 1000)
|
||||
# - PALMR_GID=1000 # GID for the container processes (default is 1000)
|
||||
# - SECURE_SITE=false # Set to true if you are using a reverse proxy
|
||||
# - DEFAULT_LANGUAGE=en-US # Default language for the application (optional, defaults to en-US)
|
||||
# - PRESIGNED_URL_EXPIRATION=3600 # Duration in seconds for presigned URL expiration (optional, defaults to 3600 seconds / 1 hour)
|
||||
|
||||
# Download Memory Management (OPTIONAL - See Download Memory Management documentation)
|
||||
# - DOWNLOAD_MAX_CONCURRENT=5 # Maximum simultaneous downloads (auto-scales if not set)
|
||||
# - DOWNLOAD_MEMORY_THRESHOLD_MB=2048 # Memory threshold in MB before throttling (auto-scales if not set)
|
||||
# - DOWNLOAD_QUEUE_SIZE=25 # Maximum queue size for pending downloads (auto-scales if not set)
|
||||
# - DOWNLOAD_MIN_FILE_SIZE_GB=3.0 # Minimum file size in GB to activate memory management (default: 3.0)
|
||||
# - DOWNLOAD_AUTO_SCALE=true # Enable auto-scaling based on system memory (default: true)
|
||||
# - NODE_OPTIONS=--expose-gc # Enable garbage collection for large downloads (recommended for production)
|
||||
volumes:
|
||||
- palmr_data:/app/server
|
||||
|
||||
volumes:
|
||||
palmr_data:
|
||||
```
|
||||
|
||||
<Callout type="info">
|
||||
**Having upload or permission issues?** Add `PALMR_UID=1000` and `PALMR_GID=1000` to your environment variables. Check our [UID/GID Configuration](/docs/3.2-beta/uid-gid-configuration) guide for more details.
|
||||
</Callout>
|
||||
|
||||
### Deploy
|
||||
|
||||
```bash
|
||||
docker-compose up -d
|
||||
```
|
||||
|
||||
</Tab>
|
||||
<Tab value="Bind Mounts">
|
||||
Direct mapping to host filesystem directories, providing direct file access:
|
||||
|
||||
- **Direct Access**: Files are directly accessible from your host system
|
||||
- **Development Friendly**: Easy to inspect, modify, or backup files manually
|
||||
- **Platform Dependent**: May require UID/GID configuration, especially on NAS systems
|
||||
|
||||
### Configuration
|
||||
|
||||
Create a `docker-compose.yml` file:
|
||||
|
||||
```yaml
|
||||
services:
|
||||
palmr:
|
||||
image: kyantech/palmr:latest
|
||||
container_name: palmr
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "5487:5487" # Web interface
|
||||
# - "3333:3333" # API (optional)
|
||||
environment:
|
||||
# Optional: Uncomment and configure as needed (if you don`t use, you can remove)
|
||||
# - ENABLE_S3=true # Set to true to enable S3-compatible storage
|
||||
# - DISABLE_FILESYSTEM_ENCRYPTION=false # Set to false to enable file encryption
|
||||
# - ENCRYPTION_KEY=your-secure-key-min-32-chars # Required only if encryption is enabled
|
||||
# - PALMR_UID=1000 # UID for the container processes (default is 1000)
|
||||
# - PALMR_GID=1000 # GID for the container processes (default is 1000)
|
||||
# - SECURE_SITE=false # Set to true if you are using a reverse proxy
|
||||
# - DEFAULT_LANGUAGE=en-US # Default language for the application (optional, defaults to en-US)
|
||||
# - PRESIGNED_URL_EXPIRATION=3600 # Duration in seconds for presigned URL expiration (optional, defaults to 3600 seconds / 1 hour)
|
||||
|
||||
# Download Memory Management (OPTIONAL - See Download Memory Management documentation)
|
||||
# - DOWNLOAD_MAX_CONCURRENT=5 # Maximum simultaneous downloads (auto-scales if not set)
|
||||
# - DOWNLOAD_MEMORY_THRESHOLD_MB=2048 # Memory threshold in MB before throttling (auto-scales if not set)
|
||||
# - DOWNLOAD_QUEUE_SIZE=25 # Maximum queue size for pending downloads (auto-scales if not set)
|
||||
# - DOWNLOAD_MIN_FILE_SIZE_GB=3.0 # Minimum file size in GB to activate memory management (default: 3.0)
|
||||
# - DOWNLOAD_AUTO_SCALE=true # Enable auto-scaling based on system memory (default: true)
|
||||
# - NODE_OPTIONS=--expose-gc # Enable garbage collection for large downloads (recommended for production)
|
||||
volumes:
|
||||
- ./data:/app/server
|
||||
```
|
||||
|
||||
<Callout type="info">
|
||||
**Having upload or permission issues?** Add `PALMR_UID=1000` and `PALMR_GID=1000` to your environment variables. Check our [UID/GID Configuration](/docs/3.2-beta/uid-gid-configuration) guide for more details.
|
||||
</Callout>
|
||||
|
||||
### Deploy
|
||||
|
||||
```bash
|
||||
docker-compose up -d
|
||||
```
|
||||
|
||||
</Tab>
|
||||
</Tabs>
|
||||
|
||||
## Configuration
|
||||
|
||||
Customize Palmr's behavior with these environment variables:
|
||||
|
||||
| Variable | Default | Description |
|
||||
| ------------------------------- | ---------- | ---------------------------------------------------------------------------------------------------------------------- |
|
||||
| `ENABLE_S3` | `false` | Enable S3-compatible storage backends |
|
||||
| `S3_REJECT_UNAUTHORIZED` | `true` | Enable strict SSL certificate validation for S3 (set to `false` for self-signed certificates) |
|
||||
| `ENCRYPTION_KEY` | - | **Required when encryption is enabled**: 32+ character key for file encryption |
|
||||
| `DISABLE_FILESYSTEM_ENCRYPTION` | `true` | Disable file encryption for better performance (set to `false` to enable encryption) |
|
||||
| `SECURE_SITE` | `false` | Enable secure cookies for HTTPS/reverse proxy deployments |
|
||||
| `DEFAULT_LANGUAGE` | `en-US` | Default application language ([see available languages](/docs/3.2-beta/available-languages)) |
|
||||
| `PALMR_UID` | `1000` | User ID for container processes (helps with file permissions) |
|
||||
| `PALMR_GID` | `1000` | Group ID for container processes (helps with file permissions) |
|
||||
| `DOWNLOAD_MAX_CONCURRENT` | auto-scale | Maximum number of simultaneous downloads (see [Download Memory Management](/docs/3.2-beta/download-memory-management)) |
|
||||
| `DOWNLOAD_MEMORY_THRESHOLD_MB` | auto-scale | Memory threshold in MB before throttling |
|
||||
| `DOWNLOAD_QUEUE_SIZE` | auto-scale | Maximum queue size for pending downloads |
|
||||
| `DOWNLOAD_MIN_FILE_SIZE_GB` | `3.0` | Minimum file size in GB to activate memory management |
|
||||
| `DOWNLOAD_AUTO_SCALE` | `true` | Enable auto-scaling based on system memory |
|
||||
|
||||
<Callout type="info">
|
||||
**Performance First**: Palmr runs without encryption by default for optimal speed and lower resource usage—perfect for
|
||||
most use cases.
|
||||
</Callout>
|
||||
|
||||
<Callout type="warn">
|
||||
**Encryption Notice**: To enable encryption, set `DISABLE_FILESYSTEM_ENCRYPTION=false` and provide a 32+ character
|
||||
`ENCRYPTION_KEY`. **Important**: This choice is permanent—switching encryption modes after uploading files will break
|
||||
access to existing uploads.
|
||||
</Callout>
|
||||
|
||||
<Callout>
|
||||
**Using a Reverse Proxy?** Set `SECURE_SITE=true` and check our [Reverse Proxy
|
||||
Configuration](/docs/3.2-beta/reverse-proxy-configuration) guide for proper HTTPS setup.
|
||||
</Callout>
|
||||
|
||||
### Generate Encryption Keys (Optional)
|
||||
|
||||
Need file encryption? Generate a secure key:
|
||||
|
||||
<KeyGenerator />
|
||||
|
||||
> **Pro Tip**: Only enable encryption if you're handling sensitive data. For most users, the default unencrypted mode provides better performance.
|
||||
|
||||
## Access Your Instance
|
||||
|
||||
Once deployed, open Palmr in your browser:
|
||||
|
||||
- **Web Interface**: `http://localhost:5487` (local) or `http://YOUR_SERVER_IP:5487` (remote)
|
||||
- **API Documentation**: `http://localhost:3333/docs` (if port 3333 is exposed)
|
||||
|
||||
<Callout type="info">
|
||||
**Learn More**: For complete API documentation, authentication, and integration examples, see our [API
|
||||
Reference](/docs/3.2-beta/api) guide
|
||||
</Callout>
|
||||
|
||||
<Callout type="warn">
|
||||
**Production Ready?** Configure HTTPS with a valid SSL certificate for secure production deployments.
|
||||
</Callout>
|
||||
|
||||
---
|
||||
|
||||
## Docker CLI Alternative
|
||||
|
||||
Prefer Docker commands over Compose? Here are the equivalent commands:
|
||||
|
||||
<Tabs items={["Named Volume", "Bind Mount"]}>
|
||||
<Tab value="Named Volume">
|
||||
|
||||
```bash
|
||||
docker run -d \
|
||||
--name palmr \
|
||||
# Optional: Uncomment and configure as needed (if you don`t use, you can remove)
|
||||
# -e ENABLE_S3=true \ # Set to true to enable S3-compatible storage (OPTIONAL - default is false)
|
||||
# -e DISABLE_FILESYSTEM_ENCRYPTION=false \ # Set to false to enable file encryption (ENCRYPTION_KEY becomes required) | (OPTIONAL - default is true)
|
||||
# -e ENCRYPTION_KEY=your-secure-key-min-32-chars # Required only if encryption is enabled
|
||||
# -e PALMR_UID=1000 # UID for the container processes (default is 1000)
|
||||
# -e PALMR_GID=1000 # GID for the container processes (default is 1000)
|
||||
# -e SECURE_SITE=false # Set to true if you are using a reverse proxy
|
||||
# -e DEFAULT_LANGUAGE=en-US # Default language for the application (optional, defaults to en-US)
|
||||
-p 5487:5487 \
|
||||
-p 3333:3333 \
|
||||
-v palmr_data:/app/server \
|
||||
--restart unless-stopped \
|
||||
kyantech/palmr:latest
|
||||
```
|
||||
|
||||
|
||||
<Callout type="info">
|
||||
**Permission Issues?** Add `-e PALMR_UID=1000 -e PALMR_GID=1000` to the command above. See our [UID/GID Configuration](/docs/3.2-beta/uid-gid-configuration) guide for details.
|
||||
</Callout>
|
||||
|
||||
</Tab>
|
||||
|
||||
<Tab value="Bind Mount">
|
||||
|
||||
```bash
|
||||
docker run -d \
|
||||
--name palmr \
|
||||
# Optional: Uncomment and configure as needed (if you don`t use, you can remove)
|
||||
# -e ENABLE_S3=true \ # Set to true to enable S3-compatible storage (OPTIONAL - default is false)
|
||||
# -e DISABLE_FILESYSTEM_ENCRYPTION=true \ # Set to false to enable file encryption (ENCRYPTION_KEY becomes required) | (OPTIONAL - default is true)
|
||||
# -e ENCRYPTION_KEY=your-secure-key-min-32-chars # Required only if encryption is enabled
|
||||
# -e PALMR_UID=1000 # UID for the container processes (default is 1000)
|
||||
# -e PALMR_GID=1000 # GID for the container processes (default is 1000)
|
||||
# -e SECURE_SITE=false # Set to true if you are using a reverse proxy
|
||||
# -e DEFAULT_LANGUAGE=en-US # Default language for the application (optional, defaults to en-US)
|
||||
-p 5487:5487 \
|
||||
-p 3333:3333 \
|
||||
-v $(pwd)/data:/app/server \
|
||||
--restart unless-stopped \
|
||||
kyantech/palmr:latest
|
||||
```
|
||||
|
||||
<Callout type="info">
|
||||
**Permission Issues?** Add `-e PALMR_UID=1000 -e PALMR_GID=1000` to the command above. See our [UID/GID Configuration](/docs/3.2-beta/uid-gid-configuration) guide for details.
|
||||
</Callout>
|
||||
|
||||
</Tab>
|
||||
</Tabs>
|
||||
|
||||
---
|
||||
|
||||
## Common Configuration Options
|
||||
|
||||
### Presigned URL Expiration
|
||||
|
||||
Palmr. uses temporary URLs (presigned URLs) for secure file access. These URLs expire after a configurable time period to enhance security.
|
||||
|
||||
**Default:** 1 hour (3600 seconds)
|
||||
|
||||
You can customize this for all storage types (filesystem or S3) by adding:
|
||||
|
||||
```yaml
|
||||
environment:
|
||||
- PRESIGNED_URL_EXPIRATION=7200 # 2 hours
|
||||
```
|
||||
|
||||
**When to adjust:**
|
||||
|
||||
- **Shorter time (1800 = 30 min):** Higher security, but users may need to refresh download links
|
||||
- **Longer time (7200-21600 = 2-6 hours):** Better for large file transfers, but URLs stay valid longer
|
||||
- **Default (3600 = 1 hour):** Good balance for most use cases
|
||||
|
||||
### File Encryption
|
||||
|
||||
For filesystem storage, you can enable file encryption:
|
||||
|
||||
```yaml
|
||||
environment:
|
||||
- DISABLE_FILESYSTEM_ENCRYPTION=false
|
||||
- ENCRYPTION_KEY=your-secure-32-character-key-here
|
||||
```
|
||||
|
||||
**Note:** S3 storage handles encryption through your S3 provider's encryption features.
|
||||
|
||||
---
|
||||
|
||||
## Maintenance
|
||||
|
||||
### Updates
|
||||
|
||||
Keep Palmr up to date with the latest features and security patches:
|
||||
|
||||
```bash
|
||||
docker-compose pull
|
||||
docker-compose up -d
|
||||
```
|
||||
|
||||
### Backup Your Data
|
||||
|
||||
**Named Volumes:**
|
||||
|
||||
```bash
|
||||
docker run --rm -v palmr_data:/data -v $(pwd):/backup alpine tar czf /backup/palmr-backup.tar.gz -C /data .
|
||||
```
|
||||
|
||||
**Bind Mounts:**
|
||||
|
||||
```bash
|
||||
tar czf palmr-backup.tar.gz ./data
|
||||
```
|
||||
|
||||
### Restore From Backup
|
||||
|
||||
**Named Volumes:**
|
||||
|
||||
```bash
|
||||
docker-compose down
|
||||
docker run --rm -v palmr_data:/data -v $(pwd):/backup alpine tar xzf /backup/palmr-backup.tar.gz -C /data
|
||||
docker-compose up -d
|
||||
```
|
||||
|
||||
**Bind Mounts:**
|
||||
|
||||
```bash
|
||||
docker-compose down
|
||||
tar xzf palmr-backup.tar.gz
|
||||
docker-compose up -d
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## What's Next?
|
||||
|
||||
Your Palmr instance is ready! Here's what you can explore:
|
||||
|
||||
### Advanced Configuration
|
||||
|
||||
- **[UID/GID Configuration](/docs/3.2-beta/uid-gid-configuration)** - Configure user permissions for NAS systems and custom environments
|
||||
- **[Download Memory Management](/docs/3.2-beta/download-memory-management)** - Configure large file download handling and queue system
|
||||
- **[S3 Storage](/docs/3.2-beta/s3-configuration)** - Scale with Amazon S3 or compatible storage providers
|
||||
- **[Manual Installation](/docs/3.2-beta/manual-installation)** - Manual installation and custom configurations
|
||||
|
||||
### Integration & Development
|
||||
|
||||
- **[API Reference](/docs/3.2-beta/api)** - Integrate Palmr. with your applications
|
||||
|
||||
<Callout type="info">
|
||||
**Need help?** Check our [Troubleshooting Guide](/docs/3.2-beta/troubleshooting) for common issues and solutions.
|
||||
</Callout>
|
||||
|
||||
---
|
||||
|
||||
**Questions?** Visit our [GitHub Issues](https://github.com/kyantech/Palmr/issues) or join the community discussions.
|
@@ -127,10 +127,11 @@ proxy_pass_header Set-Cookie;
|
||||
environment:
|
||||
- PALMR_UID=1000 # Your host UID (check with: id)
|
||||
- PALMR_GID=1000 # Your host GID
|
||||
- ENCRYPTION_KEY=your-key-here
|
||||
- DISABLE_FILESYSTEM_ENCRYPTION=true # Set to false to enable file encryption
|
||||
# - ENCRYPTION_KEY=your-key-here # Required only if encryption is enabled
|
||||
```
|
||||
|
||||
> **💡 Note**: Check your host UID/GID with `id` command and use those values. See [UID/GID Configuration](/docs/3.1-beta/uid-gid-configuration) for detailed setup.
|
||||
> **💡 Note**: Check your host UID/GID with `id` command and use those values. See [UID/GID Configuration](/docs/3.2-beta/uid-gid-configuration) for detailed setup.
|
||||
|
||||
---
|
||||
|
@@ -7,6 +7,8 @@ This guide provides comprehensive configuration instructions for integrating Pal
|
||||
|
||||
> **Overview:** Palmr. supports any S3-compatible storage provider, giving you flexibility to choose the solution that best fits your needs and budget.
|
||||
|
||||
> **Note:** Some configuration options (like presigned URL expiration) apply to **all storage types**, including filesystem storage. These are marked accordingly in the documentation.
|
||||
|
||||
## When to use S3-compatible storage
|
||||
|
||||
Consider using S3-compatible storage when you need:
|
||||
@@ -19,18 +21,27 @@ Consider using S3-compatible storage when you need:
|
||||
|
||||
## Environment variables
|
||||
|
||||
### General configuration (applies to all storage types)
|
||||
|
||||
| Variable | Description | Required | Default |
|
||||
| -------------------------- | ------------------------------------------------ | -------- | --------------- |
|
||||
| `PRESIGNED_URL_EXPIRATION` | Duration in seconds for presigned URL expiration | No | `3600` (1 hour) |
|
||||
|
||||
### S3-specific configuration
|
||||
|
||||
To enable S3-compatible storage, set `ENABLE_S3=true` and configure the following environment variables:
|
||||
|
||||
| Variable | Description | Required | Default |
|
||||
| --------------------- | ----------------------------- | -------- | ----------------- |
|
||||
| `S3_ENDPOINT` | S3 provider endpoint URL | Yes | - |
|
||||
| `S3_PORT` | Connection port | No | Based on protocol |
|
||||
| `S3_USE_SSL` | Enable SSL/TLS encryption | Yes | `true` |
|
||||
| `S3_ACCESS_KEY` | Access key for authentication | Yes | - |
|
||||
| `S3_SECRET_KEY` | Secret key for authentication | Yes | - |
|
||||
| `S3_REGION` | Storage region | Yes | - |
|
||||
| `S3_BUCKET_NAME` | Bucket/container name | Yes | - |
|
||||
| `S3_FORCE_PATH_STYLE` | Use path-style URLs | No | `false` |
|
||||
| Variable | Description | Required | Default |
|
||||
| ------------------------ | ---------------------------------------- | -------- | ----------------- |
|
||||
| `S3_ENDPOINT` | S3 provider endpoint URL | Yes | - |
|
||||
| `S3_PORT` | Connection port | No | Based on protocol |
|
||||
| `S3_USE_SSL` | Enable SSL/TLS encryption | Yes | `true` |
|
||||
| `S3_ACCESS_KEY` | Access key for authentication | Yes | - |
|
||||
| `S3_SECRET_KEY` | Secret key for authentication | Yes | - |
|
||||
| `S3_REGION` | Storage region | Yes | - |
|
||||
| `S3_BUCKET_NAME` | Bucket/container name | Yes | - |
|
||||
| `S3_FORCE_PATH_STYLE` | Use path-style URLs | No | `false` |
|
||||
| `S3_REJECT_UNAUTHORIZED` | Enable strict SSL certificate validation | No | `true` |
|
||||
|
||||
## Provider configurations
|
||||
|
||||
@@ -51,6 +62,7 @@ S3_SECRET_KEY=your-secret-access-key
|
||||
S3_REGION=us-east-1
|
||||
S3_BUCKET_NAME=your-bucket-name
|
||||
S3_FORCE_PATH_STYLE=false
|
||||
# PRESIGNED_URL_EXPIRATION=3600 # Optional: 1 hour (default)
|
||||
```
|
||||
|
||||
**Getting credentials:**
|
||||
@@ -81,6 +93,21 @@ S3_FORCE_PATH_STYLE=true
|
||||
- Default MinIO port is 9000
|
||||
- SSL can be disabled for local development
|
||||
|
||||
**For MinIO with self-signed SSL certificates:**
|
||||
|
||||
```bash
|
||||
ENABLE_S3=true
|
||||
S3_ENDPOINT=your-minio-domain.com
|
||||
S3_PORT=9000
|
||||
S3_USE_SSL=true
|
||||
S3_ACCESS_KEY=your-minio-access-key
|
||||
S3_SECRET_KEY=your-minio-secret-key
|
||||
S3_REGION=us-east-1
|
||||
S3_BUCKET_NAME=your-bucket-name
|
||||
S3_FORCE_PATH_STYLE=true
|
||||
S3_REJECT_UNAUTHORIZED=false # Allows self-signed certificates
|
||||
```
|
||||
|
||||
### Google Cloud Storage
|
||||
|
||||
Google Cloud Storage offers competitive pricing and global infrastructure.
|
||||
@@ -137,6 +164,7 @@ S3_SECRET_KEY=your-application-key
|
||||
S3_REGION=us-west-002
|
||||
S3_BUCKET_NAME=your-bucket-name
|
||||
S3_FORCE_PATH_STYLE=false
|
||||
# PRESIGNED_URL_EXPIRATION=7200 # Optional: 2 hours for large files
|
||||
```
|
||||
|
||||
**Cost advantage:**
|
||||
@@ -187,6 +215,93 @@ S3_FORCE_PATH_STYLE=false
|
||||
- Use container name as bucket name
|
||||
- Configure appropriate access policies
|
||||
|
||||
## Presigned URL configuration
|
||||
|
||||
Palmr. uses presigned URLs to provide secure, temporary access to files stored in **both S3-compatible storage and filesystem storage**. These URLs have a configurable expiration time to balance security and usability.
|
||||
|
||||
> **Note:** This configuration applies to **all storage types** (S3, filesystem, etc.), not just S3-compatible storage.
|
||||
|
||||
### Understanding presigned URLs
|
||||
|
||||
Presigned URLs are temporary URLs that allow direct access to files without exposing storage credentials or requiring authentication. They automatically expire after a specified time period, enhancing security by limiting access duration.
|
||||
|
||||
**How it works:**
|
||||
|
||||
- **S3 Storage:** URLs are signed by AWS/S3-compatible provider credentials
|
||||
- **Filesystem Storage:** URLs use temporary tokens that are validated by Palmr server
|
||||
|
||||
**Default behavior:**
|
||||
|
||||
- Upload URLs: 1 hour (3600 seconds)
|
||||
- Download URLs: 1 hour (3600 seconds)
|
||||
|
||||
### Configuring expiration time
|
||||
|
||||
You can customize the expiration time using the `PRESIGNED_URL_EXPIRATION` environment variable:
|
||||
|
||||
```bash
|
||||
# Set URLs to expire after 2 hours (7200 seconds)
|
||||
PRESIGNED_URL_EXPIRATION=7200
|
||||
|
||||
# Set URLs to expire after 30 minutes (1800 seconds)
|
||||
PRESIGNED_URL_EXPIRATION=1800
|
||||
|
||||
# Set URLs to expire after 6 hours (21600 seconds)
|
||||
PRESIGNED_URL_EXPIRATION=21600
|
||||
```
|
||||
|
||||
### Choosing the right expiration time
|
||||
|
||||
**Shorter expiration (15-30 minutes):**
|
||||
|
||||
- [+] Higher security
|
||||
- [+] Reduced risk of unauthorized access
|
||||
- [-] May interrupt long uploads/downloads
|
||||
- [-] Users may need to refresh links more often
|
||||
|
||||
**Longer expiration (2-6 hours):**
|
||||
|
||||
- [+] Better user experience for large files
|
||||
- [+] Fewer interruptions during transfers
|
||||
- [-] Longer exposure window if URLs are compromised
|
||||
- [-] Potential for increased storage costs if users leave downloads incomplete
|
||||
|
||||
**Recommended settings:**
|
||||
|
||||
- **High security environments:** 1800 seconds (30 minutes)
|
||||
- **Standard usage:** 3600 seconds (1 hour) - default
|
||||
- **Large file transfers:** 7200-21600 seconds (2-6 hours)
|
||||
|
||||
### Example configurations
|
||||
|
||||
**For Backblaze B2 with extended expiration:**
|
||||
|
||||
```bash
|
||||
ENABLE_S3=true
|
||||
S3_ENDPOINT=s3.us-west-002.backblazeb2.com
|
||||
S3_USE_SSL=true
|
||||
S3_ACCESS_KEY=your-key-id
|
||||
S3_SECRET_KEY=your-application-key
|
||||
S3_REGION=us-west-002
|
||||
S3_BUCKET_NAME=your-bucket-name
|
||||
S3_FORCE_PATH_STYLE=false
|
||||
PRESIGNED_URL_EXPIRATION=7200 # 2 hours for large file transfers
|
||||
```
|
||||
|
||||
**For high-security environments:**
|
||||
|
||||
```bash
|
||||
ENABLE_S3=true
|
||||
S3_ENDPOINT=s3.amazonaws.com
|
||||
S3_USE_SSL=true
|
||||
S3_ACCESS_KEY=your-access-key-id
|
||||
S3_SECRET_KEY=your-secret-access-key
|
||||
S3_REGION=us-east-1
|
||||
S3_BUCKET_NAME=your-bucket-name
|
||||
S3_FORCE_PATH_STYLE=false
|
||||
PRESIGNED_URL_EXPIRATION=1800 # 30 minutes for enhanced security
|
||||
```
|
||||
|
||||
## Configuration best practices
|
||||
|
||||
### Security considerations
|
||||
@@ -212,6 +327,19 @@ S3_FORCE_PATH_STYLE=false
|
||||
- Check firewall and network connectivity
|
||||
- Ensure SSL/TLS settings match provider requirements
|
||||
|
||||
**SSL certificate errors (self-signed certificates):**
|
||||
|
||||
If you encounter errors like `unable to verify the first certificate` or `UNABLE_TO_VERIFY_LEAF_SIGNATURE`, you're likely using self-signed SSL certificates. This is common with self-hosted MinIO or other S3-compatible services.
|
||||
|
||||
**Solution:**
|
||||
Set `S3_REJECT_UNAUTHORIZED=false` in your environment variables to allow self-signed certificates:
|
||||
|
||||
```bash
|
||||
S3_REJECT_UNAUTHORIZED=false
|
||||
```
|
||||
|
||||
**Note:** SSL certificate validation is enabled by default (`true`) for security. Set it to `false` only when using self-hosted S3 services with self-signed certificates.
|
||||
|
||||
**Authentication failures:**
|
||||
|
||||
- Confirm access key and secret key are correct
|
@@ -17,7 +17,7 @@ docker-compose logs palmr | grep -i "permission\|denied\|eacces"
|
||||
|
||||
# Common error messages:
|
||||
# EACCES: permission denied, open '/app/server/uploads/file.txt'
|
||||
# Error: EACCES: permission denied, mkdir '/app/server/temp-chunks'
|
||||
# Error: EACCES: permission denied, mkdir '/app/server/temp-uploads'
|
||||
```
|
||||
|
||||
### The Root Cause
|
||||
@@ -25,7 +25,7 @@ docker-compose logs palmr | grep -i "permission\|denied\|eacces"
|
||||
**Palmr. defaults**: UID 1001, GID 1001
|
||||
**Linux standard**: UID 1000, GID 1000
|
||||
|
||||
When using bind mounts, your host directories are owned by UID 1000, but Palmr. runs as UID 1001.
|
||||
When using bind mounts, your host directories may have different ownership than Palmr's default UID/GID.
|
||||
|
||||
### Solution 1: Environment Variables (Recommended)
|
||||
|
||||
@@ -63,8 +63,8 @@ If you prefer to keep Palmr's defaults:
|
||||
chown -R 1001:1001 ./data
|
||||
|
||||
# For separate upload/temp directories
|
||||
mkdir -p uploads temp-chunks
|
||||
chown -R 1001:1001 uploads temp-chunks
|
||||
mkdir -p uploads temp-uploads
|
||||
chown -R 1001:1001 uploads temp-uploads
|
||||
```
|
||||
|
||||
### Solution 3: Docker Volume (Avoid the Issue)
|
||||
@@ -109,16 +109,19 @@ docker-compose logs palmr
|
||||
2. **Invalid encryption key**
|
||||
|
||||
```bash
|
||||
# Error: Encryption key must be at least 32 characters
|
||||
# Fix: Update ENCRYPTION_KEY in docker-compose.yaml
|
||||
# Error: Encryption key must be at least 32 characters (only if encryption is enabled)
|
||||
# Fix: Either disable encryption or provide a valid key
|
||||
environment:
|
||||
- ENCRYPTION_KEY=your-very-long-secure-key-at-least-32-characters
|
||||
- DISABLE_FILESYSTEM_ENCRYPTION=true # Disable encryption (default)
|
||||
# OR enable encryption with:
|
||||
# - DISABLE_FILESYSTEM_ENCRYPTION=false
|
||||
# - ENCRYPTION_KEY=your-very-long-secure-key-at-least-32-characters
|
||||
```
|
||||
|
||||
3. **Missing environment variables**
|
||||
```bash
|
||||
# Check required variables are set
|
||||
docker exec palmr env | grep -E "ENCRYPTION_KEY|DATABASE_URL"
|
||||
# Check variables are set (encryption is optional)
|
||||
docker exec palmr env | grep -E "DISABLE_FILESYSTEM_ENCRYPTION|ENCRYPTION_KEY|DATABASE_URL"
|
||||
```
|
||||
|
||||
### Container Starts But App Doesn't Load
|
||||
@@ -151,7 +154,7 @@ curl http://localhost:3333/health
|
||||
|
||||
```bash
|
||||
docker exec palmr ls -la /app/server/uploads/
|
||||
# Should show ownership by palmr user
|
||||
# Should show ownership by palmr user (UID 1001)
|
||||
```
|
||||
|
||||
3. **Check upload limits:**
|
||||
@@ -178,13 +181,13 @@ docker exec palmr stat /app/server/uploads/your-file.txt
|
||||
|
||||
```bash
|
||||
# Using the built-in reset script
|
||||
docker exec -it palmr /app/reset-password.sh
|
||||
docker exec -it palmr /app/palmr-app/reset-password.sh
|
||||
```
|
||||
|
||||
2. **Check database permissions:**
|
||||
```bash
|
||||
docker exec palmr ls -la /app/server/prisma/
|
||||
# palmr.db should be writable by palmr user
|
||||
# palmr.db should be writable by palmr user (UID 1001)
|
||||
```
|
||||
|
||||
### OIDC Authentication Not Working
|
||||
@@ -243,8 +246,8 @@ docker exec palmr ls -la /app/server/prisma/palmr.db
|
||||
# Check database logs
|
||||
docker-compose logs palmr | grep -i database
|
||||
|
||||
# Verify Prisma schema
|
||||
docker exec palmr npx prisma db push --schema=./prisma/schema.prisma
|
||||
# Verify Prisma schema (run from palmr-app directory)
|
||||
docker exec palmr sh -c "cd /app/palmr-app && npx prisma db push"
|
||||
```
|
||||
|
||||
### Database Corruption
|
||||
@@ -283,7 +286,7 @@ docker-compose up -d
|
||||
|
||||
3. **Check temp directory permissions:**
|
||||
```bash
|
||||
docker exec palmr ls -la /app/server/temp-chunks/
|
||||
docker exec palmr ls -la /app/server/temp-uploads/
|
||||
```
|
||||
|
||||
### High Memory Usage
|
||||
@@ -318,16 +321,19 @@ docker port palmr
|
||||
echo "4. File Permissions:"
|
||||
docker exec palmr ls -la /app/server/
|
||||
|
||||
echo "5. Environment Variables:"
|
||||
docker exec palmr env | grep -E "PALMR_|ENCRYPTION_|DATABASE_"
|
||||
echo "5. Application Files:"
|
||||
docker exec palmr ls -la /app/palmr-app/
|
||||
|
||||
echo "6. API Health:"
|
||||
echo "6. Environment Variables:"
|
||||
docker exec palmr env | grep -E "PALMR_|DISABLE_FILESYSTEM_ENCRYPTION|ENCRYPTION_|DATABASE_"
|
||||
|
||||
echo "7. API Health:"
|
||||
curl -s http://localhost:3333/health || echo "API not accessible"
|
||||
|
||||
echo "7. Web Interface:"
|
||||
echo "8. Web Interface:"
|
||||
curl -s -o /dev/null -w "%{http_code}" http://localhost:5487 || echo "Web interface not accessible"
|
||||
|
||||
echo "8. Disk Space:"
|
||||
echo "9. Disk Space:"
|
||||
df -h
|
||||
|
||||
echo "=== End Health Check ==="
|
@@ -9,15 +9,15 @@ Configure user and group permissions for seamless bind mount compatibility acros
|
||||
|
||||
Palmr. supports runtime UID/GID configuration to resolve permission conflicts when using bind mounts. This eliminates the need for manual permission management on your host system.
|
||||
|
||||
**⚠️ Important**: Palmr uses **UID 1001, GID 1001** by default, which is different from the standard Linux convention of **UID 1000, GID 1000**. This is the most common cause of permission issues with bind mounts.
|
||||
**⚠️ Important**: Palmr uses **UID 1000, GID 1000** by default, which matches the standard Linux convention. However, some systems may use different UID/GID values, which can cause permission issues with bind mounts.
|
||||
|
||||
## The Permission Problem
|
||||
|
||||
### Why This Happens
|
||||
|
||||
- **Palmr Default**: UID 1001, GID 1001 (container)
|
||||
- **Palmr Default**: UID 1000, GID 1000 (container)
|
||||
- **Linux Standard**: UID 1000, GID 1000 (most host systems)
|
||||
- **Result**: Container can't write to host directories
|
||||
- **Result**: Usually compatible, but some systems may use different values
|
||||
|
||||
### Common Error Scenarios
|
||||
|
||||
@@ -30,7 +30,7 @@ EACCES: permission denied, open '/app/server/uploads/file.txt'
|
||||
# Or when checking permissions:
|
||||
$ ls -la uploads/
|
||||
drwxr-xr-x 2 user user 4096 Jan 15 10:00 uploads/
|
||||
# Container tries to write as UID 1001, but directory is owned by UID 1000
|
||||
# Container tries to write with different UID/GID than directory owner
|
||||
```
|
||||
|
||||
## Quick Fix
|
||||
@@ -45,15 +45,13 @@ services:
|
||||
image: kyantech/palmr:latest
|
||||
container_name: palmr
|
||||
environment:
|
||||
- ENABLE_S3=false
|
||||
- ENCRYPTION_KEY=your-secure-key-min-32-chars
|
||||
- PALMR_UID=1000
|
||||
- PALMR_GID=1000
|
||||
ports:
|
||||
- "5487:5487"
|
||||
volumes:
|
||||
- ./uploads:/app/server/uploads:rw
|
||||
- ./temp-chunks:/app/server/temp-chunks:rw
|
||||
- ./temp-uploads:/app/server/temp-uploads:rw
|
||||
restart: unless-stopped
|
||||
```
|
||||
|
||||
@@ -63,8 +61,8 @@ If you prefer to keep Palmr's defaults:
|
||||
|
||||
```bash
|
||||
# Create directories with correct ownership
|
||||
mkdir -p uploads temp-chunks
|
||||
chown -R 1001:1001 uploads temp-chunks
|
||||
mkdir -p uploads temp-uploads
|
||||
chown -R 1001:1001 uploads temp-uploads
|
||||
```
|
||||
|
||||
## Environment Variables
|
||||
@@ -104,8 +102,6 @@ services:
|
||||
image: kyantech/palmr:latest
|
||||
container_name: palmr
|
||||
environment:
|
||||
- ENABLE_S3=false
|
||||
- ENCRYPTION_KEY=your-secure-key-min-32-chars
|
||||
- PALMR_UID=1000
|
||||
- PALMR_GID=1000
|
||||
ports:
|
||||
@@ -123,8 +119,6 @@ services:
|
||||
image: kyantech/palmr:latest
|
||||
container_name: palmr
|
||||
environment:
|
||||
- ENABLE_S3=false
|
||||
- ENCRYPTION_KEY=your-secure-key-min-32-chars
|
||||
- PALMR_UID=1026
|
||||
- PALMR_GID=100
|
||||
ports:
|
||||
@@ -142,8 +136,6 @@ services:
|
||||
image: kyantech/palmr:latest
|
||||
container_name: palmr
|
||||
environment:
|
||||
- ENABLE_S3=false
|
||||
- ENCRYPTION_KEY=your-secure-key-min-32-chars
|
||||
- PALMR_UID=1000
|
||||
- PALMR_GID=100
|
||||
ports:
|
||||
@@ -166,7 +158,7 @@ services:
|
||||
id
|
||||
|
||||
# 2. Check directory ownership
|
||||
ls -la uploads/ temp-chunks/
|
||||
ls -la uploads/ temp-uploads/
|
||||
|
||||
# 3. Fix via environment variables (preferred)
|
||||
# Add to docker-compose.yaml:
|
||||
@@ -174,7 +166,7 @@ ls -la uploads/ temp-chunks/
|
||||
# - PALMR_GID=1000
|
||||
|
||||
# 4. Or fix via chown (alternative)
|
||||
chown -R 1001:1001 uploads temp-chunks
|
||||
chown -R 1001:1001 uploads temp-uploads
|
||||
```
|
||||
|
||||
**Error**: Container starts but files aren't accessible
|
||||
@@ -225,11 +217,11 @@ cat /etc/passwd | grep -v nobody
|
||||
```bash
|
||||
# Check if directories exist and are writable
|
||||
test -w uploads && echo "uploads writable" || echo "uploads NOT writable"
|
||||
test -w temp-chunks && echo "temp-chunks writable" || echo "temp-chunks NOT writable"
|
||||
test -w temp-uploads && echo "temp-uploads writable" || echo "temp-uploads NOT writable"
|
||||
|
||||
# Create directories with correct permissions
|
||||
mkdir -p uploads temp-chunks
|
||||
sudo chown -R $(id -u):$(id -g) uploads temp-chunks
|
||||
mkdir -p uploads temp-uploads
|
||||
sudo chown -R $(id -u):$(id -g) uploads temp-uploads
|
||||
```
|
||||
|
||||
---
|
||||
@@ -270,7 +262,7 @@ To add UID/GID configuration to running installations:
|
||||
cp -r ./data ./data-backup
|
||||
# or
|
||||
cp -r ./uploads ./uploads-backup
|
||||
cp -r ./temp-chunks ./temp-chunks-backup
|
||||
cp -r ./temp-uploads ./temp-uploads-backup
|
||||
```
|
||||
|
||||
3. **Check your UID/GID**
|
||||
@@ -344,4 +336,4 @@ For most users experiencing permission issues with bind mounts:
|
||||
```
|
||||
3. **Restart**: `docker-compose down && docker-compose up -d`
|
||||
|
||||
This resolves the mismatch between Palmr's default UID 1001 and the standard Linux UID 1000.
|
||||
This ensures compatibility between Palmr's UID/GID and your host system's file ownership.
|
@@ -1,3 +1,6 @@
|
||||
{
|
||||
"pages": ["3.1-beta", "2.0.0-beta"]
|
||||
}
|
||||
"pages": [
|
||||
"3.2-beta",
|
||||
"2.0.0-beta"
|
||||
]
|
||||
}
|
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "palmr-docs",
|
||||
"version": "3.1-beta",
|
||||
"version": "3.2.0-beta",
|
||||
"description": "Docs for Palmr",
|
||||
"private": true,
|
||||
"author": "Daniel Luiz Alves <daniel@kyantech.com.br>",
|
||||
|
BIN
apps/docs/public/assets/v3/oidc/pocket-id/config-urls.png
Normal file
After Width: | Height: | Size: 152 KiB |
After Width: | Height: | Size: 89 KiB |
BIN
apps/docs/public/assets/v3/oidc/pocket-id/create-oidc-client.png
Normal file
After Width: | Height: | Size: 140 KiB |
BIN
apps/docs/public/assets/v3/oidc/pocket-id/credentials.png
Normal file
After Width: | Height: | Size: 188 KiB |
BIN
apps/docs/public/assets/v3/oidc/pocket-id/edit-pocket-id.png
Normal file
After Width: | Height: | Size: 161 KiB |
BIN
apps/docs/public/assets/v3/oidc/pocket-id/enabled-pocket-id.png
Normal file
After Width: | Height: | Size: 149 KiB |
BIN
apps/docs/public/assets/v3/oidc/pocket-id/oidc-clients.png
Normal file
After Width: | Height: | Size: 89 KiB |
BIN
apps/docs/public/assets/v3/oidc/pocket-id/pocket-id-console.png
Normal file
After Width: | Height: | Size: 172 KiB |
BIN
apps/docs/public/assets/v3/oidc/pocket-id/pocket-id-icon.png
Normal file
After Width: | Height: | Size: 142 KiB |
After Width: | Height: | Size: 843 KiB |
Before Width: | Height: | Size: 104 KiB After Width: | Height: | Size: 130 KiB |
@@ -1,3 +1,5 @@
|
||||
"use client";
|
||||
|
||||
import type { ReactNode } from "react";
|
||||
import Link from "next/link";
|
||||
import {
|
||||
@@ -57,13 +59,13 @@ const images = [
|
||||
"https://res.cloudinary.com/technical-intelligence/image/upload/v1745546005/Palmr./profile_mizwvg.png",
|
||||
];
|
||||
|
||||
const docsLink = "/docs/3.1-beta";
|
||||
const docsLink = "/docs/3.2-beta";
|
||||
|
||||
function Hero() {
|
||||
return (
|
||||
<section className="relative z-[2] flex flex-col border-x border-t px-6 pt-12 pb-10 md:px-12 md:pt-16 max-md:text-center">
|
||||
<h1 className="mb-8 text-6xl font-bold">
|
||||
Palmr. <span className="text-[13px] font-light text-muted-foreground/50 font-mono">v3.1-beta</span>
|
||||
Palmr. <span className="text-[13px] font-light text-muted-foreground/50 font-mono">v3.2-beta</span>
|
||||
</h1>
|
||||
<h1 className="hidden text-4xl font-medium max-w-[600px] md:block mb-4">Modern & efficient file sharing</h1>
|
||||
<p className="mb-8 text-fd-muted-foreground md:max-w-[80%] md:text-xl">
|
||||
@@ -79,7 +81,6 @@ function Hero() {
|
||||
<Link href={docsLink}>Documentation</Link>
|
||||
</div>
|
||||
</PulsatingButton>
|
||||
|
||||
<RippleButton>
|
||||
<a
|
||||
href="https://github.com/kyantech/Palmr"
|
||||
|
@@ -9,6 +9,6 @@ export const { GET } = createFromSource(source, (page) => {
|
||||
url: page.url,
|
||||
id: page.url,
|
||||
structuredData: page.data.structuredData,
|
||||
tag: page.url.startsWith("/docs/3.1-beta") ? "v3.1-beta" : "v2.0.0-beta",
|
||||
tag: page.url.startsWith("/docs/3.2-beta") ? "v3.2-beta" : "v2.0.0-beta",
|
||||
};
|
||||
});
|
||||
|
@@ -11,11 +11,10 @@ import { Sponsor } from "../components/sponsor";
|
||||
export default async function Page(props: { params: Promise<{ slug?: string[] }> }) {
|
||||
const params = await props.params;
|
||||
const page = source.getPage(params.slug);
|
||||
if (!page) redirect("/docs/3.1-beta");
|
||||
if (!page) redirect("/docs/3.2-beta");
|
||||
|
||||
const MDXContent = page.data.body;
|
||||
|
||||
// Check if this is an older version page that needs a warning
|
||||
const shouldShowWarning = page.url.startsWith("/docs/2.0.0-beta");
|
||||
|
||||
return (
|
||||
@@ -50,7 +49,7 @@ export async function generateStaticParams() {
|
||||
export async function generateMetadata(props: { params: Promise<{ slug?: string[] }> }) {
|
||||
const params = await props.params;
|
||||
const page = source.getPage(params.slug);
|
||||
if (!page) redirect("/docs/3.1-beta");
|
||||
if (!page) redirect("/docs/3.2-beta");
|
||||
|
||||
return {
|
||||
title: page.data.title + " | Palmr. Docs",
|
||||
|
@@ -28,7 +28,7 @@ export default function Layout({ children }: { children: ReactNode }) {
|
||||
<RootProvider
|
||||
search={{
|
||||
options: {
|
||||
defaultTag: "3.1-beta",
|
||||
defaultTag: "3.2-beta",
|
||||
tags: [
|
||||
{
|
||||
name: "v2.0.0 Beta",
|
||||
@@ -36,7 +36,7 @@ export default function Layout({ children }: { children: ReactNode }) {
|
||||
},
|
||||
{
|
||||
name: "v3.0 Beta ✨",
|
||||
value: "3.1-beta",
|
||||
value: "3.2-beta",
|
||||
props: {
|
||||
style: {
|
||||
border: "1px solid rgba(0,165,80,0.2)",
|
||||
|
@@ -6,55 +6,61 @@ const providers = [
|
||||
{
|
||||
name: "Google",
|
||||
description: "Configure authentication using Google OAuth2 services",
|
||||
href: "/docs/3.1-beta/oidc-authentication/google",
|
||||
href: "/docs/3.2-beta/oidc-authentication/google",
|
||||
icon: <Chrome className="w-4 h-4" />,
|
||||
},
|
||||
{
|
||||
name: "Discord",
|
||||
description: "Set up Discord OAuth2 for community-based authentication",
|
||||
href: "/docs/3.1-beta/oidc-authentication/discord",
|
||||
href: "/docs/3.2-beta/oidc-authentication/discord",
|
||||
icon: <MessageSquare className="w-4 h-4" />,
|
||||
},
|
||||
{
|
||||
name: "GitHub",
|
||||
description: "Enable GitHub OAuth for developer-friendly sign-in",
|
||||
href: "/docs/3.1-beta/oidc-authentication/github",
|
||||
href: "/docs/3.2-beta/oidc-authentication/github",
|
||||
icon: <Github className="w-4 h-4" />,
|
||||
},
|
||||
{
|
||||
name: "Zitadel",
|
||||
description: "Enterprise-grade identity and access management",
|
||||
href: "/docs/3.1-beta/oidc-authentication/zitadel",
|
||||
href: "/docs/3.2-beta/oidc-authentication/zitadel",
|
||||
icon: <Shield className="w-4 h-4" />,
|
||||
},
|
||||
{
|
||||
name: "Auth0",
|
||||
description: "Flexible identity platform with extensive customization",
|
||||
href: "/docs/3.1-beta/oidc-authentication/auth0",
|
||||
href: "/docs/3.2-beta/oidc-authentication/auth0",
|
||||
icon: <Lock className="w-4 h-4" />,
|
||||
},
|
||||
{
|
||||
name: "Authentik",
|
||||
description: "Open-source identity provider with modern features",
|
||||
href: "/docs/3.1-beta/oidc-authentication/authentik",
|
||||
href: "/docs/3.2-beta/oidc-authentication/authentik",
|
||||
icon: <Key className="w-4 h-4" />,
|
||||
},
|
||||
{
|
||||
name: "Frontegg",
|
||||
description: "User management platform for B2B applications",
|
||||
href: "/docs/3.1-beta/oidc-authentication/frontegg",
|
||||
href: "/docs/3.2-beta/oidc-authentication/frontegg",
|
||||
icon: <Egg className="w-4 h-4" />,
|
||||
},
|
||||
{
|
||||
name: "Kinde Auth",
|
||||
description: "Developer-first authentication and user management",
|
||||
href: "/docs/3.1-beta/oidc-authentication/kinde-auth",
|
||||
href: "/docs/3.2-beta/oidc-authentication/kinde-auth",
|
||||
icon: <Users className="w-4 h-" />,
|
||||
},
|
||||
{
|
||||
name: "Pocket ID",
|
||||
description: "Open-source identity provider with OIDC support",
|
||||
href: "/docs/3.2-beta/oidc-authentication/pocket-id",
|
||||
icon: <Key className="w-4 h-4" />,
|
||||
},
|
||||
{
|
||||
name: "Other",
|
||||
description: "Configure any other OIDC-compliant identity provider",
|
||||
href: "/docs/3.1-beta/oidc-authentication/other",
|
||||
href: "/docs/3.2-beta/oidc-authentication/other",
|
||||
icon: <Settings className="w-4 h-4" />,
|
||||
},
|
||||
];
|
||||
|
33
apps/docs/src/components/ui/background-lights.tsx
Normal file
@@ -0,0 +1,33 @@
|
||||
import { motion } from "motion/react";
|
||||
|
||||
export function BackgroundLights() {
|
||||
return (
|
||||
<div className="absolute inset-0 -z-10 overflow-hidden">
|
||||
<motion.div
|
||||
animate={{
|
||||
scale: [1, 1.1, 1],
|
||||
opacity: [0.3, 0.5, 0.3],
|
||||
}}
|
||||
className="absolute -top-[20%] -left-[20%] w-[140%] h-[140%] bg-[radial-gradient(circle,rgba(34,197,94,0.15)_0%,transparent_70%)] dark:opacity-100 opacity-50"
|
||||
transition={{
|
||||
duration: 5,
|
||||
repeat: Infinity,
|
||||
ease: "easeInOut",
|
||||
}}
|
||||
/>
|
||||
<motion.div
|
||||
animate={{
|
||||
scale: [1, 1.1, 1],
|
||||
opacity: [0.3, 0.5, 0.3],
|
||||
}}
|
||||
className="absolute -bottom-[20%] -right-[20%] w-[140%] h-[140%] bg-[radial-gradient(circle,rgba(34,197,94,0.15)_0%,transparent_70%)] dark:opacity-100 opacity-50"
|
||||
transition={{
|
||||
duration: 5,
|
||||
repeat: Infinity,
|
||||
ease: "easeInOut",
|
||||
delay: 2.5,
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
@@ -5,14 +5,15 @@ import { cn } from "@/lib/utils";
|
||||
|
||||
interface CardProps {
|
||||
title: string;
|
||||
description: string;
|
||||
description?: string;
|
||||
href?: string;
|
||||
icon?: ReactNode;
|
||||
className?: string;
|
||||
onClick?: () => void;
|
||||
children?: ReactNode;
|
||||
}
|
||||
|
||||
export const Card = ({ title, description, href, icon, className, onClick }: CardProps) => {
|
||||
export const Card = ({ title, description, href, icon, className, onClick, children }: CardProps) => {
|
||||
const cardContent = (
|
||||
<div
|
||||
className={cn(
|
||||
@@ -37,9 +38,16 @@ export const Card = ({ title, description, href, icon, className, onClick }: Car
|
||||
<h3 className="font-medium text-sm text-foreground mb-1 group-hover:text-primary transition-colors duration-200 mt-3 text-decoration-none">
|
||||
{title}
|
||||
</h3>
|
||||
<p className="text-xs text-muted-foreground/80 leading-relaxed line-clamp-2 group-hover:text-muted-foreground transition-colors duration-200">
|
||||
{description}
|
||||
</p>
|
||||
{description && (
|
||||
<p className="text-xs text-muted-foreground/80 leading-relaxed line-clamp-2 group-hover:text-muted-foreground transition-colors duration-200">
|
||||
{description}
|
||||
</p>
|
||||
)}
|
||||
{children && (
|
||||
<div className="text-xs text-muted-foreground/80 leading-relaxed group-hover:text-muted-foreground transition-colors duration-200 mt-2">
|
||||
{children}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
<div className="flex-shrink-0 ml-2">
|
||||
<div className="w-5 h-5 rounded-full bg-muted/40 flex items-center justify-center opacity-0 group-hover:opacity-100 group-hover:bg-primary/10 transition-all duration-200">
|
||||
|
@@ -1,2 +1,2 @@
|
||||
export const LATEST_VERSION_PATH = "/docs/3.1-beta";
|
||||
export const LATEST_VERSION = "v3.1-beta";
|
||||
export const LATEST_VERSION_PATH = "/docs/3.2-beta";
|
||||
export const LATEST_VERSION = "v3.2-beta";
|
||||
|
@@ -1,6 +1,7 @@
|
||||
# FOR FILESYSTEM STORAGE ENV VARS
|
||||
ENABLE_S3=false
|
||||
ENCRYPTION_KEY=change-this-key-in-production-min-32-chars
|
||||
DISABLE_FILESYSTEM_ENCRYPTION=true
|
||||
# ENCRYPTION_KEY=change-this-key-in-production-min-32-chars # Required only if encryption is enabled (DISABLE_FILESYSTEM_ENCRYPTION=false)
|
||||
DATABASE_URL="file:./palmr.db"
|
||||
|
||||
# FOR USE WITH S3 COMPATIBLE STORAGE
|
||||
@@ -13,3 +14,5 @@ DATABASE_URL="file:./palmr.db"
|
||||
# S3_REGION=
|
||||
# S3_BUCKET_NAME=
|
||||
# S3_FORCE_PATH_STYLE=
|
||||
# S3_REJECT_UNAUTHORIZED=true # Set to false to disable strict SSL certificate validation for self-signed certificates (optional, defaults to true)
|
||||
# PRESIGNED_URL_EXPIRATION=3600 # Duration in seconds for presigned URL expiration (optional, defaults to 3600 seconds / 1 hour)
|
||||
|
@@ -1,5 +0,0 @@
|
||||
|
||||
|
||||
> palmr-api@3.1-beta lint /Users/daniel/clones/Palmr/apps/server
|
||||
> eslint "src/**/*.+(ts|tsx)"
|
||||
|
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "palmr-api",
|
||||
"version": "3.1-beta",
|
||||
"version": "3.2.0-beta",
|
||||
"description": "API for Palmr",
|
||||
"private": true,
|
||||
"author": "Daniel Luiz Alves <daniel@kyantech.com.br>",
|
||||
@@ -51,7 +51,9 @@
|
||||
"node-fetch": "^3.3.2",
|
||||
"nodemailer": "^6.10.0",
|
||||
"openid-client": "^6.6.2",
|
||||
"qrcode": "^1.5.4",
|
||||
"sharp": "^0.34.2",
|
||||
"speakeasy": "^2.0.0",
|
||||
"zod": "^3.25.67"
|
||||
},
|
||||
"devDependencies": {
|
||||
@@ -61,6 +63,8 @@
|
||||
"@types/bcryptjs": "^2.4.6",
|
||||
"@types/node": "^22.13.4",
|
||||
"@types/nodemailer": "^6.4.17",
|
||||
"@types/qrcode": "^1.5.5",
|
||||
"@types/speakeasy": "^2.0.10",
|
||||
"@typescript-eslint/eslint-plugin": "8.35.1",
|
||||
"@typescript-eslint/parser": "8.35.1",
|
||||
"eslint": "9.30.0",
|
||||
|
4568
apps/server/pnpm-lock.yaml
generated
@@ -20,6 +20,11 @@ model User {
|
||||
createdAt DateTime @default(now())
|
||||
updatedAt DateTime @updatedAt
|
||||
|
||||
twoFactorEnabled Boolean @default(false)
|
||||
twoFactorSecret String?
|
||||
twoFactorBackupCodes String?
|
||||
twoFactorVerified Boolean @default(false)
|
||||
|
||||
files File[]
|
||||
shares Share[]
|
||||
reverseShares ReverseShare[]
|
||||
@@ -28,6 +33,7 @@ model User {
|
||||
|
||||
passwordResets PasswordReset[]
|
||||
authProviders UserAuthProvider[]
|
||||
trustedDevices TrustedDevice[]
|
||||
|
||||
@@map("users")
|
||||
}
|
||||
@@ -263,3 +269,19 @@ enum PageLayout {
|
||||
DEFAULT
|
||||
WETRANSFER
|
||||
}
|
||||
|
||||
model TrustedDevice {
|
||||
id String @id @default(cuid())
|
||||
userId String
|
||||
user User @relation(fields: [userId], references: [id], onDelete: Cascade)
|
||||
deviceHash String @unique
|
||||
deviceName String?
|
||||
userAgent String?
|
||||
ipAddress String?
|
||||
lastUsedAt DateTime @default(now())
|
||||
expiresAt DateTime
|
||||
createdAt DateTime @default(now())
|
||||
updatedAt DateTime @updatedAt
|
||||
|
||||
@@map("trusted_devices")
|
||||
}
|
||||
|
@@ -147,6 +147,12 @@ const defaultConfigs = [
|
||||
type: "boolean",
|
||||
group: "auth-providers",
|
||||
},
|
||||
{
|
||||
key: "passwordAuthEnabled",
|
||||
value: "true",
|
||||
type: "boolean",
|
||||
group: "security",
|
||||
},
|
||||
{
|
||||
key: "serverUrl",
|
||||
value: "http://localhost:3333",
|
||||
@@ -303,6 +309,24 @@ const defaultAuthProviders = [
|
||||
supportsDiscovery: true,
|
||||
}),
|
||||
},
|
||||
{
|
||||
name: "pocketid",
|
||||
displayName: "Pocket ID",
|
||||
type: "oidc",
|
||||
icon: "BsFillPSquareFill",
|
||||
enabled: false,
|
||||
issuerUrl: "https://your-pocket-id.domain.com",
|
||||
authorizationEndpoint: "/authorize",
|
||||
tokenEndpoint: "/api/oidc/token",
|
||||
userInfoEndpoint: "/api/oidc/userinfo",
|
||||
scope: "openid profile email",
|
||||
sortOrder: 9,
|
||||
metadata: JSON.stringify({
|
||||
description: "Sign in with Pocket ID - Replace with your Pocket ID instance URL",
|
||||
docs: "https://docs.pocket-id.org",
|
||||
supportsDiscovery: true,
|
||||
}),
|
||||
},
|
||||
];
|
||||
|
||||
async function main() {
|
||||
|
@@ -6,7 +6,7 @@
|
||||
echo "🔐 Palmr Password Reset Tool"
|
||||
echo "============================="
|
||||
|
||||
# Check if we're in the right directory
|
||||
# Check if we're in the right directory and set DATABASE_URL
|
||||
if [ ! -f "package.json" ]; then
|
||||
echo "❌ Error: This script must be run from the server directory (/app/server)"
|
||||
echo " Current directory: $(pwd)"
|
||||
@@ -14,18 +14,26 @@ if [ ! -f "package.json" ]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Set DATABASE_URL if not already set
|
||||
if [ -z "$DATABASE_URL" ]; then
|
||||
export DATABASE_URL="file:/app/server/prisma/palmr.db"
|
||||
fi
|
||||
|
||||
# Ensure database directory exists
|
||||
mkdir -p /app/server/prisma
|
||||
|
||||
# Function to check if tsx is available
|
||||
check_tsx() {
|
||||
# Check if tsx binary exists in node_modules
|
||||
if [ -f "node_modules/.bin/tsx" ]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
|
||||
# Fallback: try npx
|
||||
if npx tsx --version >/dev/null 2>&1; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
|
||||
return 1
|
||||
}
|
||||
|
||||
@@ -39,7 +47,7 @@ install_tsx_only() {
|
||||
else
|
||||
return 1
|
||||
fi
|
||||
|
||||
|
||||
return $?
|
||||
}
|
||||
|
||||
@@ -62,7 +70,7 @@ ensure_prisma() {
|
||||
if [ -d "node_modules/@prisma/client" ] && [ -f "node_modules/@prisma/client/index.js" ]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
|
||||
echo "📦 Generating Prisma client..."
|
||||
if npx prisma generate --silent >/dev/null 2>&1; then
|
||||
echo "✅ Prisma client ready"
|
||||
@@ -81,14 +89,14 @@ if check_tsx; then
|
||||
echo "✅ tsx is ready"
|
||||
else
|
||||
echo "📦 tsx not found, installing..."
|
||||
|
||||
|
||||
# Try quick tsx-only install first
|
||||
if install_tsx_only && check_tsx; then
|
||||
echo "✅ tsx installed successfully"
|
||||
else
|
||||
echo "⚠️ Quick install failed, installing all dependencies..."
|
||||
install_all_deps
|
||||
|
||||
|
||||
# Final check
|
||||
if ! check_tsx; then
|
||||
echo "❌ Error: tsx is still not available after full installation"
|
||||
@@ -119,4 +127,4 @@ if [ -f "node_modules/.bin/tsx" ]; then
|
||||
node_modules/.bin/tsx src/scripts/reset-password.ts "$@"
|
||||
else
|
||||
npx tsx src/scripts/reset-password.ts "$@"
|
||||
fi
|
||||
fi
|
@@ -1,4 +1,5 @@
|
||||
import crypto from "node:crypto";
|
||||
import * as http from "node:http";
|
||||
import fastifyCookie from "@fastify/cookie";
|
||||
import { fastifyCors } from "@fastify/cors";
|
||||
import fastifyJwt from "@fastify/jwt";
|
||||
@@ -24,13 +25,38 @@ export async function buildApp() {
|
||||
},
|
||||
},
|
||||
logger: {
|
||||
level: "info",
|
||||
level: "warn",
|
||||
},
|
||||
bodyLimit: 1024 * 1024 * 1024 * 1024 * 1024,
|
||||
connectionTimeout: 0,
|
||||
keepAliveTimeout: envTimeoutOverrides.keepAliveTimeout,
|
||||
requestTimeout: envTimeoutOverrides.requestTimeout,
|
||||
trustProxy: true,
|
||||
maxParamLength: 500,
|
||||
onProtoPoisoning: "ignore",
|
||||
onConstructorPoisoning: "ignore",
|
||||
ignoreTrailingSlash: true,
|
||||
serverFactory: (handler: (req: any, res: any) => void) => {
|
||||
const server = http.createServer((req: http.IncomingMessage, res: http.ServerResponse) => {
|
||||
res.setTimeout(0);
|
||||
req.setTimeout(0);
|
||||
|
||||
req.on("close", () => {
|
||||
if (typeof global !== "undefined" && global.gc) {
|
||||
setImmediate(() => global.gc!());
|
||||
}
|
||||
});
|
||||
|
||||
handler(req, res);
|
||||
});
|
||||
|
||||
server.maxHeadersCount = 0;
|
||||
server.timeout = 0;
|
||||
server.keepAliveTimeout = envTimeoutOverrides.keepAliveTimeout;
|
||||
server.headersTimeout = envTimeoutOverrides.keepAliveTimeout + 1000;
|
||||
|
||||
return server;
|
||||
},
|
||||
}).withTypeProvider<ZodTypeProvider>();
|
||||
|
||||
app.setValidatorCompiler(validatorCompiler);
|
||||
|
@@ -1,3 +1,4 @@
|
||||
import process from "node:process";
|
||||
import { S3Client } from "@aws-sdk/client-s3";
|
||||
|
||||
import { env } from "../env";
|
||||
@@ -14,6 +15,14 @@ export const storageConfig: StorageConfig = {
|
||||
forcePathStyle: env.S3_FORCE_PATH_STYLE === "true",
|
||||
};
|
||||
|
||||
if (storageConfig.useSSL && env.S3_REJECT_UNAUTHORIZED === "false") {
|
||||
const originalRejectUnauthorized = process.env.NODE_TLS_REJECT_UNAUTHORIZED;
|
||||
if (!originalRejectUnauthorized) {
|
||||
process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0";
|
||||
(global as any).PALMR_ORIGINAL_TLS_SETTING = originalRejectUnauthorized;
|
||||
}
|
||||
}
|
||||
|
||||
export const s3Client =
|
||||
env.ENABLE_S3 === "true"
|
||||
? new S3Client({
|
||||
|
@@ -2,7 +2,8 @@ import { z } from "zod";
|
||||
|
||||
const envSchema = z.object({
|
||||
ENABLE_S3: z.union([z.literal("true"), z.literal("false")]).default("false"),
|
||||
ENCRYPTION_KEY: z.string().optional().default("palmr-default-encryption-key-2025"),
|
||||
ENCRYPTION_KEY: z.string().optional(),
|
||||
DISABLE_FILESYSTEM_ENCRYPTION: z.union([z.literal("true"), z.literal("false")]).default("true"),
|
||||
S3_ENDPOINT: z.string().optional(),
|
||||
S3_PORT: z.string().optional(),
|
||||
S3_USE_SSL: z.string().optional(),
|
||||
@@ -11,8 +12,27 @@ const envSchema = z.object({
|
||||
S3_REGION: z.string().optional(),
|
||||
S3_BUCKET_NAME: z.string().optional(),
|
||||
S3_FORCE_PATH_STYLE: z.union([z.literal("true"), z.literal("false")]).default("false"),
|
||||
S3_REJECT_UNAUTHORIZED: z.union([z.literal("true"), z.literal("false")]).default("true"),
|
||||
PRESIGNED_URL_EXPIRATION: z.string().optional().default("3600"),
|
||||
SECURE_SITE: z.union([z.literal("true"), z.literal("false")]).default("false"),
|
||||
DATABASE_URL: z.string().optional().default("file:/app/server/prisma/palmr.db"),
|
||||
DOWNLOAD_MAX_CONCURRENT: z
|
||||
.string()
|
||||
.optional()
|
||||
.transform((val) => (val ? parseInt(val, 10) : undefined)),
|
||||
DOWNLOAD_MEMORY_THRESHOLD_MB: z
|
||||
.string()
|
||||
.optional()
|
||||
.transform((val) => (val ? parseInt(val, 10) : undefined)),
|
||||
DOWNLOAD_QUEUE_SIZE: z
|
||||
.string()
|
||||
.optional()
|
||||
.transform((val) => (val ? parseInt(val, 10) : undefined)),
|
||||
DOWNLOAD_AUTO_SCALE: z.union([z.literal("true"), z.literal("false")]).default("true"),
|
||||
DOWNLOAD_MIN_FILE_SIZE_GB: z
|
||||
.string()
|
||||
.optional()
|
||||
.transform((val) => (val ? parseFloat(val) : undefined)),
|
||||
});
|
||||
|
||||
export const env = envSchema.parse(process.env);
|
||||
|
@@ -9,7 +9,7 @@ export class AppController {
|
||||
private logoService = new LogoService();
|
||||
private emailService = new EmailService();
|
||||
|
||||
async getAppInfo(request: FastifyRequest, reply: FastifyReply) {
|
||||
async getAppInfo(_request: FastifyRequest, reply: FastifyReply) {
|
||||
try {
|
||||
const appInfo = await this.appService.getAppInfo();
|
||||
return reply.send(appInfo);
|
||||
@@ -18,7 +18,16 @@ export class AppController {
|
||||
}
|
||||
}
|
||||
|
||||
async getAllConfigs(request: FastifyRequest, reply: FastifyReply) {
|
||||
async getSystemInfo(_request: FastifyRequest, reply: FastifyReply) {
|
||||
try {
|
||||
const systemInfo = await this.appService.getSystemInfo();
|
||||
return reply.send(systemInfo);
|
||||
} catch (error: any) {
|
||||
return reply.status(400).send({ error: error.message });
|
||||
}
|
||||
}
|
||||
|
||||
async getAllConfigs(_request: FastifyRequest, reply: FastifyReply) {
|
||||
try {
|
||||
const configs = await this.appService.getAllConfigs();
|
||||
return reply.send({ configs });
|
||||
@@ -27,6 +36,15 @@ export class AppController {
|
||||
}
|
||||
}
|
||||
|
||||
async getPublicConfigs(_request: FastifyRequest, reply: FastifyReply) {
|
||||
try {
|
||||
const configs = await this.appService.getPublicConfigs();
|
||||
return reply.send({ configs });
|
||||
} catch (error: any) {
|
||||
return reply.status(400).send({ error: error.message });
|
||||
}
|
||||
}
|
||||
|
||||
async updateConfig(request: FastifyRequest, reply: FastifyReply) {
|
||||
try {
|
||||
const { key } = request.params as { key: string };
|
||||
@@ -81,9 +99,8 @@ export class AppController {
|
||||
return reply.status(400).send({ error: "Only images are allowed" });
|
||||
}
|
||||
|
||||
// Logo files should be small (max 5MB), so we can safely use streaming to buffer
|
||||
const chunks: Buffer[] = [];
|
||||
const maxLogoSize = 5 * 1024 * 1024; // 5MB
|
||||
const maxLogoSize = 5 * 1024 * 1024;
|
||||
let totalSize = 0;
|
||||
|
||||
for await (const chunk of file.file) {
|
||||
@@ -105,7 +122,7 @@ export class AppController {
|
||||
}
|
||||
}
|
||||
|
||||
async removeLogo(request: FastifyRequest, reply: FastifyReply) {
|
||||
async removeLogo(_request: FastifyRequest, reply: FastifyReply) {
|
||||
try {
|
||||
await this.logoService.deleteLogo();
|
||||
return reply.send({ message: "Logo removed successfully" });
|
||||
|
@@ -53,6 +53,26 @@ export async function appRoutes(app: FastifyInstance) {
|
||||
appController.getAppInfo.bind(appController)
|
||||
);
|
||||
|
||||
app.get(
|
||||
"/app/system-info",
|
||||
{
|
||||
schema: {
|
||||
tags: ["App"],
|
||||
operationId: "getSystemInfo",
|
||||
summary: "Get system information",
|
||||
description: "Get system information including storage provider",
|
||||
response: {
|
||||
200: z.object({
|
||||
storageProvider: z.enum(["s3", "filesystem"]).describe("The active storage provider"),
|
||||
s3Enabled: z.boolean().describe("Whether S3 storage is enabled"),
|
||||
}),
|
||||
400: z.object({ error: z.string().describe("Error message") }),
|
||||
},
|
||||
},
|
||||
},
|
||||
appController.getSystemInfo.bind(appController)
|
||||
);
|
||||
|
||||
app.patch(
|
||||
"/app/configs/:key",
|
||||
{
|
||||
@@ -82,15 +102,34 @@ export async function appRoutes(app: FastifyInstance) {
|
||||
appController.updateConfig.bind(appController)
|
||||
);
|
||||
|
||||
app.get(
|
||||
"/app/configs/public",
|
||||
{
|
||||
schema: {
|
||||
tags: ["App"],
|
||||
operationId: "getPublicConfigs",
|
||||
summary: "List public configurations",
|
||||
description: "List public configurations (excludes sensitive data like SMTP credentials)",
|
||||
response: {
|
||||
200: z.object({
|
||||
configs: z.array(ConfigResponseSchema),
|
||||
}),
|
||||
400: z.object({ error: z.string().describe("Error message") }),
|
||||
},
|
||||
},
|
||||
},
|
||||
appController.getPublicConfigs.bind(appController)
|
||||
);
|
||||
|
||||
app.get(
|
||||
"/app/configs",
|
||||
{
|
||||
// preValidation: adminPreValidation,
|
||||
preValidation: adminPreValidation,
|
||||
schema: {
|
||||
tags: ["App"],
|
||||
operationId: "getAllConfigs",
|
||||
summary: "List all configurations",
|
||||
description: "List all configurations (admin only)",
|
||||
description: "List all configurations including sensitive data (admin only)",
|
||||
response: {
|
||||
200: z.object({
|
||||
configs: z.array(ConfigResponseSchema),
|
||||
|
@@ -1,3 +1,4 @@
|
||||
import { isS3Enabled } from "../../config/storage.config";
|
||||
import { prisma } from "../../shared/prisma";
|
||||
import { ConfigService } from "../config/service";
|
||||
|
||||
@@ -20,6 +21,13 @@ export class AppService {
|
||||
};
|
||||
}
|
||||
|
||||
async getSystemInfo() {
|
||||
return {
|
||||
storageProvider: isS3Enabled ? "s3" : "filesystem",
|
||||
s3Enabled: isS3Enabled,
|
||||
};
|
||||
}
|
||||
|
||||
async getAllConfigs() {
|
||||
return prisma.appConfig.findMany({
|
||||
where: {
|
||||
@@ -33,11 +41,46 @@ export class AppService {
|
||||
});
|
||||
}
|
||||
|
||||
async getPublicConfigs() {
|
||||
const sensitiveKeys = [
|
||||
"smtpHost",
|
||||
"smtpPort",
|
||||
"smtpUser",
|
||||
"smtpPass",
|
||||
"smtpSecure",
|
||||
"smtpNoAuth",
|
||||
"smtpTrustSelfSigned",
|
||||
"jwtSecret",
|
||||
];
|
||||
|
||||
return prisma.appConfig.findMany({
|
||||
where: {
|
||||
key: {
|
||||
notIn: sensitiveKeys,
|
||||
},
|
||||
},
|
||||
orderBy: {
|
||||
group: "asc",
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async updateConfig(key: string, value: string) {
|
||||
if (key === "jwtSecret") {
|
||||
throw new Error("JWT Secret cannot be updated through this endpoint");
|
||||
}
|
||||
|
||||
if (key === "passwordAuthEnabled") {
|
||||
if (value === "false") {
|
||||
const canDisable = await this.configService.validatePasswordAuthDisable();
|
||||
if (!canDisable) {
|
||||
throw new Error(
|
||||
"Password authentication cannot be disabled. At least one authentication provider must be active."
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const config = await prisma.appConfig.findUnique({
|
||||
where: { key },
|
||||
});
|
||||
@@ -56,6 +99,15 @@ export class AppService {
|
||||
if (updates.some((update) => update.key === "jwtSecret")) {
|
||||
throw new Error("JWT Secret cannot be updated through this endpoint");
|
||||
}
|
||||
const passwordAuthUpdate = updates.find((update) => update.key === "passwordAuthEnabled");
|
||||
if (passwordAuthUpdate && passwordAuthUpdate.value === "false") {
|
||||
const canDisable = await this.configService.validatePasswordAuthDisable();
|
||||
if (!canDisable) {
|
||||
throw new Error(
|
||||
"Password authentication cannot be disabled. At least one authentication provider must be active."
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const keys = updates.map((update) => update.key);
|
||||
const existingConfigs = await prisma.appConfig.findMany({
|
||||
|
@@ -1,5 +1,6 @@
|
||||
import { FastifyReply, FastifyRequest } from "fastify";
|
||||
|
||||
import { ConfigService } from "../config/service";
|
||||
import { UpdateAuthProviderSchema } from "./dto";
|
||||
import { AuthProvidersService } from "./service";
|
||||
import {
|
||||
@@ -39,9 +40,11 @@ const ERROR_MESSAGES = {
|
||||
|
||||
export class AuthProvidersController {
|
||||
private authProvidersService: AuthProvidersService;
|
||||
private configService: ConfigService;
|
||||
|
||||
constructor() {
|
||||
this.authProvidersService = new AuthProvidersService();
|
||||
this.configService = new ConfigService();
|
||||
}
|
||||
|
||||
private buildRequestContext(request: FastifyRequest): RequestContext {
|
||||
@@ -223,13 +226,24 @@ export class AuthProvidersController {
|
||||
|
||||
try {
|
||||
const { id } = request.params;
|
||||
const data = request.body;
|
||||
const data = request.body as any;
|
||||
|
||||
const existingProvider = await this.authProvidersService.getProviderById(id);
|
||||
if (!existingProvider) {
|
||||
return this.sendErrorResponse(reply, 404, ERROR_MESSAGES.PROVIDER_NOT_FOUND);
|
||||
}
|
||||
|
||||
if (data.enabled === false && existingProvider.enabled === true) {
|
||||
const canDisable = await this.configService.validateAllProvidersDisable();
|
||||
if (!canDisable) {
|
||||
return this.sendErrorResponse(
|
||||
reply,
|
||||
400,
|
||||
"Cannot disable the last authentication provider when password authentication is disabled"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const isOfficial = this.authProvidersService.isOfficialProvider(existingProvider.name);
|
||||
|
||||
if (isOfficial) {
|
||||
@@ -300,6 +314,17 @@ export class AuthProvidersController {
|
||||
return this.sendErrorResponse(reply, 400, ERROR_MESSAGES.OFFICIAL_CANNOT_DELETE);
|
||||
}
|
||||
|
||||
if (provider.enabled) {
|
||||
const canDisable = await this.configService.validateAllProvidersDisable();
|
||||
if (!canDisable) {
|
||||
return this.sendErrorResponse(
|
||||
reply,
|
||||
400,
|
||||
"Cannot delete the last authentication provider when password authentication is disabled"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
await this.authProvidersService.deleteProvider(id);
|
||||
return this.sendSuccessResponse(reply, undefined, "Provider deleted successfully");
|
||||
} catch (error) {
|
||||
|
@@ -13,6 +13,7 @@ export const PROVIDER_PATTERNS = [
|
||||
{ pattern: "okta.com", type: "okta" },
|
||||
{ pattern: "kinde.com", type: "kinde" },
|
||||
{ pattern: "zitadel.com", type: "zitadel" },
|
||||
{ pattern: "pocketid", type: "pocketid" },
|
||||
] as const;
|
||||
|
||||
export const DEFAULT_SCOPES_BY_TYPE: Record<string, string[]> = {
|
||||
@@ -28,6 +29,7 @@ export const DEFAULT_SCOPES_BY_TYPE: Record<string, string[]> = {
|
||||
okta: ["openid", "profile", "email"],
|
||||
kinde: ["openid", "profile", "email"],
|
||||
zitadel: ["openid", "profile", "email"],
|
||||
pocketid: ["openid", "profile", "email"],
|
||||
} as const;
|
||||
|
||||
export const DISCOVERY_SUPPORTED_PROVIDERS = [
|
||||
@@ -41,6 +43,7 @@ export const DISCOVERY_SUPPORTED_PROVIDERS = [
|
||||
"microsoft",
|
||||
"kinde",
|
||||
"zitadel",
|
||||
"pocketid",
|
||||
] as const;
|
||||
|
||||
export const DISCOVERY_PATHS = [
|
||||
@@ -75,6 +78,11 @@ export const FALLBACK_ENDPOINTS: Record<string, any> = {
|
||||
tokenEndpoint: "/oauth2/token",
|
||||
userInfoEndpoint: "/oauth2/userinfo",
|
||||
},
|
||||
pocketid: {
|
||||
authorizationEndpoint: "/authorize",
|
||||
tokenEndpoint: "/api/oidc/token",
|
||||
userInfoEndpoint: "/api/oidc/userinfo",
|
||||
},
|
||||
} as const;
|
||||
|
||||
/**
|
||||
@@ -232,6 +240,29 @@ const fronteggConfig: ProviderConfig = {
|
||||
},
|
||||
};
|
||||
|
||||
/**
|
||||
* Configuração técnica oficial do Pocket ID
|
||||
* OIDC com discovery automático
|
||||
* Endpoints vêm do banco de dados
|
||||
*/
|
||||
const pocketidConfig: ProviderConfig = {
|
||||
supportsDiscovery: true,
|
||||
discoveryEndpoint: "/.well-known/openid-configuration",
|
||||
authMethod: "body",
|
||||
fieldMappings: {
|
||||
id: ["sub"],
|
||||
email: ["email"],
|
||||
name: ["name", "preferred_username"],
|
||||
firstName: ["given_name"],
|
||||
lastName: ["family_name"],
|
||||
avatar: ["picture"],
|
||||
},
|
||||
specialHandling: {
|
||||
emailFetchRequired: false,
|
||||
responseFormat: "json",
|
||||
},
|
||||
};
|
||||
|
||||
/**
|
||||
* Template genérico ULTRA-INTELIGENTE para providers customizados
|
||||
* Detecta automaticamente padrões comuns e se adapta
|
||||
@@ -275,6 +306,7 @@ export const providersConfig: ProvidersConfigFile = {
|
||||
zitadel: zitadelConfig,
|
||||
authentik: authentikConfig,
|
||||
frontegg: fronteggConfig,
|
||||
pocketid: pocketidConfig,
|
||||
},
|
||||
genericProviderTemplate,
|
||||
};
|
||||
|
@@ -18,7 +18,6 @@ import {
|
||||
TokenResponse,
|
||||
} from "./types";
|
||||
|
||||
// Constants
|
||||
const DEFAULT_BASE_URL = "http://localhost:3000";
|
||||
const STATE_EXPIRY_TIME = 600000; // 10 minutes
|
||||
const CLEANUP_INTERVAL = 5 * 60 * 1000; // 5 minutes
|
||||
@@ -43,7 +42,6 @@ export class AuthProvidersService {
|
||||
setInterval(() => this.cleanupExpiredStates(), CLEANUP_INTERVAL);
|
||||
}
|
||||
|
||||
// Utility methods
|
||||
private buildBaseUrl(requestContext?: RequestContextService): string {
|
||||
return requestContext ? `${requestContext.protocol}://${requestContext.host}` : DEFAULT_BASE_URL;
|
||||
}
|
||||
@@ -87,7 +85,6 @@ export class AuthProvidersService {
|
||||
}
|
||||
}
|
||||
|
||||
// Provider configuration methods
|
||||
private isOfficial(providerName: string): boolean {
|
||||
return providerName in providersConfig.officialProviders;
|
||||
}
|
||||
@@ -114,7 +111,6 @@ export class AuthProvidersService {
|
||||
}
|
||||
|
||||
private async resolveEndpoints(provider: any, config: ProviderConfig): Promise<ProviderEndpoints> {
|
||||
// Use custom endpoints if all are provided
|
||||
if (provider.authorizationEndpoint && provider.tokenEndpoint && provider.userInfoEndpoint) {
|
||||
return {
|
||||
authorizationEndpoint: this.resolveEndpointUrl(provider.authorizationEndpoint, provider.issuerUrl),
|
||||
@@ -123,7 +119,6 @@ export class AuthProvidersService {
|
||||
};
|
||||
}
|
||||
|
||||
// Try discovery if supported
|
||||
if (config.supportsDiscovery && provider.issuerUrl) {
|
||||
const discoveredEndpoints = await this.attemptDiscovery(provider.issuerUrl);
|
||||
if (discoveredEndpoints) {
|
||||
@@ -131,7 +126,6 @@ export class AuthProvidersService {
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback to intelligent endpoints
|
||||
const baseUrl = provider.issuerUrl?.replace(/\/$/, "") || "";
|
||||
const detectedType = detectProviderType(provider.issuerUrl || "");
|
||||
const fallbackPattern = getFallbackEndpoints(detectedType);
|
||||
@@ -224,7 +218,6 @@ export class AuthProvidersService {
|
||||
return config.specialHandling?.emailEndpoint || null;
|
||||
}
|
||||
|
||||
// PKCE and OAuth setup methods
|
||||
private setupPkceIfNeeded(provider: any): { codeVerifier?: string; codeChallenge?: string } {
|
||||
const needsPkce = provider.type === DEFAULT_PROVIDER_TYPE;
|
||||
|
||||
@@ -263,7 +256,6 @@ export class AuthProvidersService {
|
||||
return authUrl.toString();
|
||||
}
|
||||
|
||||
// Callback handling methods
|
||||
private validateAndGetPendingState(state: string): PendingState {
|
||||
const pendingState = this.pendingStates.get(state);
|
||||
|
||||
@@ -299,7 +291,6 @@ export class AuthProvidersService {
|
||||
};
|
||||
}
|
||||
|
||||
// Public methods
|
||||
async getEnabledProviders(requestContext?: RequestContextService) {
|
||||
const providers = await prisma.authProvider.findMany({
|
||||
where: { enabled: true },
|
||||
@@ -605,16 +596,13 @@ export class AuthProvidersService {
|
||||
throw new Error(ERROR_MESSAGES.MISSING_USER_INFO);
|
||||
}
|
||||
|
||||
// First, check if there's already an auth provider entry for this external ID
|
||||
const existingAuthProvider = await this.findExistingAuthProvider(provider.id, String(externalId));
|
||||
if (existingAuthProvider) {
|
||||
return await this.updateExistingUserFromProvider(existingAuthProvider.user, userInfo);
|
||||
}
|
||||
|
||||
// Check if there's a user with this email
|
||||
const existingUser = await this.findExistingUserByEmail(userInfo.email);
|
||||
if (existingUser) {
|
||||
// Check if this user already has this provider linked
|
||||
const existingUserProvider = await prisma.userAuthProvider.findFirst({
|
||||
where: {
|
||||
userId: existingUser.id,
|
||||
|
@@ -1,16 +1,70 @@
|
||||
import { FastifyReply, FastifyRequest } from "fastify";
|
||||
|
||||
import { env } from "../../env";
|
||||
import { createResetPasswordSchema, LoginSchema, RequestPasswordResetSchema } from "./dto";
|
||||
import { ConfigService } from "../config/service";
|
||||
import {
|
||||
CompleteTwoFactorLoginSchema,
|
||||
createResetPasswordSchema,
|
||||
LoginSchema,
|
||||
RequestPasswordResetSchema,
|
||||
} from "./dto";
|
||||
import { AuthService } from "./service";
|
||||
|
||||
export class AuthController {
|
||||
private authService = new AuthService();
|
||||
private configService = new ConfigService();
|
||||
|
||||
private getClientInfo(request: FastifyRequest) {
|
||||
const realIP = request.headers["x-real-ip"] as string;
|
||||
const realUserAgent = request.headers["x-user-agent"] as string;
|
||||
|
||||
const userAgent = realUserAgent || request.headers["user-agent"] || "";
|
||||
const ipAddress = realIP || request.ip || request.socket.remoteAddress || "";
|
||||
|
||||
return { userAgent, ipAddress };
|
||||
}
|
||||
|
||||
async login(request: FastifyRequest, reply: FastifyReply) {
|
||||
try {
|
||||
const input = LoginSchema.parse(request.body);
|
||||
const user = await this.authService.login(input);
|
||||
const { userAgent, ipAddress } = this.getClientInfo(request);
|
||||
const result = await this.authService.login(input, userAgent, ipAddress);
|
||||
|
||||
if ("requiresTwoFactor" in result) {
|
||||
return reply.send(result);
|
||||
}
|
||||
|
||||
const user = result;
|
||||
const token = await request.jwtSign({
|
||||
userId: user.id,
|
||||
isAdmin: user.isAdmin,
|
||||
});
|
||||
|
||||
reply.setCookie("token", token, {
|
||||
httpOnly: true,
|
||||
path: "/",
|
||||
secure: env.SECURE_SITE === "true" ? true : false,
|
||||
sameSite: env.SECURE_SITE === "true" ? "lax" : "strict",
|
||||
});
|
||||
|
||||
return reply.send({ user });
|
||||
} catch (error: any) {
|
||||
return reply.status(400).send({ error: error.message });
|
||||
}
|
||||
}
|
||||
|
||||
async completeTwoFactorLogin(request: FastifyRequest, reply: FastifyReply) {
|
||||
try {
|
||||
const input = CompleteTwoFactorLoginSchema.parse(request.body);
|
||||
const { userAgent, ipAddress } = this.getClientInfo(request);
|
||||
const user = await this.authService.completeTwoFactorLogin(
|
||||
input.userId,
|
||||
input.token,
|
||||
input.rememberDevice,
|
||||
userAgent,
|
||||
ipAddress
|
||||
);
|
||||
|
||||
const token = await request.jwtSign({
|
||||
userId: user.id,
|
||||
isAdmin: user.isAdmin,
|
||||
@@ -59,14 +113,21 @@ export class AuthController {
|
||||
|
||||
async getCurrentUser(request: FastifyRequest, reply: FastifyReply) {
|
||||
try {
|
||||
const userId = (request as any).user?.userId;
|
||||
let userId: string | null = null;
|
||||
try {
|
||||
await request.jwtVerify();
|
||||
userId = (request as any).user?.userId;
|
||||
} catch (err) {
|
||||
return reply.send({ user: null });
|
||||
}
|
||||
|
||||
if (!userId) {
|
||||
return reply.status(401).send({ error: "Unauthorized: a valid token is required to access this resource." });
|
||||
return reply.send({ user: null });
|
||||
}
|
||||
|
||||
const user = await this.authService.getUserById(userId);
|
||||
if (!user) {
|
||||
return reply.status(404).send({ error: "User not found" });
|
||||
return reply.send({ user: null });
|
||||
}
|
||||
|
||||
return reply.send({ user });
|
||||
@@ -74,4 +135,58 @@ export class AuthController {
|
||||
return reply.status(400).send({ error: error.message });
|
||||
}
|
||||
}
|
||||
|
||||
async getTrustedDevices(request: FastifyRequest, reply: FastifyReply) {
|
||||
try {
|
||||
const userId = (request as any).user?.userId;
|
||||
if (!userId) {
|
||||
return reply.status(401).send({ error: "Unauthorized: a valid token is required to access this resource." });
|
||||
}
|
||||
|
||||
const devices = await this.authService.getTrustedDevices(userId);
|
||||
return reply.send({ devices });
|
||||
} catch (error: any) {
|
||||
return reply.status(400).send({ error: error.message });
|
||||
}
|
||||
}
|
||||
|
||||
async removeTrustedDevice(request: FastifyRequest, reply: FastifyReply) {
|
||||
try {
|
||||
const userId = (request as any).user?.userId;
|
||||
if (!userId) {
|
||||
return reply.status(401).send({ error: "Unauthorized: a valid token is required to access this resource." });
|
||||
}
|
||||
|
||||
const { id } = request.params as { id: string };
|
||||
await this.authService.removeTrustedDevice(userId, id);
|
||||
return reply.send({ success: true, message: "Trusted device removed successfully" });
|
||||
} catch (error: any) {
|
||||
return reply.status(400).send({ error: error.message });
|
||||
}
|
||||
}
|
||||
|
||||
async removeAllTrustedDevices(request: FastifyRequest, reply: FastifyReply) {
|
||||
try {
|
||||
const userId = (request as any).user?.userId;
|
||||
if (!userId) {
|
||||
return reply.status(401).send({ error: "Unauthorized: a valid token is required to access this resource." });
|
||||
}
|
||||
|
||||
const result = await this.authService.removeAllTrustedDevices(userId);
|
||||
return reply.send(result);
|
||||
} catch (error: any) {
|
||||
return reply.status(400).send({ error: error.message });
|
||||
}
|
||||
}
|
||||
|
||||
async getAuthConfig(request: FastifyRequest, reply: FastifyReply) {
|
||||
try {
|
||||
const passwordAuthEnabled = await this.configService.getValue("passwordAuthEnabled");
|
||||
return reply.send({
|
||||
passwordAuthEnabled: passwordAuthEnabled === "true",
|
||||
});
|
||||
} catch (error: any) {
|
||||
return reply.status(400).send({ error: error.message });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@@ -36,3 +36,11 @@ export const createResetPasswordSchema = async () => {
|
||||
export type ResetPasswordInput = BaseResetPasswordInput & {
|
||||
password: string;
|
||||
};
|
||||
|
||||
export const CompleteTwoFactorLoginSchema = z.object({
|
||||
userId: z.string().min(1, "User ID is required").describe("User ID"),
|
||||
token: z.string().min(6, "Two-factor authentication code must be at least 6 characters").describe("2FA token"),
|
||||
rememberDevice: z.boolean().optional().default(false).describe("Remember this device for 30 days"),
|
||||
});
|
||||
|
||||
export type CompleteTwoFactorLoginInput = z.infer<typeof CompleteTwoFactorLoginSchema>;
|
||||
|
@@ -4,7 +4,7 @@ import { z } from "zod";
|
||||
import { ConfigService } from "../config/service";
|
||||
import { validatePasswordMiddleware } from "../user/middleware";
|
||||
import { AuthController } from "./controller";
|
||||
import { createResetPasswordSchema, RequestPasswordResetSchema } from "./dto";
|
||||
import { CompleteTwoFactorLoginSchema, createResetPasswordSchema, RequestPasswordResetSchema } from "./dto";
|
||||
|
||||
const configService = new ConfigService();
|
||||
|
||||
@@ -31,6 +31,43 @@ export async function authRoutes(app: FastifyInstance) {
|
||||
summary: "Login",
|
||||
description: "Performs login and returns user data",
|
||||
body: loginSchema,
|
||||
response: {
|
||||
200: z.union([
|
||||
z.object({
|
||||
user: z.object({
|
||||
id: z.string().describe("User ID"),
|
||||
firstName: z.string().describe("User first name"),
|
||||
lastName: z.string().describe("User last name"),
|
||||
username: z.string().describe("User username"),
|
||||
email: z.string().email().describe("User email"),
|
||||
isAdmin: z.boolean().describe("User is admin"),
|
||||
isActive: z.boolean().describe("User is active"),
|
||||
createdAt: z.date().describe("User creation date"),
|
||||
updatedAt: z.date().describe("User last update date"),
|
||||
}),
|
||||
}),
|
||||
z.object({
|
||||
requiresTwoFactor: z.boolean().describe("Whether 2FA is required"),
|
||||
userId: z.string().describe("User ID for 2FA verification"),
|
||||
message: z.string().describe("2FA required message"),
|
||||
}),
|
||||
]),
|
||||
400: z.object({ error: z.string().describe("Error message") }),
|
||||
},
|
||||
},
|
||||
},
|
||||
authController.login.bind(authController)
|
||||
);
|
||||
|
||||
app.post(
|
||||
"/auth/2fa/login",
|
||||
{
|
||||
schema: {
|
||||
tags: ["Authentication"],
|
||||
operationId: "completeTwoFactorLogin",
|
||||
summary: "Complete Two-Factor Login",
|
||||
description: "Complete login process with 2FA verification",
|
||||
body: CompleteTwoFactorLoginSchema,
|
||||
response: {
|
||||
200: z.object({
|
||||
user: z.object({
|
||||
@@ -49,7 +86,7 @@ export async function authRoutes(app: FastifyInstance) {
|
||||
},
|
||||
},
|
||||
},
|
||||
authController.login.bind(authController)
|
||||
authController.completeTwoFactorLogin.bind(authController)
|
||||
);
|
||||
|
||||
app.post(
|
||||
@@ -116,21 +153,54 @@ export async function authRoutes(app: FastifyInstance) {
|
||||
tags: ["Authentication"],
|
||||
operationId: "getCurrentUser",
|
||||
summary: "Get Current User",
|
||||
description: "Returns the current authenticated user's information",
|
||||
description: "Returns the current authenticated user's information or null if not authenticated",
|
||||
response: {
|
||||
200: z.union([
|
||||
z.object({
|
||||
user: z.object({
|
||||
id: z.string().describe("User ID"),
|
||||
firstName: z.string().describe("User first name"),
|
||||
lastName: z.string().describe("User last name"),
|
||||
username: z.string().describe("User username"),
|
||||
email: z.string().email().describe("User email"),
|
||||
image: z.string().nullable().describe("User profile image URL"),
|
||||
isAdmin: z.boolean().describe("User is admin"),
|
||||
isActive: z.boolean().describe("User is active"),
|
||||
createdAt: z.date().describe("User creation date"),
|
||||
updatedAt: z.date().describe("User last update date"),
|
||||
}),
|
||||
}),
|
||||
z.object({
|
||||
user: z.null().describe("No user when not authenticated"),
|
||||
}),
|
||||
]),
|
||||
},
|
||||
},
|
||||
},
|
||||
authController.getCurrentUser.bind(authController)
|
||||
);
|
||||
|
||||
app.get(
|
||||
"/auth/trusted-devices",
|
||||
{
|
||||
schema: {
|
||||
tags: ["Authentication"],
|
||||
operationId: "getTrustedDevices",
|
||||
summary: "Get Trusted Devices",
|
||||
description: "Get all trusted devices for the current user",
|
||||
response: {
|
||||
200: z.object({
|
||||
user: z.object({
|
||||
id: z.string().describe("User ID"),
|
||||
firstName: z.string().describe("User first name"),
|
||||
lastName: z.string().describe("User last name"),
|
||||
username: z.string().describe("User username"),
|
||||
email: z.string().email().describe("User email"),
|
||||
image: z.string().nullable().describe("User profile image URL"),
|
||||
isAdmin: z.boolean().describe("User is admin"),
|
||||
isActive: z.boolean().describe("User is active"),
|
||||
createdAt: z.date().describe("User creation date"),
|
||||
updatedAt: z.date().describe("User last update date"),
|
||||
}),
|
||||
devices: z.array(
|
||||
z.object({
|
||||
id: z.string().describe("Device ID"),
|
||||
deviceName: z.string().nullable().describe("Device name"),
|
||||
userAgent: z.string().nullable().describe("User agent"),
|
||||
ipAddress: z.string().nullable().describe("IP address"),
|
||||
createdAt: z.date().describe("Creation date"),
|
||||
lastUsedAt: z.date().describe("Last used date"),
|
||||
expiresAt: z.date().describe("Expiration date"),
|
||||
})
|
||||
),
|
||||
}),
|
||||
401: z.object({ error: z.string().describe("Error message") }),
|
||||
},
|
||||
@@ -144,6 +214,85 @@ export async function authRoutes(app: FastifyInstance) {
|
||||
}
|
||||
},
|
||||
},
|
||||
authController.getCurrentUser.bind(authController)
|
||||
authController.getTrustedDevices.bind(authController)
|
||||
);
|
||||
|
||||
app.delete(
|
||||
"/auth/trusted-devices/:id",
|
||||
{
|
||||
schema: {
|
||||
tags: ["Authentication"],
|
||||
operationId: "removeTrustedDevice",
|
||||
summary: "Remove Trusted Device",
|
||||
description: "Remove a specific trusted device",
|
||||
params: z.object({
|
||||
id: z.string().describe("Device ID"),
|
||||
}),
|
||||
response: {
|
||||
200: z.object({
|
||||
success: z.boolean().describe("Success status"),
|
||||
message: z.string().describe("Success message"),
|
||||
}),
|
||||
401: z.object({ error: z.string().describe("Error message") }),
|
||||
},
|
||||
},
|
||||
preValidation: async (request: FastifyRequest, reply: FastifyReply) => {
|
||||
try {
|
||||
await request.jwtVerify();
|
||||
} catch (err) {
|
||||
console.error(err);
|
||||
reply.status(401).send({ error: "Unauthorized: a valid token is required to access this resource." });
|
||||
}
|
||||
},
|
||||
},
|
||||
authController.removeTrustedDevice.bind(authController)
|
||||
);
|
||||
|
||||
app.delete(
|
||||
"/auth/trusted-devices",
|
||||
{
|
||||
schema: {
|
||||
tags: ["Authentication"],
|
||||
operationId: "removeAllTrustedDevices",
|
||||
summary: "Remove All Trusted Devices",
|
||||
description: "Remove all trusted devices for the current user",
|
||||
response: {
|
||||
200: z.object({
|
||||
success: z.boolean().describe("Success status"),
|
||||
message: z.string().describe("Success message"),
|
||||
removedCount: z.number().describe("Number of devices removed"),
|
||||
}),
|
||||
401: z.object({ error: z.string().describe("Error message") }),
|
||||
},
|
||||
},
|
||||
preValidation: async (request: FastifyRequest, reply: FastifyReply) => {
|
||||
try {
|
||||
await request.jwtVerify();
|
||||
} catch (err) {
|
||||
console.error(err);
|
||||
reply.status(401).send({ error: "Unauthorized: a valid token is required to access this resource." });
|
||||
}
|
||||
},
|
||||
},
|
||||
authController.removeAllTrustedDevices.bind(authController)
|
||||
);
|
||||
|
||||
app.get(
|
||||
"/auth/config",
|
||||
{
|
||||
schema: {
|
||||
tags: ["Authentication"],
|
||||
operationId: "getAuthConfig",
|
||||
summary: "Get Authentication Configuration",
|
||||
description: "Get authentication configuration settings",
|
||||
response: {
|
||||
200: z.object({
|
||||
passwordAuthEnabled: z.boolean().describe("Whether password authentication is enabled"),
|
||||
}),
|
||||
400: z.object({ error: z.string().describe("Error message") }),
|
||||
},
|
||||
},
|
||||
},
|
||||
authController.getAuthConfig.bind(authController)
|
||||
);
|
||||
}
|
||||
|
@@ -4,16 +4,25 @@ import bcrypt from "bcryptjs";
|
||||
import { prisma } from "../../shared/prisma";
|
||||
import { ConfigService } from "../config/service";
|
||||
import { EmailService } from "../email/service";
|
||||
import { TwoFactorService } from "../two-factor/service";
|
||||
import { UserResponseSchema } from "../user/dto";
|
||||
import { PrismaUserRepository } from "../user/repository";
|
||||
import { LoginInput } from "./dto";
|
||||
import { TrustedDeviceService } from "./trusted-device.service";
|
||||
|
||||
export class AuthService {
|
||||
private userRepository = new PrismaUserRepository();
|
||||
private configService = new ConfigService();
|
||||
private emailService = new EmailService();
|
||||
private twoFactorService = new TwoFactorService();
|
||||
private trustedDeviceService = new TrustedDeviceService();
|
||||
|
||||
async login(data: LoginInput, userAgent?: string, ipAddress?: string) {
|
||||
const passwordAuthEnabled = await this.configService.getValue("passwordAuthEnabled");
|
||||
if (passwordAuthEnabled === "false") {
|
||||
throw new Error("Password authentication is disabled. Please use an external authentication provider.");
|
||||
}
|
||||
|
||||
async login(data: LoginInput) {
|
||||
const user = await this.userRepository.findUserByEmailOrUsername(data.emailOrUsername);
|
||||
if (!user) {
|
||||
throw new Error("Invalid credentials");
|
||||
@@ -77,10 +86,76 @@ export class AuthService {
|
||||
});
|
||||
}
|
||||
|
||||
const has2FA = await this.twoFactorService.isEnabled(user.id);
|
||||
|
||||
if (has2FA) {
|
||||
if (userAgent && ipAddress) {
|
||||
const isDeviceTrusted = await this.trustedDeviceService.isDeviceTrusted(user.id, userAgent, ipAddress);
|
||||
if (isDeviceTrusted) {
|
||||
// Update last used timestamp for trusted device
|
||||
await this.trustedDeviceService.updateLastUsed(user.id, userAgent, ipAddress);
|
||||
return UserResponseSchema.parse(user);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
requiresTwoFactor: true,
|
||||
userId: user.id,
|
||||
message: "Two-factor authentication required",
|
||||
};
|
||||
}
|
||||
|
||||
return UserResponseSchema.parse(user);
|
||||
}
|
||||
|
||||
async completeTwoFactorLogin(
|
||||
userId: string,
|
||||
token: string,
|
||||
rememberDevice: boolean = false,
|
||||
userAgent?: string,
|
||||
ipAddress?: string
|
||||
) {
|
||||
const user = await prisma.user.findUnique({
|
||||
where: { id: userId },
|
||||
});
|
||||
|
||||
if (!user) {
|
||||
throw new Error("User not found");
|
||||
}
|
||||
|
||||
if (!user.isActive) {
|
||||
throw new Error("Account is inactive. Please contact an administrator.");
|
||||
}
|
||||
|
||||
const verificationResult = await this.twoFactorService.verifyToken(userId, token);
|
||||
|
||||
if (!verificationResult.success) {
|
||||
throw new Error("Invalid two-factor authentication code");
|
||||
}
|
||||
|
||||
await prisma.loginAttempt.deleteMany({
|
||||
where: { userId },
|
||||
});
|
||||
|
||||
if (rememberDevice && userAgent && ipAddress) {
|
||||
await this.trustedDeviceService.addTrustedDevice(userId, userAgent, ipAddress);
|
||||
} else if (userAgent && ipAddress) {
|
||||
// Update last used timestamp if this is already a trusted device
|
||||
const isDeviceTrusted = await this.trustedDeviceService.isDeviceTrusted(userId, userAgent, ipAddress);
|
||||
if (isDeviceTrusted) {
|
||||
await this.trustedDeviceService.updateLastUsed(userId, userAgent, ipAddress);
|
||||
}
|
||||
}
|
||||
|
||||
return UserResponseSchema.parse(user);
|
||||
}
|
||||
|
||||
async requestPasswordReset(email: string, origin: string) {
|
||||
const passwordAuthEnabled = await this.configService.getValue("passwordAuthEnabled");
|
||||
if (passwordAuthEnabled === "false") {
|
||||
throw new Error("Password authentication is disabled. Password reset is not available.");
|
||||
}
|
||||
|
||||
const user = await this.userRepository.findUserByEmail(email);
|
||||
if (!user) {
|
||||
return;
|
||||
@@ -106,6 +181,11 @@ export class AuthService {
|
||||
}
|
||||
|
||||
async resetPassword(token: string, newPassword: string) {
|
||||
const passwordAuthEnabled = await this.configService.getValue("passwordAuthEnabled");
|
||||
if (passwordAuthEnabled === "false") {
|
||||
throw new Error("Password authentication is disabled. Password reset is not available.");
|
||||
}
|
||||
|
||||
const resetRequest = await prisma.passwordReset.findFirst({
|
||||
where: {
|
||||
token,
|
||||
@@ -146,4 +226,21 @@ export class AuthService {
|
||||
}
|
||||
return UserResponseSchema.parse(user);
|
||||
}
|
||||
|
||||
async getTrustedDevices(userId: string) {
|
||||
return await this.trustedDeviceService.getUserTrustedDevices(userId);
|
||||
}
|
||||
|
||||
async removeTrustedDevice(userId: string, deviceId: string) {
|
||||
return await this.trustedDeviceService.removeTrustedDevice(userId, deviceId);
|
||||
}
|
||||
|
||||
async removeAllTrustedDevices(userId: string) {
|
||||
const result = await this.trustedDeviceService.removeAllTrustedDevices(userId);
|
||||
return {
|
||||
success: true,
|
||||
message: "All trusted devices removed successfully",
|
||||
removedCount: result.count,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
109
apps/server/src/modules/auth/trusted-device.service.ts
Normal file
@@ -0,0 +1,109 @@
|
||||
import crypto from "node:crypto";
|
||||
|
||||
import { prisma } from "../../shared/prisma";
|
||||
|
||||
export class TrustedDeviceService {
|
||||
private generateDeviceHash(userAgent: string, ipAddress: string): string {
|
||||
const deviceInfo = `${userAgent}-${ipAddress}`;
|
||||
return crypto.createHash("sha256").update(deviceInfo).digest("hex");
|
||||
}
|
||||
|
||||
async isDeviceTrusted(userId: string, userAgent: string, ipAddress: string): Promise<boolean> {
|
||||
const deviceHash = this.generateDeviceHash(userAgent, ipAddress);
|
||||
|
||||
const trustedDevice = await prisma.trustedDevice.findFirst({
|
||||
where: {
|
||||
userId,
|
||||
deviceHash,
|
||||
expiresAt: {
|
||||
gt: new Date(),
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
return !!trustedDevice;
|
||||
}
|
||||
|
||||
async addTrustedDevice(userId: string, userAgent: string, ipAddress: string, deviceName?: string): Promise<void> {
|
||||
const deviceHash = this.generateDeviceHash(userAgent, ipAddress);
|
||||
const expiresAt = new Date();
|
||||
expiresAt.setDate(expiresAt.getDate() + 30); // 30 dias
|
||||
|
||||
await prisma.trustedDevice.upsert({
|
||||
where: {
|
||||
deviceHash,
|
||||
},
|
||||
create: {
|
||||
userId,
|
||||
deviceHash,
|
||||
deviceName,
|
||||
userAgent,
|
||||
ipAddress,
|
||||
expiresAt,
|
||||
lastUsedAt: new Date(),
|
||||
},
|
||||
update: {
|
||||
expiresAt,
|
||||
userAgent,
|
||||
ipAddress,
|
||||
lastUsedAt: new Date(),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async cleanupExpiredDevices(): Promise<void> {
|
||||
await prisma.trustedDevice.deleteMany({
|
||||
where: {
|
||||
expiresAt: {
|
||||
lt: new Date(),
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async getUserTrustedDevices(userId: string) {
|
||||
return prisma.trustedDevice.findMany({
|
||||
where: {
|
||||
userId,
|
||||
expiresAt: {
|
||||
gt: new Date(),
|
||||
},
|
||||
},
|
||||
orderBy: {
|
||||
createdAt: "desc",
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async removeTrustedDevice(userId: string, deviceId: string): Promise<void> {
|
||||
await prisma.trustedDevice.deleteMany({
|
||||
where: {
|
||||
id: deviceId,
|
||||
userId,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async removeAllTrustedDevices(userId: string): Promise<{ count: number }> {
|
||||
const result = await prisma.trustedDevice.deleteMany({
|
||||
where: {
|
||||
userId,
|
||||
},
|
||||
});
|
||||
return { count: result.count };
|
||||
}
|
||||
|
||||
async updateLastUsed(userId: string, userAgent: string, ipAddress: string): Promise<void> {
|
||||
const deviceHash = this.generateDeviceHash(userAgent, ipAddress);
|
||||
|
||||
await prisma.trustedDevice.updateMany({
|
||||
where: {
|
||||
userId,
|
||||
deviceHash,
|
||||
},
|
||||
data: {
|
||||
lastUsedAt: new Date(),
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
@@ -13,6 +13,26 @@ export class ConfigService {
|
||||
return config.value;
|
||||
}
|
||||
|
||||
async setValue(key: string, value: string): Promise<void> {
|
||||
await prisma.appConfig.update({
|
||||
where: { key },
|
||||
data: { value },
|
||||
});
|
||||
}
|
||||
|
||||
async validatePasswordAuthDisable(): Promise<boolean> {
|
||||
const enabledProviders = await prisma.authProvider.findMany({
|
||||
where: { enabled: true },
|
||||
});
|
||||
|
||||
return enabledProviders.length > 0;
|
||||
}
|
||||
|
||||
async validateAllProvidersDisable(): Promise<boolean> {
|
||||
const passwordAuthEnabled = await this.getValue("passwordAuthEnabled");
|
||||
return passwordAuthEnabled === "true";
|
||||
}
|
||||
|
||||
async getGroupConfigs(group: string) {
|
||||
const configs = await prisma.appConfig.findMany({
|
||||
where: { group },
|
||||
|
@@ -72,10 +72,8 @@ export class EmailService {
|
||||
let smtpConfig: SmtpConfig;
|
||||
|
||||
if (config) {
|
||||
// Use provided configuration
|
||||
smtpConfig = config;
|
||||
} else {
|
||||
// Fallback to saved configuration
|
||||
smtpConfig = {
|
||||
smtpEnabled: await this.configService.getValue("smtpEnabled"),
|
||||
smtpHost: await this.configService.getValue("smtpHost"),
|
||||
@@ -169,7 +167,7 @@ export class EmailService {
|
||||
});
|
||||
}
|
||||
|
||||
async sendShareNotification(to: string, shareLink: string, shareName?: string) {
|
||||
async sendShareNotification(to: string, shareLink: string, shareName?: string, senderName?: string) {
|
||||
const transporter = await this.createTransporter();
|
||||
if (!transporter) {
|
||||
throw new Error("SMTP is not enabled");
|
||||
@@ -180,19 +178,151 @@ export class EmailService {
|
||||
const appName = await this.configService.getValue("appName");
|
||||
|
||||
const shareTitle = shareName || "Files";
|
||||
const sender = senderName || "Someone";
|
||||
|
||||
await transporter.sendMail({
|
||||
from: `"${fromName}" <${fromEmail}>`,
|
||||
to,
|
||||
subject: `${appName} - ${shareTitle} shared with you`,
|
||||
html: `
|
||||
<h1>${appName} - Shared Files</h1>
|
||||
<p>Someone has shared "${shareTitle}" with you.</p>
|
||||
<p>Click the link below to access the shared files:</p>
|
||||
<a href="${shareLink}">
|
||||
Access Shared Files
|
||||
</a>
|
||||
<p>Note: This share may have an expiration date or view limit.</p>
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>${appName} - Shared Files</title>
|
||||
</head>
|
||||
<body style="margin: 0; padding: 0; font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, 'Helvetica Neue', Arial, sans-serif; background-color: #f5f5f5; color: #333333;">
|
||||
<div style="max-width: 600px; margin: 0 auto; background-color: #ffffff; box-shadow: 0 2px 8px rgba(0, 0, 0, 0.1); overflow: hidden; margin-top: 40px; margin-bottom: 40px;">
|
||||
<!-- Header -->
|
||||
<div style="background-color: #22B14C; padding: 30px 20px; text-align: center;">
|
||||
<h1 style="margin: 0; color: #ffffff; font-size: 28px; font-weight: 600; letter-spacing: -0.5px;">${appName}</h1>
|
||||
<p style="margin: 2px 0 0 0; color: #ffffff; font-size: 16px; opacity: 0.9;">Shared Files</p>
|
||||
</div>
|
||||
|
||||
<!-- Content -->
|
||||
<div style="padding: 40px 30px;">
|
||||
<div style="text-align: center; margin-bottom: 32px;">
|
||||
<h2 style="margin: 0 0 12px 0; color: #1f2937; font-size: 24px; font-weight: 600;">Files Shared With You</h2>
|
||||
<p style="margin: 0; color: #6b7280; font-size: 16px; line-height: 1.6;">
|
||||
<strong style="color: #374151;">${sender}</strong> has shared <strong style="color: #374151;">"${shareTitle}"</strong> with you.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<!-- CTA Button -->
|
||||
<div style="text-align: center; margin: 32px 0;">
|
||||
<a href="${shareLink}" style="display: inline-block; background-color: #22B14C; color: #ffffff; text-decoration: none; padding: 12px 24px; font-weight: 600; font-size: 16px; border: 2px solid #22B14C; border-radius: 8px; transition: all 0.3s ease;">
|
||||
Access Shared Files
|
||||
</a>
|
||||
</div>
|
||||
|
||||
<!-- Info Box -->
|
||||
<div style="background-color: #f9fafb; border-left: 4px solid #22B14C; padding: 16px 20px; margin-top: 32px;">
|
||||
<p style="margin: 0; color: #4b5563; font-size: 14px; line-height: 1.5;">
|
||||
<strong>Important:</strong> This share may have an expiration date or view limit. Access it as soon as possible to ensure availability.
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Footer -->
|
||||
<div style="background-color: #f9fafb; padding: 24px 30px; text-align: center; border-top: 1px solid #e5e7eb;">
|
||||
<p style="margin: 0; color: #6b7280; font-size: 14px;">
|
||||
This email was sent by <strong>${appName}</strong>
|
||||
</p>
|
||||
<p style="margin: 8px 0 0 0; color: #9ca3af; font-size: 12px;">
|
||||
If you didn't expect this email, you can safely ignore it.
|
||||
</p>
|
||||
<p style="margin: 4px 0 0 0; color: #9ca3af; font-size: 10px;">
|
||||
Powered by <a href="https://kyantech.com.br" style="color: #9ca3af; text-decoration: none;">Kyantech Solutions</a>
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
`,
|
||||
});
|
||||
}
|
||||
|
||||
async sendReverseShareBatchFileNotification(
|
||||
recipientEmail: string,
|
||||
reverseShareName: string,
|
||||
fileCount: number,
|
||||
fileList: string,
|
||||
uploaderName: string
|
||||
) {
|
||||
const transporter = await this.createTransporter();
|
||||
if (!transporter) {
|
||||
throw new Error("SMTP is not enabled");
|
||||
}
|
||||
|
||||
const fromName = await this.configService.getValue("smtpFromName");
|
||||
const fromEmail = await this.configService.getValue("smtpFromEmail");
|
||||
const appName = await this.configService.getValue("appName");
|
||||
|
||||
await transporter.sendMail({
|
||||
from: `"${fromName}" <${fromEmail}>`,
|
||||
to: recipientEmail,
|
||||
subject: `${appName} - ${fileCount} file${fileCount > 1 ? "s" : ""} uploaded to "${reverseShareName}"`,
|
||||
html: `
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>${appName} - File Upload Notification</title>
|
||||
</head>
|
||||
<body style="margin: 0; padding: 0; font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, 'Helvetica Neue', Arial, sans-serif; background-color: #f5f5f5; color: #333333;">
|
||||
<div style="max-width: 600px; margin: 0 auto; background-color: #ffffff; box-shadow: 0 2px 8px rgba(0, 0, 0, 0.1); overflow: hidden; margin-top: 40px; margin-bottom: 40px;">
|
||||
<!-- Header -->
|
||||
<div style="background-color: #22B14C; padding: 30px 20px; text-align: center;">
|
||||
<h1 style="margin: 0; color: #ffffff; font-size: 28px; font-weight: 600; letter-spacing: -0.5px;">${appName}</h1>
|
||||
<p style="margin: 2px 0 0 0; color: #ffffff; font-size: 16px; opacity: 0.9;">File Upload Notification</p>
|
||||
</div>
|
||||
|
||||
<!-- Content -->
|
||||
<div style="padding: 40px 30px;">
|
||||
<div style="text-align: center; margin-bottom: 32px;">
|
||||
<h2 style="margin: 0 0 12px 0; color: #1f2937; font-size: 24px; font-weight: 600;">New File Uploaded</h2>
|
||||
<p style="margin: 0; color: #6b7280; font-size: 16px; line-height: 1.6;">
|
||||
<strong style="color: #374151;">${uploaderName}</strong> has uploaded <strong style="color: #374151;">${fileCount} file${fileCount > 1 ? "s" : ""}</strong> to your reverse share <strong style="color: #374151;">"${reverseShareName}"</strong>.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<!-- File List -->
|
||||
<div style="background-color: #f9fafb; border-radius: 8px; padding: 16px; margin: 32px 0; border-left: 4px solid #22B14C;">
|
||||
<p style="margin: 0 0 8px 0; color: #374151; font-size: 14px;"><strong>Files (${fileCount}):</strong></p>
|
||||
<ul style="margin: 0; padding-left: 20px; color: #6b7280; font-size: 14px; line-height: 1.5;">
|
||||
${fileList
|
||||
.split(", ")
|
||||
.map((file) => `<li style="margin: 4px 0;">${file}</li>`)
|
||||
.join("")}
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
<!-- Info Text -->
|
||||
<div style="text-align: center; margin-top: 32px;">
|
||||
<p style="margin: 0; color: #9ca3af; font-size: 12px;">
|
||||
You can now access and manage these files through your dashboard.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
<!-- Footer -->
|
||||
<div style="background-color: #f9fafb; padding: 24px 30px; text-align: center; border-top: 1px solid #e5e7eb;">
|
||||
<p style="margin: 0; color: #6b7280; font-size: 14px;">
|
||||
This email was sent by <strong>${appName}</strong>
|
||||
</p>
|
||||
<p style="margin: 8px 0 0 0; color: #9ca3af; font-size: 12px;">
|
||||
If you didn't expect this email, you can safely ignore it.
|
||||
</p>
|
||||
<p style="margin: 4px 0 0 0; color: #9ca3af; font-size: 10px;">
|
||||
Powered by <a href="https://kyantech.com.br" style="color: #9ca3af; text-decoration: none;">Kyantech Solutions</a>
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
`,
|
||||
});
|
||||
}
|
||||
|
@@ -1,5 +1,6 @@
|
||||
import { FastifyReply, FastifyRequest } from "fastify";
|
||||
|
||||
import { env } from "../../env";
|
||||
import { prisma } from "../../shared/prisma";
|
||||
import { ConfigService } from "../config/service";
|
||||
import { CheckFileInput, CheckFileSchema, RegisterFileInput, RegisterFileSchema, UpdateFileSchema } from "./dto";
|
||||
@@ -27,7 +28,7 @@ export class FileController {
|
||||
}
|
||||
|
||||
const objectName = `${userId}/${Date.now()}-${filename}.${extension}`;
|
||||
const expires = 3600;
|
||||
const expires = parseInt(env.PRESIGNED_URL_EXPIRATION);
|
||||
|
||||
const url = await this.fileService.getPresignedPutUrl(objectName, expires);
|
||||
return reply.send({ url, objectName });
|
||||
@@ -171,7 +172,7 @@ export class FileController {
|
||||
return reply.status(404).send({ error: "File not found." });
|
||||
}
|
||||
const fileName = fileRecord.name;
|
||||
const expires = 3600;
|
||||
const expires = parseInt(env.PRESIGNED_URL_EXPIRATION);
|
||||
const url = await this.fileService.getPresignedGetUrl(objectName, expires, fileName);
|
||||
return reply.send({ url, expiresIn: expires });
|
||||
} catch (error) {
|
||||
|
345
apps/server/src/modules/filesystem/chunk-manager.ts
Normal file
@@ -0,0 +1,345 @@
|
||||
import * as fs from "fs";
|
||||
import * as path from "path";
|
||||
|
||||
import { getTempFilePath } from "../../config/directories.config";
|
||||
import { FilesystemStorageProvider } from "../../providers/filesystem-storage.provider";
|
||||
|
||||
export interface ChunkMetadata {
|
||||
fileId: string;
|
||||
chunkIndex: number;
|
||||
totalChunks: number;
|
||||
chunkSize: number;
|
||||
totalSize: number;
|
||||
fileName: string;
|
||||
isLastChunk: boolean;
|
||||
}
|
||||
|
||||
export interface ChunkInfo {
|
||||
fileId: string;
|
||||
fileName: string;
|
||||
totalSize: number;
|
||||
totalChunks: number;
|
||||
uploadedChunks: Set<number>;
|
||||
tempPath: string;
|
||||
createdAt: number;
|
||||
}
|
||||
|
||||
export class ChunkManager {
|
||||
private static instance: ChunkManager;
|
||||
private activeUploads = new Map<string, ChunkInfo>();
|
||||
private finalizingUploads = new Set<string>(); // Track uploads currently being finalized
|
||||
private cleanupInterval: NodeJS.Timeout;
|
||||
|
||||
private constructor() {
|
||||
// Cleanup expired uploads every 30 minutes
|
||||
this.cleanupInterval = setInterval(
|
||||
() => {
|
||||
this.cleanupExpiredUploads();
|
||||
},
|
||||
30 * 60 * 1000
|
||||
);
|
||||
}
|
||||
|
||||
public static getInstance(): ChunkManager {
|
||||
if (!ChunkManager.instance) {
|
||||
ChunkManager.instance = new ChunkManager();
|
||||
}
|
||||
return ChunkManager.instance;
|
||||
}
|
||||
|
||||
/**
|
||||
* Process a chunk upload with streaming
|
||||
*/
|
||||
async processChunk(
|
||||
metadata: ChunkMetadata,
|
||||
inputStream: NodeJS.ReadableStream,
|
||||
originalObjectName: string
|
||||
): Promise<{ isComplete: boolean; finalPath?: string }> {
|
||||
const startTime = Date.now();
|
||||
const { fileId, chunkIndex, totalChunks, fileName, totalSize, isLastChunk } = metadata;
|
||||
|
||||
console.log(`Processing chunk ${chunkIndex + 1}/${totalChunks} for file ${fileName} (${fileId})`);
|
||||
|
||||
let chunkInfo = this.activeUploads.get(fileId);
|
||||
if (!chunkInfo) {
|
||||
if (chunkIndex !== 0) {
|
||||
throw new Error("First chunk must be chunk 0");
|
||||
}
|
||||
|
||||
const tempPath = getTempFilePath(fileId);
|
||||
chunkInfo = {
|
||||
fileId,
|
||||
fileName,
|
||||
totalSize,
|
||||
totalChunks,
|
||||
uploadedChunks: new Set(),
|
||||
tempPath,
|
||||
createdAt: Date.now(),
|
||||
};
|
||||
this.activeUploads.set(fileId, chunkInfo);
|
||||
console.log(`Created new upload session for ${fileName} at ${tempPath}`);
|
||||
}
|
||||
|
||||
console.log(
|
||||
`Validating chunk ${chunkIndex} (total: ${totalChunks}, uploaded: ${Array.from(chunkInfo.uploadedChunks).join(",")})`
|
||||
);
|
||||
|
||||
if (chunkIndex < 0 || chunkIndex >= totalChunks) {
|
||||
throw new Error(`Invalid chunk index: ${chunkIndex} (must be 0-${totalChunks - 1})`);
|
||||
}
|
||||
|
||||
if (chunkInfo.uploadedChunks.has(chunkIndex)) {
|
||||
console.log(`Chunk ${chunkIndex} already uploaded, treating as success`);
|
||||
|
||||
if (isLastChunk && chunkInfo.uploadedChunks.size === totalChunks) {
|
||||
if (this.finalizingUploads.has(fileId)) {
|
||||
console.log(`Upload ${fileId} is already being finalized, waiting...`);
|
||||
return { isComplete: false };
|
||||
}
|
||||
|
||||
console.log(`All chunks uploaded, finalizing ${fileName}`);
|
||||
return await this.finalizeUpload(chunkInfo, metadata, originalObjectName);
|
||||
}
|
||||
|
||||
return { isComplete: false };
|
||||
}
|
||||
|
||||
const tempDir = path.dirname(chunkInfo.tempPath);
|
||||
await fs.promises.mkdir(tempDir, { recursive: true });
|
||||
console.log(`Temp directory ensured: ${tempDir}`);
|
||||
|
||||
await this.writeChunkToFile(chunkInfo.tempPath, inputStream, chunkIndex === 0);
|
||||
|
||||
chunkInfo.uploadedChunks.add(chunkIndex);
|
||||
|
||||
try {
|
||||
const stats = await fs.promises.stat(chunkInfo.tempPath);
|
||||
const processingTime = Date.now() - startTime;
|
||||
console.log(
|
||||
`Chunk ${chunkIndex + 1}/${totalChunks} uploaded successfully in ${processingTime}ms. Temp file size: ${stats.size} bytes`
|
||||
);
|
||||
} catch (error) {
|
||||
console.warn(`Could not get temp file stats:`, error);
|
||||
}
|
||||
|
||||
console.log(
|
||||
`Checking completion: isLastChunk=${isLastChunk}, uploadedChunks.size=${chunkInfo.uploadedChunks.size}, totalChunks=${totalChunks}`
|
||||
);
|
||||
|
||||
if (isLastChunk && chunkInfo.uploadedChunks.size === totalChunks) {
|
||||
if (this.finalizingUploads.has(fileId)) {
|
||||
console.log(`Upload ${fileId} is already being finalized, waiting...`);
|
||||
return { isComplete: false };
|
||||
}
|
||||
|
||||
console.log(`All chunks uploaded, finalizing ${fileName}`);
|
||||
|
||||
const uploadedChunksArray = Array.from(chunkInfo.uploadedChunks).sort((a, b) => a - b);
|
||||
console.log(`Uploaded chunks in order: ${uploadedChunksArray.join(", ")}`);
|
||||
|
||||
const expectedChunks = Array.from({ length: totalChunks }, (_, i) => i);
|
||||
const missingChunks = expectedChunks.filter((chunk) => !chunkInfo.uploadedChunks.has(chunk));
|
||||
|
||||
if (missingChunks.length > 0) {
|
||||
throw new Error(`Missing chunks: ${missingChunks.join(", ")}`);
|
||||
}
|
||||
|
||||
return await this.finalizeUpload(chunkInfo, metadata, originalObjectName);
|
||||
} else {
|
||||
console.log(
|
||||
`Not ready for finalization: isLastChunk=${isLastChunk}, uploadedChunks.size=${chunkInfo.uploadedChunks.size}, totalChunks=${totalChunks}`
|
||||
);
|
||||
}
|
||||
|
||||
return { isComplete: false };
|
||||
}
|
||||
|
||||
/**
|
||||
* Write chunk to file using streaming
|
||||
*/
|
||||
private async writeChunkToFile(
|
||||
filePath: string,
|
||||
inputStream: NodeJS.ReadableStream,
|
||||
isFirstChunk: boolean
|
||||
): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
console.log(`Writing chunk to ${filePath} (first: ${isFirstChunk})`);
|
||||
|
||||
if (isFirstChunk) {
|
||||
const writeStream = fs.createWriteStream(filePath, {
|
||||
highWaterMark: 64 * 1024 * 1024, // 64MB buffer for better performance
|
||||
});
|
||||
writeStream.on("error", (error) => {
|
||||
console.error("Write stream error:", error);
|
||||
reject(error);
|
||||
});
|
||||
writeStream.on("finish", () => {
|
||||
console.log("Write stream finished successfully");
|
||||
resolve();
|
||||
});
|
||||
inputStream.pipe(writeStream);
|
||||
} else {
|
||||
const writeStream = fs.createWriteStream(filePath, {
|
||||
flags: "a",
|
||||
highWaterMark: 64 * 1024 * 1024, // 64MB buffer for better performance
|
||||
});
|
||||
writeStream.on("error", (error) => {
|
||||
console.error("Write stream error:", error);
|
||||
reject(error);
|
||||
});
|
||||
writeStream.on("finish", () => {
|
||||
console.log("Write stream finished successfully");
|
||||
resolve();
|
||||
});
|
||||
inputStream.pipe(writeStream);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Finalize upload by moving temp file to final location and encrypting (if enabled)
|
||||
*/
|
||||
private async finalizeUpload(
|
||||
chunkInfo: ChunkInfo,
|
||||
metadata: ChunkMetadata,
|
||||
originalObjectName: string
|
||||
): Promise<{ isComplete: boolean; finalPath: string }> {
|
||||
// Mark as finalizing to prevent race conditions
|
||||
this.finalizingUploads.add(chunkInfo.fileId);
|
||||
|
||||
try {
|
||||
console.log(`Finalizing upload for ${chunkInfo.fileName}`);
|
||||
|
||||
const tempStats = await fs.promises.stat(chunkInfo.tempPath);
|
||||
console.log(`Temp file size: ${tempStats.size} bytes, expected: ${chunkInfo.totalSize} bytes`);
|
||||
|
||||
if (tempStats.size !== chunkInfo.totalSize) {
|
||||
console.warn(`Size mismatch! Temp: ${tempStats.size}, Expected: ${chunkInfo.totalSize}`);
|
||||
}
|
||||
|
||||
const provider = FilesystemStorageProvider.getInstance();
|
||||
const finalObjectName = originalObjectName;
|
||||
const filePath = provider.getFilePath(finalObjectName);
|
||||
const dir = path.dirname(filePath);
|
||||
|
||||
console.log(`Starting finalization: ${finalObjectName}`);
|
||||
|
||||
await fs.promises.mkdir(dir, { recursive: true });
|
||||
|
||||
const tempReadStream = fs.createReadStream(chunkInfo.tempPath, {
|
||||
highWaterMark: 64 * 1024 * 1024, // 64MB buffer for better performance
|
||||
});
|
||||
const writeStream = fs.createWriteStream(filePath, {
|
||||
highWaterMark: 64 * 1024 * 1024,
|
||||
});
|
||||
const encryptStream = provider.createEncryptStream();
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const startTime = Date.now();
|
||||
|
||||
tempReadStream
|
||||
.pipe(encryptStream)
|
||||
.pipe(writeStream)
|
||||
.on("finish", () => {
|
||||
const duration = Date.now() - startTime;
|
||||
console.log(`File processed and saved to: ${filePath} in ${duration}ms`);
|
||||
resolve();
|
||||
})
|
||||
.on("error", (error) => {
|
||||
console.error("Error during processing:", error);
|
||||
reject(error);
|
||||
});
|
||||
});
|
||||
|
||||
console.log(`File successfully uploaded and processed: ${finalObjectName}`);
|
||||
|
||||
await this.cleanupTempFile(chunkInfo.tempPath);
|
||||
|
||||
this.activeUploads.delete(chunkInfo.fileId);
|
||||
this.finalizingUploads.delete(chunkInfo.fileId);
|
||||
|
||||
return { isComplete: true, finalPath: finalObjectName };
|
||||
} catch (error) {
|
||||
console.error("Error during finalization:", error);
|
||||
await this.cleanupTempFile(chunkInfo.tempPath);
|
||||
this.activeUploads.delete(chunkInfo.fileId);
|
||||
this.finalizingUploads.delete(chunkInfo.fileId);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Cleanup temporary file
|
||||
*/
|
||||
private async cleanupTempFile(tempPath: string): Promise<void> {
|
||||
try {
|
||||
await fs.promises.access(tempPath);
|
||||
await fs.promises.unlink(tempPath);
|
||||
console.log(`Temp file cleaned up: ${tempPath}`);
|
||||
} catch (error: any) {
|
||||
if (error.code === "ENOENT") {
|
||||
console.log(`Temp file already cleaned up: ${tempPath}`);
|
||||
} else {
|
||||
console.warn(`Failed to cleanup temp file ${tempPath}:`, error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Cleanup expired uploads (older than 2 hours)
|
||||
*/
|
||||
private async cleanupExpiredUploads(): Promise<void> {
|
||||
const now = Date.now();
|
||||
const maxAge = 2 * 60 * 60 * 1000; // 2 hours
|
||||
|
||||
for (const [fileId, chunkInfo] of this.activeUploads.entries()) {
|
||||
if (now - chunkInfo.createdAt > maxAge) {
|
||||
console.log(`Cleaning up expired upload: ${fileId}`);
|
||||
await this.cleanupTempFile(chunkInfo.tempPath);
|
||||
this.activeUploads.delete(fileId);
|
||||
this.finalizingUploads.delete(fileId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get upload progress
|
||||
*/
|
||||
getUploadProgress(fileId: string): { uploaded: number; total: number; percentage: number } | null {
|
||||
const chunkInfo = this.activeUploads.get(fileId);
|
||||
if (!chunkInfo) return null;
|
||||
|
||||
return {
|
||||
uploaded: chunkInfo.uploadedChunks.size,
|
||||
total: chunkInfo.totalChunks,
|
||||
percentage: Math.round((chunkInfo.uploadedChunks.size / chunkInfo.totalChunks) * 100),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Cancel upload
|
||||
*/
|
||||
async cancelUpload(fileId: string): Promise<void> {
|
||||
const chunkInfo = this.activeUploads.get(fileId);
|
||||
if (chunkInfo) {
|
||||
await this.cleanupTempFile(chunkInfo.tempPath);
|
||||
this.activeUploads.delete(fileId);
|
||||
this.finalizingUploads.delete(fileId);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Cleanup on shutdown
|
||||
*/
|
||||
destroy(): void {
|
||||
if (this.cleanupInterval) {
|
||||
clearInterval(this.cleanupInterval);
|
||||
}
|
||||
|
||||
for (const [fileId, chunkInfo] of this.activeUploads.entries()) {
|
||||
this.cleanupTempFile(chunkInfo.tempPath);
|
||||
}
|
||||
this.activeUploads.clear();
|
||||
this.finalizingUploads.clear();
|
||||
}
|
||||
}
|
@@ -1,17 +1,17 @@
|
||||
import * as fs from "fs";
|
||||
import * as path from "path";
|
||||
import { pipeline } from "stream/promises";
|
||||
import { FastifyReply, FastifyRequest } from "fastify";
|
||||
|
||||
import { FilesystemStorageProvider } from "../../providers/filesystem-storage.provider";
|
||||
import { FileService } from "../file/service";
|
||||
import { DownloadCancelResponse, QueueClearResponse, QueueStatusResponse } from "../../types/download-queue";
|
||||
import { DownloadMemoryManager } from "../../utils/download-memory-manager";
|
||||
import { getContentType } from "../../utils/mime-types";
|
||||
import { ChunkManager, ChunkMetadata } from "./chunk-manager";
|
||||
|
||||
export class FilesystemController {
|
||||
private fileService = new FileService();
|
||||
private chunkManager = ChunkManager.getInstance();
|
||||
private memoryManager = DownloadMemoryManager.getInstance();
|
||||
|
||||
/**
|
||||
* Safely encode filename for Content-Disposition header
|
||||
*/
|
||||
private encodeFilenameForHeader(filename: string): string {
|
||||
if (!filename || filename.trim() === "") {
|
||||
return 'attachment; filename="download"';
|
||||
@@ -65,26 +65,114 @@ export class FilesystemController {
|
||||
return reply.status(400).send({ error: "Invalid or expired upload token" });
|
||||
}
|
||||
|
||||
// Use streaming for all files to avoid loading into RAM
|
||||
await this.uploadFileStream(request, provider, tokenData.objectName);
|
||||
const chunkMetadata = this.extractChunkMetadata(request);
|
||||
|
||||
provider.consumeUploadToken(token);
|
||||
reply.status(200).send({ message: "File uploaded successfully" });
|
||||
if (chunkMetadata) {
|
||||
try {
|
||||
const result = await this.handleChunkedUpload(request, chunkMetadata, tokenData.objectName);
|
||||
|
||||
if (result.isComplete) {
|
||||
provider.consumeUploadToken(token);
|
||||
reply.status(200).send({
|
||||
message: "File uploaded successfully",
|
||||
objectName: result.finalPath,
|
||||
finalObjectName: result.finalPath,
|
||||
});
|
||||
} else {
|
||||
reply.status(200).send({
|
||||
message: "Chunk uploaded successfully",
|
||||
progress: this.chunkManager.getUploadProgress(chunkMetadata.fileId),
|
||||
});
|
||||
}
|
||||
} catch (chunkError: any) {
|
||||
return reply.status(400).send({
|
||||
error: chunkError.message || "Chunked upload failed",
|
||||
details: chunkError.toString(),
|
||||
});
|
||||
}
|
||||
} else {
|
||||
await this.uploadFileStream(request, provider, tokenData.objectName);
|
||||
provider.consumeUploadToken(token);
|
||||
reply.status(200).send({ message: "File uploaded successfully" });
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Error in filesystem upload:", error);
|
||||
return reply.status(500).send({ error: "Internal server error" });
|
||||
}
|
||||
}
|
||||
|
||||
private async uploadFileStream(request: FastifyRequest, provider: FilesystemStorageProvider, objectName: string) {
|
||||
// Use the provider's streaming upload method directly
|
||||
await provider.uploadFileFromStream(objectName, request.raw);
|
||||
}
|
||||
|
||||
private extractChunkMetadata(request: FastifyRequest): ChunkMetadata | null {
|
||||
const fileId = request.headers["x-file-id"] as string;
|
||||
const chunkIndex = request.headers["x-chunk-index"] as string;
|
||||
const totalChunks = request.headers["x-total-chunks"] as string;
|
||||
const chunkSize = request.headers["x-chunk-size"] as string;
|
||||
const totalSize = request.headers["x-total-size"] as string;
|
||||
const fileName = request.headers["x-file-name"] as string;
|
||||
const isLastChunk = request.headers["x-is-last-chunk"] as string;
|
||||
|
||||
if (!fileId || !chunkIndex || !totalChunks || !chunkSize || !totalSize || !fileName) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const metadata = {
|
||||
fileId,
|
||||
chunkIndex: parseInt(chunkIndex, 10),
|
||||
totalChunks: parseInt(totalChunks, 10),
|
||||
chunkSize: parseInt(chunkSize, 10),
|
||||
totalSize: parseInt(totalSize, 10),
|
||||
fileName,
|
||||
isLastChunk: isLastChunk === "true",
|
||||
};
|
||||
|
||||
return metadata;
|
||||
}
|
||||
|
||||
private async handleChunkedUpload(request: FastifyRequest, metadata: ChunkMetadata, originalObjectName: string) {
|
||||
const stream = request.raw;
|
||||
|
||||
stream.on("error", (error) => {
|
||||
console.error("Request stream error:", error);
|
||||
});
|
||||
|
||||
return await this.chunkManager.processChunk(metadata, stream, originalObjectName);
|
||||
}
|
||||
|
||||
async getUploadProgress(request: FastifyRequest, reply: FastifyReply) {
|
||||
try {
|
||||
const { fileId } = request.params as { fileId: string };
|
||||
|
||||
const progress = this.chunkManager.getUploadProgress(fileId);
|
||||
|
||||
if (!progress) {
|
||||
return reply.status(404).send({ error: "Upload not found" });
|
||||
}
|
||||
|
||||
reply.status(200).send(progress);
|
||||
} catch (error) {
|
||||
return reply.status(500).send({ error: "Internal server error" });
|
||||
}
|
||||
}
|
||||
|
||||
async cancelUpload(request: FastifyRequest, reply: FastifyReply) {
|
||||
try {
|
||||
const { fileId } = request.params as { fileId: string };
|
||||
|
||||
await this.chunkManager.cancelUpload(fileId);
|
||||
|
||||
reply.status(200).send({ message: "Upload cancelled successfully" });
|
||||
} catch (error) {
|
||||
return reply.status(500).send({ error: "Internal server error" });
|
||||
}
|
||||
}
|
||||
|
||||
async download(request: FastifyRequest, reply: FastifyReply) {
|
||||
const downloadId = `${Date.now()}-${Math.random().toString(36).substring(2, 11)}`;
|
||||
|
||||
try {
|
||||
const { token } = request.params as { token: string };
|
||||
|
||||
const provider = FilesystemStorageProvider.getInstance();
|
||||
|
||||
const tokenData = provider.validateDownloadToken(token);
|
||||
@@ -96,71 +184,233 @@ export class FilesystemController {
|
||||
const filePath = provider.getFilePath(tokenData.objectName);
|
||||
const stats = await fs.promises.stat(filePath);
|
||||
const fileSize = stats.size;
|
||||
const isLargeFile = fileSize > 50 * 1024 * 1024;
|
||||
|
||||
const fileName = tokenData.fileName || "download";
|
||||
|
||||
const fileSizeMB = fileSize / (1024 * 1024);
|
||||
console.log(`[DOWNLOAD] Requesting slot for ${downloadId}: ${tokenData.objectName} (${fileSizeMB.toFixed(1)}MB)`);
|
||||
|
||||
try {
|
||||
await this.memoryManager.requestDownloadSlot(downloadId, {
|
||||
fileName,
|
||||
fileSize,
|
||||
objectName: tokenData.objectName,
|
||||
});
|
||||
} catch (error: any) {
|
||||
console.warn(`[DOWNLOAD] Queue full for ${downloadId}: ${error.message}`);
|
||||
return reply.status(503).send({
|
||||
error: "Download queue is full",
|
||||
message: error.message,
|
||||
retryAfter: 60,
|
||||
});
|
||||
}
|
||||
|
||||
console.log(`[DOWNLOAD] Starting ${downloadId}: ${tokenData.objectName} (${fileSizeMB.toFixed(1)}MB)`);
|
||||
this.memoryManager.startDownload(downloadId);
|
||||
|
||||
const range = request.headers.range;
|
||||
|
||||
reply.header("Content-Disposition", this.encodeFilenameForHeader(fileName));
|
||||
reply.header("Content-Type", "application/octet-stream");
|
||||
reply.header("Content-Type", getContentType(fileName));
|
||||
reply.header("Accept-Ranges", "bytes");
|
||||
reply.header("X-Download-ID", downloadId);
|
||||
|
||||
if (range) {
|
||||
const parts = range.replace(/bytes=/, "").split("-");
|
||||
const start = parseInt(parts[0], 10);
|
||||
const end = parts[1] ? parseInt(parts[1], 10) : fileSize - 1;
|
||||
const chunkSize = end - start + 1;
|
||||
reply.raw.on("close", () => {
|
||||
this.memoryManager.endDownload(downloadId);
|
||||
console.log(`[DOWNLOAD] Client disconnected: ${downloadId}`);
|
||||
});
|
||||
|
||||
reply.status(206);
|
||||
reply.header("Content-Range", `bytes ${start}-${end}/${fileSize}`);
|
||||
reply.header("Content-Length", chunkSize);
|
||||
reply.raw.on("error", () => {
|
||||
this.memoryManager.endDownload(downloadId);
|
||||
console.log(`[DOWNLOAD] Client error: ${downloadId}`);
|
||||
});
|
||||
|
||||
if (isLargeFile) {
|
||||
await this.downloadLargeFileRange(reply, provider, tokenData.objectName, start, end);
|
||||
try {
|
||||
if (range) {
|
||||
const parts = range.replace(/bytes=/, "").split("-");
|
||||
const start = parseInt(parts[0], 10);
|
||||
const end = parts[1] ? parseInt(parts[1], 10) : fileSize - 1;
|
||||
|
||||
reply.status(206);
|
||||
reply.header("Content-Range", `bytes ${start}-${end}/${fileSize}`);
|
||||
reply.header("Content-Length", end - start + 1);
|
||||
|
||||
await this.downloadFileRange(reply, provider, tokenData.objectName, start, end, downloadId);
|
||||
} else {
|
||||
const buffer = await provider.downloadFile(tokenData.objectName);
|
||||
const chunk = buffer.slice(start, end + 1);
|
||||
reply.send(chunk);
|
||||
reply.header("Content-Length", fileSize);
|
||||
await this.downloadFileStream(reply, provider, tokenData.objectName, downloadId);
|
||||
}
|
||||
} else {
|
||||
reply.header("Content-Length", fileSize);
|
||||
|
||||
if (isLargeFile) {
|
||||
await this.downloadLargeFile(reply, provider, filePath);
|
||||
} else {
|
||||
const buffer = await provider.downloadFile(tokenData.objectName);
|
||||
reply.send(buffer);
|
||||
}
|
||||
provider.consumeDownloadToken(token);
|
||||
} finally {
|
||||
this.memoryManager.endDownload(downloadId);
|
||||
}
|
||||
|
||||
provider.consumeDownloadToken(token);
|
||||
} catch (error) {
|
||||
console.error("Error in filesystem download:", error);
|
||||
this.memoryManager.endDownload(downloadId);
|
||||
console.error(`[DOWNLOAD] Error in ${downloadId}:`, error);
|
||||
return reply.status(500).send({ error: "Internal server error" });
|
||||
}
|
||||
}
|
||||
|
||||
private async downloadLargeFile(reply: FastifyReply, provider: FilesystemStorageProvider, filePath: string) {
|
||||
const readStream = fs.createReadStream(filePath);
|
||||
const decryptStream = provider.createDecryptStream();
|
||||
|
||||
private async downloadFileStream(
|
||||
reply: FastifyReply,
|
||||
provider: FilesystemStorageProvider,
|
||||
objectName: string,
|
||||
downloadId?: string
|
||||
) {
|
||||
try {
|
||||
await pipeline(readStream, decryptStream, reply.raw);
|
||||
FilesystemStorageProvider.logMemoryUsage(`Download start: ${objectName} (${downloadId})`);
|
||||
|
||||
const downloadStream = provider.createDownloadStream(objectName);
|
||||
|
||||
downloadStream.on("error", (error) => {
|
||||
console.error("Download stream error:", error);
|
||||
FilesystemStorageProvider.logMemoryUsage(`Download error: ${objectName} (${downloadId})`);
|
||||
if (!reply.sent) {
|
||||
reply.status(500).send({ error: "Download failed" });
|
||||
}
|
||||
});
|
||||
|
||||
reply.raw.on("close", () => {
|
||||
if (downloadStream.readable && typeof (downloadStream as any).destroy === "function") {
|
||||
(downloadStream as any).destroy();
|
||||
}
|
||||
FilesystemStorageProvider.logMemoryUsage(`Download client disconnect: ${objectName} (${downloadId})`);
|
||||
});
|
||||
|
||||
if (this.memoryManager.shouldThrottleStream()) {
|
||||
console.log(
|
||||
`[MEMORY THROTTLE] ${objectName} - Pausing stream due to high memory usage: ${this.memoryManager.getCurrentMemoryUsageMB().toFixed(0)}MB`
|
||||
);
|
||||
|
||||
const { Transform } = require("stream");
|
||||
const memoryManager = this.memoryManager;
|
||||
const throttleStream = new Transform({
|
||||
highWaterMark: 256 * 1024,
|
||||
transform(chunk: Buffer, _encoding: BufferEncoding, callback: (error?: Error | null, data?: any) => void) {
|
||||
if (memoryManager.shouldThrottleStream()) {
|
||||
setImmediate(() => {
|
||||
this.push(chunk);
|
||||
callback();
|
||||
});
|
||||
} else {
|
||||
this.push(chunk);
|
||||
callback();
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
await pipeline(downloadStream, throttleStream, reply.raw);
|
||||
} else {
|
||||
await pipeline(downloadStream, reply.raw);
|
||||
}
|
||||
|
||||
FilesystemStorageProvider.logMemoryUsage(`Download complete: ${objectName} (${downloadId})`);
|
||||
} catch (error) {
|
||||
console.error("Error streaming large file:", error);
|
||||
throw error;
|
||||
console.error("Download error:", error);
|
||||
FilesystemStorageProvider.logMemoryUsage(`Download failed: ${objectName} (${downloadId})`);
|
||||
if (!reply.sent) {
|
||||
reply.status(500).send({ error: "Download failed" });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async downloadLargeFileRange(
|
||||
private async downloadFileRange(
|
||||
reply: FastifyReply,
|
||||
provider: FilesystemStorageProvider,
|
||||
objectName: string,
|
||||
start: number,
|
||||
end: number
|
||||
end: number,
|
||||
downloadId?: string
|
||||
) {
|
||||
const buffer = await provider.downloadFile(objectName);
|
||||
const chunk = buffer.slice(start, end + 1);
|
||||
reply.send(chunk);
|
||||
try {
|
||||
FilesystemStorageProvider.logMemoryUsage(`Range download start: ${objectName} (${start}-${end}) (${downloadId})`);
|
||||
|
||||
const rangeStream = await provider.createDownloadRangeStream(objectName, start, end);
|
||||
|
||||
rangeStream.on("error", (error) => {
|
||||
console.error("Range download stream error:", error);
|
||||
FilesystemStorageProvider.logMemoryUsage(
|
||||
`Range download error: ${objectName} (${start}-${end}) (${downloadId})`
|
||||
);
|
||||
if (!reply.sent) {
|
||||
reply.status(500).send({ error: "Download failed" });
|
||||
}
|
||||
});
|
||||
|
||||
reply.raw.on("close", () => {
|
||||
if (rangeStream.readable && typeof (rangeStream as any).destroy === "function") {
|
||||
(rangeStream as any).destroy();
|
||||
}
|
||||
FilesystemStorageProvider.logMemoryUsage(
|
||||
`Range download client disconnect: ${objectName} (${start}-${end}) (${downloadId})`
|
||||
);
|
||||
});
|
||||
|
||||
await pipeline(rangeStream, reply.raw);
|
||||
|
||||
FilesystemStorageProvider.logMemoryUsage(
|
||||
`Range download complete: ${objectName} (${start}-${end}) (${downloadId})`
|
||||
);
|
||||
} catch (error) {
|
||||
console.error("Range download error:", error);
|
||||
FilesystemStorageProvider.logMemoryUsage(
|
||||
`Range download failed: ${objectName} (${start}-${end}) (${downloadId})`
|
||||
);
|
||||
if (!reply.sent) {
|
||||
reply.status(500).send({ error: "Download failed" });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async getQueueStatus(_request: FastifyRequest, reply: FastifyReply) {
|
||||
try {
|
||||
const queueStatus = this.memoryManager.getQueueStatus();
|
||||
const response: QueueStatusResponse = {
|
||||
status: "success",
|
||||
data: queueStatus,
|
||||
};
|
||||
reply.status(200).send(response);
|
||||
} catch (error) {
|
||||
console.error("Error getting queue status:", error);
|
||||
return reply.status(500).send({ error: "Internal server error" });
|
||||
}
|
||||
}
|
||||
|
||||
async cancelQueuedDownload(request: FastifyRequest, reply: FastifyReply) {
|
||||
try {
|
||||
const { downloadId } = request.params as { downloadId: string };
|
||||
|
||||
const cancelled = this.memoryManager.cancelQueuedDownload(downloadId);
|
||||
|
||||
if (cancelled) {
|
||||
const response: DownloadCancelResponse = {
|
||||
message: "Download cancelled successfully",
|
||||
downloadId,
|
||||
};
|
||||
reply.status(200).send(response);
|
||||
} else {
|
||||
reply.status(404).send({
|
||||
error: "Download not found in queue",
|
||||
downloadId,
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Error cancelling queued download:", error);
|
||||
return reply.status(500).send({ error: "Internal server error" });
|
||||
}
|
||||
}
|
||||
|
||||
async clearDownloadQueue(_request: FastifyRequest, reply: FastifyReply) {
|
||||
try {
|
||||
const clearedCount = this.memoryManager.clearQueue();
|
||||
const response: QueueClearResponse = {
|
||||
message: "Download queue cleared successfully",
|
||||
clearedCount,
|
||||
};
|
||||
reply.status(200).send(response);
|
||||
} catch (error) {
|
||||
console.error("Error clearing download queue:", error);
|
||||
return reply.status(500).send({ error: "Internal server error" });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
95
apps/server/src/modules/filesystem/download-queue-routes.ts
Normal file
@@ -0,0 +1,95 @@
|
||||
import { FastifyInstance } from "fastify";
|
||||
import { z } from "zod";
|
||||
|
||||
import { FilesystemController } from "./controller";
|
||||
|
||||
export async function downloadQueueRoutes(app: FastifyInstance) {
|
||||
const filesystemController = new FilesystemController();
|
||||
|
||||
app.get(
|
||||
"/filesystem/download-queue/status",
|
||||
{
|
||||
schema: {
|
||||
tags: ["Download Queue"],
|
||||
operationId: "getDownloadQueueStatus",
|
||||
summary: "Get download queue status",
|
||||
description: "Get current status of the download queue including active downloads and queue length",
|
||||
response: {
|
||||
200: z.object({
|
||||
status: z.string(),
|
||||
data: z.object({
|
||||
queueLength: z.number(),
|
||||
maxQueueSize: z.number(),
|
||||
activeDownloads: z.number(),
|
||||
maxConcurrent: z.number(),
|
||||
queuedDownloads: z.array(
|
||||
z.object({
|
||||
downloadId: z.string(),
|
||||
position: z.number(),
|
||||
waitTime: z.number(),
|
||||
fileName: z.string().optional(),
|
||||
fileSize: z.number().optional(),
|
||||
})
|
||||
),
|
||||
}),
|
||||
}),
|
||||
500: z.object({
|
||||
error: z.string(),
|
||||
}),
|
||||
},
|
||||
},
|
||||
},
|
||||
filesystemController.getQueueStatus.bind(filesystemController)
|
||||
);
|
||||
|
||||
app.delete(
|
||||
"/filesystem/download-queue/:downloadId",
|
||||
{
|
||||
schema: {
|
||||
tags: ["Download Queue"],
|
||||
operationId: "cancelQueuedDownload",
|
||||
summary: "Cancel a queued download",
|
||||
description: "Cancel a specific download that is waiting in the queue",
|
||||
params: z.object({
|
||||
downloadId: z.string().describe("Download ID"),
|
||||
}),
|
||||
response: {
|
||||
200: z.object({
|
||||
message: z.string(),
|
||||
downloadId: z.string(),
|
||||
}),
|
||||
404: z.object({
|
||||
error: z.string(),
|
||||
downloadId: z.string(),
|
||||
}),
|
||||
500: z.object({
|
||||
error: z.string(),
|
||||
}),
|
||||
},
|
||||
},
|
||||
},
|
||||
filesystemController.cancelQueuedDownload.bind(filesystemController)
|
||||
);
|
||||
|
||||
app.delete(
|
||||
"/filesystem/download-queue",
|
||||
{
|
||||
schema: {
|
||||
tags: ["Download Queue"],
|
||||
operationId: "clearDownloadQueue",
|
||||
summary: "Clear entire download queue",
|
||||
description: "Cancel all downloads waiting in the queue (admin operation)",
|
||||
response: {
|
||||
200: z.object({
|
||||
message: z.string(),
|
||||
clearedCount: z.number(),
|
||||
}),
|
||||
500: z.object({
|
||||
error: z.string(),
|
||||
}),
|
||||
},
|
||||
},
|
||||
},
|
||||
filesystemController.clearDownloadQueue.bind(filesystemController)
|
||||
);
|
||||
}
|
@@ -67,4 +67,57 @@ export async function filesystemRoutes(app: FastifyInstance) {
|
||||
},
|
||||
filesystemController.download.bind(filesystemController)
|
||||
);
|
||||
|
||||
app.get(
|
||||
"/filesystem/upload-progress/:fileId",
|
||||
{
|
||||
schema: {
|
||||
tags: ["Filesystem"],
|
||||
operationId: "getUploadProgress",
|
||||
summary: "Get chunked upload progress",
|
||||
description: "Get the progress of a chunked upload",
|
||||
params: z.object({
|
||||
fileId: z.string().describe("File ID"),
|
||||
}),
|
||||
response: {
|
||||
200: z.object({
|
||||
uploaded: z.number(),
|
||||
total: z.number(),
|
||||
percentage: z.number(),
|
||||
}),
|
||||
404: z.object({
|
||||
error: z.string(),
|
||||
}),
|
||||
500: z.object({
|
||||
error: z.string(),
|
||||
}),
|
||||
},
|
||||
},
|
||||
},
|
||||
filesystemController.getUploadProgress.bind(filesystemController)
|
||||
);
|
||||
|
||||
app.delete(
|
||||
"/filesystem/cancel-upload/:fileId",
|
||||
{
|
||||
schema: {
|
||||
tags: ["Filesystem"],
|
||||
operationId: "cancelUpload",
|
||||
summary: "Cancel chunked upload",
|
||||
description: "Cancel an ongoing chunked upload",
|
||||
params: z.object({
|
||||
fileId: z.string().describe("File ID"),
|
||||
}),
|
||||
response: {
|
||||
200: z.object({
|
||||
message: z.string(),
|
||||
}),
|
||||
500: z.object({
|
||||
error: z.string(),
|
||||
}),
|
||||
},
|
||||
},
|
||||
},
|
||||
filesystemController.cancelUpload.bind(filesystemController)
|
||||
);
|
||||
}
|
||||
|
@@ -2,7 +2,6 @@ import { FastifyReply, FastifyRequest } from "fastify";
|
||||
|
||||
import {
|
||||
CreateReverseShareSchema,
|
||||
GetPresignedUrlSchema,
|
||||
ReverseSharePasswordSchema,
|
||||
UpdateReverseSharePasswordSchema,
|
||||
UpdateReverseShareSchema,
|
||||
@@ -319,8 +318,60 @@ export class ReverseShareController {
|
||||
}
|
||||
|
||||
const { fileId } = request.params as { fileId: string };
|
||||
const result = await this.reverseShareService.downloadReverseShareFile(fileId, userId);
|
||||
return reply.send(result);
|
||||
|
||||
const fileInfo = await this.reverseShareService.getFileInfo(fileId, userId);
|
||||
const downloadId = `reverse-${Date.now()}-${Math.random().toString(36).substring(2, 11)}`;
|
||||
|
||||
const { DownloadMemoryManager } = await import("../../utils/download-memory-manager.js");
|
||||
const memoryManager = DownloadMemoryManager.getInstance();
|
||||
|
||||
const fileSizeMB = Number(fileInfo.size) / (1024 * 1024);
|
||||
console.log(
|
||||
`[REVERSE-DOWNLOAD] Requesting slot for ${downloadId}: ${fileInfo.name} (${fileSizeMB.toFixed(1)}MB)`
|
||||
);
|
||||
|
||||
try {
|
||||
await memoryManager.requestDownloadSlot(downloadId, {
|
||||
fileName: fileInfo.name,
|
||||
fileSize: Number(fileInfo.size),
|
||||
objectName: fileInfo.objectName,
|
||||
});
|
||||
} catch (error: any) {
|
||||
console.warn(`[REVERSE-DOWNLOAD] Queued ${downloadId}: ${error.message}`);
|
||||
return reply.status(202).send({
|
||||
queued: true,
|
||||
downloadId: downloadId,
|
||||
message: "Download queued due to memory constraints",
|
||||
estimatedWaitTime: error.estimatedWaitTime || 60,
|
||||
});
|
||||
}
|
||||
|
||||
console.log(`[REVERSE-DOWNLOAD] Starting ${downloadId}: ${fileInfo.name} (${fileSizeMB.toFixed(1)}MB)`);
|
||||
memoryManager.startDownload(downloadId);
|
||||
|
||||
try {
|
||||
const result = await this.reverseShareService.downloadReverseShareFile(fileId, userId);
|
||||
|
||||
const originalUrl = result.url;
|
||||
reply.header("X-Download-ID", downloadId);
|
||||
|
||||
reply.raw.on("finish", () => {
|
||||
memoryManager.endDownload(downloadId);
|
||||
});
|
||||
|
||||
reply.raw.on("close", () => {
|
||||
memoryManager.endDownload(downloadId);
|
||||
});
|
||||
|
||||
reply.raw.on("error", () => {
|
||||
memoryManager.endDownload(downloadId);
|
||||
});
|
||||
|
||||
return reply.send(result);
|
||||
} catch (downloadError) {
|
||||
memoryManager.endDownload(downloadId);
|
||||
throw downloadError;
|
||||
}
|
||||
} catch (error: any) {
|
||||
if (error.message === "File not found") {
|
||||
return reply.status(404).send({ error: error.message });
|
||||
@@ -454,6 +505,7 @@ export class ReverseShareController {
|
||||
async copyFileToUserFiles(request: FastifyRequest, reply: FastifyReply) {
|
||||
try {
|
||||
await request.jwtVerify();
|
||||
|
||||
const { fileId } = request.params as { fileId: string };
|
||||
const userId = (request as any).user?.userId;
|
||||
|
||||
@@ -461,9 +513,16 @@ export class ReverseShareController {
|
||||
return reply.status(401).send({ error: "Unauthorized" });
|
||||
}
|
||||
|
||||
console.log(`Copy to my files: User ${userId} copying file ${fileId}`);
|
||||
|
||||
const file = await this.reverseShareService.copyReverseShareFileToUserFiles(fileId, userId);
|
||||
|
||||
console.log(`Copy to my files: Successfully copied file ${fileId}`);
|
||||
|
||||
return reply.send({ file, message: "File copied to your files successfully" });
|
||||
} catch (error: any) {
|
||||
console.error(`Copy to my files: Error:`, error.message);
|
||||
|
||||
if (error.message === "File not found") {
|
||||
return reply.status(404).send({ error: "File not found" });
|
||||
}
|
||||
|
@@ -401,6 +401,12 @@ export async function reverseShareRoutes(app: FastifyInstance) {
|
||||
url: z.string().describe("Presigned download URL - expires after 1 hour"),
|
||||
expiresIn: z.number().describe("URL expiration time in seconds (3600 = 1 hour)"),
|
||||
}),
|
||||
202: z.object({
|
||||
queued: z.boolean().describe("Download was queued due to memory constraints"),
|
||||
downloadId: z.string().describe("Download identifier for tracking"),
|
||||
message: z.string().describe("Queue status message"),
|
||||
estimatedWaitTime: z.number().describe("Estimated wait time in seconds"),
|
||||
}),
|
||||
401: z.object({ error: z.string() }),
|
||||
404: z.object({ error: z.string() }),
|
||||
},
|
||||
|
@@ -1,6 +1,9 @@
|
||||
import { PrismaClient } from "@prisma/client";
|
||||
|
||||
import { env } from "../../env";
|
||||
import { EmailService } from "../email/service";
|
||||
import { FileService } from "../file/service";
|
||||
import { UserService } from "../user/service";
|
||||
import {
|
||||
CreateReverseShareInput,
|
||||
ReverseShareResponseSchema,
|
||||
@@ -40,6 +43,19 @@ const prisma = new PrismaClient();
|
||||
export class ReverseShareService {
|
||||
private reverseShareRepository = new ReverseShareRepository();
|
||||
private fileService = new FileService();
|
||||
private emailService = new EmailService();
|
||||
private userService = new UserService();
|
||||
|
||||
private uploadSessions = new Map<
|
||||
string,
|
||||
{
|
||||
reverseShareId: string;
|
||||
uploaderName: string;
|
||||
uploaderEmail?: string;
|
||||
files: string[];
|
||||
timeout: NodeJS.Timeout;
|
||||
}
|
||||
>();
|
||||
|
||||
async createReverseShare(data: CreateReverseShareInput, creatorId: string) {
|
||||
const reverseShare = await this.reverseShareRepository.create(data, creatorId);
|
||||
@@ -211,7 +227,7 @@ export class ReverseShareService {
|
||||
}
|
||||
}
|
||||
|
||||
const expires = 3600; // 1 hour
|
||||
const expires = parseInt(env.PRESIGNED_URL_EXPIRATION);
|
||||
const url = await this.fileService.getPresignedPutUrl(objectName, expires);
|
||||
|
||||
return { url, expiresIn: expires };
|
||||
@@ -241,7 +257,7 @@ export class ReverseShareService {
|
||||
}
|
||||
}
|
||||
|
||||
const expires = 3600; // 1 hour
|
||||
const expires = parseInt(env.PRESIGNED_URL_EXPIRATION);
|
||||
const url = await this.fileService.getPresignedPutUrl(objectName, expires);
|
||||
|
||||
return { url, expiresIn: expires };
|
||||
@@ -294,6 +310,8 @@ export class ReverseShareService {
|
||||
size: BigInt(fileData.size),
|
||||
});
|
||||
|
||||
this.addFileToUploadSession(reverseShare, fileData);
|
||||
|
||||
return this.formatFileResponse(file);
|
||||
}
|
||||
|
||||
@@ -344,9 +362,30 @@ export class ReverseShareService {
|
||||
size: BigInt(fileData.size),
|
||||
});
|
||||
|
||||
this.addFileToUploadSession(reverseShare, fileData);
|
||||
|
||||
return this.formatFileResponse(file);
|
||||
}
|
||||
|
||||
async getFileInfo(fileId: string, creatorId: string) {
|
||||
const file = await this.reverseShareRepository.findFileById(fileId);
|
||||
if (!file) {
|
||||
throw new Error("File not found");
|
||||
}
|
||||
|
||||
if (file.reverseShare.creatorId !== creatorId) {
|
||||
throw new Error("Unauthorized to access this file");
|
||||
}
|
||||
|
||||
return {
|
||||
id: file.id,
|
||||
name: file.name,
|
||||
size: file.size,
|
||||
objectName: file.objectName,
|
||||
extension: file.extension,
|
||||
};
|
||||
}
|
||||
|
||||
async downloadReverseShareFile(fileId: string, creatorId: string) {
|
||||
const file = await this.reverseShareRepository.findFileById(fileId);
|
||||
if (!file) {
|
||||
@@ -358,7 +397,7 @@ export class ReverseShareService {
|
||||
}
|
||||
|
||||
const fileName = file.name;
|
||||
const expires = 3600; // 1 hour
|
||||
const expires = parseInt(env.PRESIGNED_URL_EXPIRATION);
|
||||
const url = await this.fileService.getPresignedGetUrl(file.objectName, expires, fileName);
|
||||
return { url, expiresIn: expires };
|
||||
}
|
||||
@@ -533,46 +572,72 @@ export class ReverseShareService {
|
||||
const { FilesystemStorageProvider } = await import("../../providers/filesystem-storage.provider.js");
|
||||
const provider = FilesystemStorageProvider.getInstance();
|
||||
|
||||
// Use streaming copy for filesystem mode
|
||||
const sourcePath = provider.getFilePath(file.objectName);
|
||||
const fs = await import("fs");
|
||||
const { pipeline } = await import("stream/promises");
|
||||
|
||||
const sourceStream = fs.createReadStream(sourcePath);
|
||||
const decryptStream = provider.createDecryptStream();
|
||||
const targetPath = provider.getFilePath(newObjectName);
|
||||
|
||||
// Create a passthrough stream to get the decrypted content
|
||||
const { PassThrough } = await import("stream");
|
||||
const passThrough = new PassThrough();
|
||||
const path = await import("path");
|
||||
const targetDir = path.dirname(targetPath);
|
||||
if (!fs.existsSync(targetDir)) {
|
||||
fs.mkdirSync(targetDir, { recursive: true });
|
||||
}
|
||||
|
||||
// First, decrypt the source file into the passthrough stream
|
||||
await pipeline(sourceStream, decryptStream, passThrough);
|
||||
|
||||
// Then upload the decrypted content
|
||||
await provider.uploadFileFromStream(newObjectName, passThrough);
|
||||
const { copyFile } = await import("fs/promises");
|
||||
await copyFile(sourcePath, targetPath);
|
||||
} else {
|
||||
const fileSizeMB = Number(file.size) / (1024 * 1024);
|
||||
const needsStreaming = fileSizeMB > 100;
|
||||
|
||||
const downloadUrl = await this.fileService.getPresignedGetUrl(file.objectName, 300);
|
||||
const uploadUrl = await this.fileService.getPresignedPutUrl(newObjectName, 300);
|
||||
|
||||
const response = await fetch(downloadUrl);
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to download file: ${response.statusText}`);
|
||||
}
|
||||
let retries = 0;
|
||||
const maxRetries = 3;
|
||||
let success = false;
|
||||
|
||||
if (!response.body) {
|
||||
throw new Error("No response body received");
|
||||
}
|
||||
while (retries < maxRetries && !success) {
|
||||
try {
|
||||
const response = await fetch(downloadUrl, {
|
||||
signal: AbortSignal.timeout(600000), // 10 minutes timeout
|
||||
});
|
||||
|
||||
const uploadResponse = await fetch(uploadUrl, {
|
||||
method: "PUT",
|
||||
body: response.body,
|
||||
headers: {
|
||||
"Content-Type": "application/octet-stream",
|
||||
},
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to download file: ${response.statusText}`);
|
||||
}
|
||||
|
||||
if (!uploadResponse.ok) {
|
||||
throw new Error(`Failed to upload file: ${uploadResponse.statusText}`);
|
||||
if (!response.body) {
|
||||
throw new Error("No response body received");
|
||||
}
|
||||
|
||||
const uploadOptions: any = {
|
||||
method: "PUT",
|
||||
body: response.body,
|
||||
headers: {
|
||||
"Content-Type": "application/octet-stream",
|
||||
"Content-Length": file.size.toString(),
|
||||
},
|
||||
signal: AbortSignal.timeout(600000), // 10 minutes timeout
|
||||
};
|
||||
|
||||
const uploadResponse = await fetch(uploadUrl, uploadOptions);
|
||||
|
||||
if (!uploadResponse.ok) {
|
||||
const errorText = await uploadResponse.text();
|
||||
throw new Error(`Failed to upload file: ${uploadResponse.statusText} - ${errorText}`);
|
||||
}
|
||||
|
||||
success = true;
|
||||
} catch (error: any) {
|
||||
retries++;
|
||||
|
||||
if (retries >= maxRetries) {
|
||||
throw new Error(`Failed to copy file after ${maxRetries} attempts: ${error.message}`);
|
||||
}
|
||||
|
||||
const delay = Math.min(1000 * Math.pow(2, retries - 1), 10000);
|
||||
await new Promise((resolve) => setTimeout(resolve, delay));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -600,6 +665,55 @@ export class ReverseShareService {
|
||||
};
|
||||
}
|
||||
|
||||
private generateSessionKey(reverseShareId: string, uploaderIdentifier: string): string {
|
||||
return `${reverseShareId}-${uploaderIdentifier}`;
|
||||
}
|
||||
|
||||
private async sendBatchFileUploadNotification(reverseShare: any, uploaderName: string, fileNames: string[]) {
|
||||
try {
|
||||
const creator = await this.userService.getUserById(reverseShare.creatorId);
|
||||
const reverseShareName = reverseShare.name || "Unnamed Reverse Share";
|
||||
const fileCount = fileNames.length;
|
||||
const fileList = fileNames.join(", ");
|
||||
|
||||
await this.emailService.sendReverseShareBatchFileNotification(
|
||||
creator.email,
|
||||
reverseShareName,
|
||||
fileCount,
|
||||
fileList,
|
||||
uploaderName
|
||||
);
|
||||
} catch (error) {
|
||||
console.error("Failed to send reverse share batch file notification:", error);
|
||||
}
|
||||
}
|
||||
|
||||
private addFileToUploadSession(reverseShare: any, fileData: UploadToReverseShareInput) {
|
||||
const uploaderIdentifier = fileData.uploaderEmail || fileData.uploaderName || "anonymous";
|
||||
const sessionKey = this.generateSessionKey(reverseShare.id, uploaderIdentifier);
|
||||
const uploaderName = fileData.uploaderName || "Someone";
|
||||
|
||||
const existingSession = this.uploadSessions.get(sessionKey);
|
||||
if (existingSession) {
|
||||
clearTimeout(existingSession.timeout);
|
||||
existingSession.files.push(fileData.name);
|
||||
} else {
|
||||
this.uploadSessions.set(sessionKey, {
|
||||
reverseShareId: reverseShare.id,
|
||||
uploaderName,
|
||||
uploaderEmail: fileData.uploaderEmail,
|
||||
files: [fileData.name],
|
||||
timeout: null as any,
|
||||
});
|
||||
}
|
||||
|
||||
const session = this.uploadSessions.get(sessionKey)!;
|
||||
session.timeout = setTimeout(async () => {
|
||||
await this.sendBatchFileUploadNotification(reverseShare, session.uploaderName, session.files);
|
||||
this.uploadSessions.delete(sessionKey);
|
||||
}, 5000);
|
||||
}
|
||||
|
||||
private formatReverseShareResponse(reverseShare: ReverseShareData) {
|
||||
const result = {
|
||||
id: reverseShare.id,
|
||||
|
@@ -2,6 +2,7 @@ import bcrypt from "bcryptjs";
|
||||
|
||||
import { prisma } from "../../shared/prisma";
|
||||
import { EmailService } from "../email/service";
|
||||
import { UserService } from "../user/service";
|
||||
import { CreateShareInput, ShareResponseSchema, UpdateShareInput } from "./dto";
|
||||
import { IShareRepository, PrismaShareRepository } from "./repository";
|
||||
|
||||
@@ -9,6 +10,7 @@ export class ShareService {
|
||||
constructor(private readonly shareRepository: IShareRepository = new PrismaShareRepository()) {}
|
||||
|
||||
private emailService = new EmailService();
|
||||
private userService = new UserService();
|
||||
|
||||
private formatShareResponse(share: any) {
|
||||
return {
|
||||
@@ -339,11 +341,26 @@ export class ShareService {
|
||||
throw new Error("No recipients found for this share");
|
||||
}
|
||||
|
||||
// Get sender information
|
||||
let senderName = "Someone";
|
||||
try {
|
||||
const sender = await this.userService.getUserById(userId);
|
||||
if (sender.firstName && sender.lastName) {
|
||||
senderName = `${sender.firstName} ${sender.lastName}`;
|
||||
} else if (sender.firstName) {
|
||||
senderName = sender.firstName;
|
||||
} else if (sender.username) {
|
||||
senderName = sender.username;
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Failed to get sender information for user ${userId}:`, error);
|
||||
}
|
||||
|
||||
const notifiedRecipients: string[] = [];
|
||||
|
||||
for (const recipient of share.recipients) {
|
||||
try {
|
||||
await this.emailService.sendShareNotification(recipient.email, shareLink, share.name || undefined);
|
||||
await this.emailService.sendShareNotification(recipient.email, shareLink, share.name || undefined, senderName);
|
||||
notifiedRecipients.push(recipient.email);
|
||||
} catch (error) {
|
||||
console.error(`Failed to send email to ${recipient.email}:`, error);
|
||||
|
@@ -3,6 +3,7 @@ import fs from "node:fs";
|
||||
import { promisify } from "util";
|
||||
import { PrismaClient } from "@prisma/client";
|
||||
|
||||
import { env } from "../../env";
|
||||
import { IS_RUNNING_IN_CONTAINER } from "../../utils/container-detection";
|
||||
import { ConfigService } from "../config/service";
|
||||
|
||||
@@ -21,6 +22,30 @@ export class StorageService {
|
||||
return Number.isNaN(parsed) ? 0 : parsed;
|
||||
}
|
||||
|
||||
private _parseSize(value: string): number {
|
||||
if (!value) return 0;
|
||||
|
||||
const cleanValue = value.trim().toLowerCase();
|
||||
|
||||
const numericMatch = cleanValue.match(/^(\d+(?:\.\d+)?)/);
|
||||
if (!numericMatch) return 0;
|
||||
|
||||
const numericValue = parseFloat(numericMatch[1]);
|
||||
if (Number.isNaN(numericValue)) return 0;
|
||||
|
||||
if (cleanValue.includes("t")) {
|
||||
return Math.round(numericValue * 1024 * 1024 * 1024 * 1024);
|
||||
} else if (cleanValue.includes("g")) {
|
||||
return Math.round(numericValue * 1024 * 1024 * 1024);
|
||||
} else if (cleanValue.includes("m")) {
|
||||
return Math.round(numericValue * 1024 * 1024);
|
||||
} else if (cleanValue.includes("k")) {
|
||||
return Math.round(numericValue * 1024);
|
||||
} else {
|
||||
return Math.round(numericValue);
|
||||
}
|
||||
}
|
||||
|
||||
private async _tryDiskSpaceCommand(command: string): Promise<{ total: number; available: number } | null> {
|
||||
try {
|
||||
const { stdout, stderr } = await execAsync(command);
|
||||
@@ -54,16 +79,61 @@ export class StorageService {
|
||||
}
|
||||
} else {
|
||||
const lines = stdout.trim().split("\n");
|
||||
if (lines.length >= 2) {
|
||||
const parts = lines[1].trim().split(/\s+/);
|
||||
if (parts.length >= 4) {
|
||||
const [, size, , avail] = parts;
|
||||
if (command.includes("-B1")) {
|
||||
total = this._safeParseInt(size);
|
||||
available = this._safeParseInt(avail);
|
||||
} else {
|
||||
total = this._safeParseInt(size) * 1024;
|
||||
available = this._safeParseInt(avail) * 1024;
|
||||
|
||||
if (command.includes("findmnt")) {
|
||||
if (lines.length >= 1) {
|
||||
const parts = lines[0].trim().split(/\s+/);
|
||||
if (parts.length >= 2) {
|
||||
const [availStr, sizeStr] = parts;
|
||||
available = this._parseSize(availStr);
|
||||
total = this._parseSize(sizeStr);
|
||||
}
|
||||
}
|
||||
} else if (command.includes("stat -f")) {
|
||||
let blockSize = 0;
|
||||
let totalBlocks = 0;
|
||||
let freeBlocks = 0;
|
||||
|
||||
for (const line of lines) {
|
||||
if (line.includes("Block size:")) {
|
||||
blockSize = this._safeParseInt(line.split(":")[1].trim());
|
||||
} else if (line.includes("Total blocks:")) {
|
||||
totalBlocks = this._safeParseInt(line.split(":")[1].trim());
|
||||
} else if (line.includes("Free blocks:")) {
|
||||
freeBlocks = this._safeParseInt(line.split(":")[1].trim());
|
||||
}
|
||||
}
|
||||
|
||||
if (blockSize > 0 && totalBlocks > 0) {
|
||||
total = totalBlocks * blockSize;
|
||||
available = freeBlocks * blockSize;
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
} else if (command.includes("--output=")) {
|
||||
if (lines.length >= 2) {
|
||||
const parts = lines[1].trim().split(/\s+/);
|
||||
if (parts.length >= 2) {
|
||||
const [availStr, sizeStr] = parts;
|
||||
available = this._safeParseInt(availStr) * 1024;
|
||||
total = this._safeParseInt(sizeStr) * 1024;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if (lines.length >= 2) {
|
||||
const parts = lines[1].trim().split(/\s+/);
|
||||
if (parts.length >= 4) {
|
||||
const [, size, , avail] = parts;
|
||||
if (command.includes("-B1")) {
|
||||
total = this._safeParseInt(size);
|
||||
available = this._safeParseInt(avail);
|
||||
} else if (command.includes("-h")) {
|
||||
total = this._parseSize(size);
|
||||
available = this._parseSize(avail);
|
||||
} else {
|
||||
total = this._safeParseInt(size) * 1024;
|
||||
available = this._safeParseInt(avail) * 1024;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -72,18 +142,13 @@ export class StorageService {
|
||||
if (total > 0 && available >= 0) {
|
||||
return { total, available };
|
||||
} else {
|
||||
console.warn(`Invalid values parsed: total=${total}, available=${available}`);
|
||||
return null;
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn(`Command failed: ${command}`, error);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets detailed mount information for debugging
|
||||
*/
|
||||
private async _getMountInfo(path: string): Promise<{ filesystem: string; mountPoint: string; type: string } | null> {
|
||||
try {
|
||||
if (!fs.existsSync("/proc/mounts")) {
|
||||
@@ -109,16 +174,11 @@ export class StorageService {
|
||||
}
|
||||
|
||||
return bestMatch;
|
||||
} catch (error) {
|
||||
console.warn(`Could not get mount info for ${path}:`, error);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Detects if a path is a bind mount or mount point by checking /proc/mounts
|
||||
* Returns the actual filesystem path for bind mounts
|
||||
*/
|
||||
private async _detectMountPoint(path: string): Promise<string | null> {
|
||||
try {
|
||||
if (!fs.existsSync("/proc/mounts")) {
|
||||
@@ -133,9 +193,8 @@ export class StorageService {
|
||||
|
||||
for (const line of lines) {
|
||||
const parts = line.split(/\s+/);
|
||||
if (parts.length >= 2) {
|
||||
const [, mountPoint] = parts;
|
||||
|
||||
if (parts.length >= 3) {
|
||||
const [device, mountPoint, filesystem] = parts;
|
||||
if (path.startsWith(mountPoint) && mountPoint.length > bestMatchLength) {
|
||||
bestMatch = mountPoint;
|
||||
bestMatchLength = mountPoint.length;
|
||||
@@ -148,24 +207,16 @@ export class StorageService {
|
||||
}
|
||||
|
||||
return null;
|
||||
} catch (error) {
|
||||
console.warn(`Could not detect mount point for ${path}:`, error);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets filesystem information for a specific path, with bind mount detection
|
||||
*/
|
||||
private async _getFileSystemInfo(
|
||||
path: string
|
||||
): Promise<{ total: number; available: number; mountPoint?: string } | null> {
|
||||
try {
|
||||
const mountInfo = await this._getMountInfo(path);
|
||||
if (mountInfo && mountInfo.mountPoint !== "/") {
|
||||
console.log(`📁 Bind mount detected: ${path} → ${mountInfo.filesystem} (${mountInfo.type})`);
|
||||
}
|
||||
|
||||
const mountPoint = await this._detectMountPoint(path);
|
||||
const targetPath = mountPoint || path;
|
||||
|
||||
@@ -174,7 +225,18 @@ export class StorageService {
|
||||
? ["wmic logicaldisk get size,freespace,caption"]
|
||||
: process.platform === "darwin"
|
||||
? [`df -k "${targetPath}"`, `df "${targetPath}"`]
|
||||
: [`df -B1 "${targetPath}"`, `df -k "${targetPath}"`, `df "${targetPath}"`];
|
||||
: [
|
||||
`df -B1 "${targetPath}"`,
|
||||
`df -k "${targetPath}"`,
|
||||
`df "${targetPath}"`,
|
||||
`df -h "${targetPath}"`,
|
||||
`df -T "${targetPath}"`,
|
||||
`stat -f "${targetPath}"`,
|
||||
`findmnt -n -o AVAIL,SIZE "${targetPath}"`,
|
||||
`findmnt -n -o AVAIL,SIZE,TARGET "${targetPath}"`,
|
||||
`df -P "${targetPath}"`,
|
||||
`df --output=avail,size "${targetPath}"`,
|
||||
];
|
||||
|
||||
for (const command of commandsToTry) {
|
||||
const result = await this._tryDiskSpaceCommand(command);
|
||||
@@ -187,25 +249,54 @@ export class StorageService {
|
||||
}
|
||||
|
||||
return null;
|
||||
} catch (error) {
|
||||
console.warn(`Error getting filesystem info for ${path}:`, error);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private async _detectSynologyVolumes(): Promise<string[]> {
|
||||
try {
|
||||
if (!fs.existsSync("/proc/mounts")) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const mountsContent = await fs.promises.readFile("/proc/mounts", "utf8");
|
||||
const lines = mountsContent.split("\n").filter((line) => line.trim());
|
||||
const synologyPaths: string[] = [];
|
||||
|
||||
for (const line of lines) {
|
||||
const parts = line.split(/\s+/);
|
||||
if (parts.length >= 2) {
|
||||
const [, mountPoint] = parts;
|
||||
|
||||
if (mountPoint.match(/^\/volume\d+$/)) {
|
||||
synologyPaths.push(mountPoint);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return synologyPaths;
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
private async _getDiskSpaceMultiplePaths(): Promise<{ total: number; available: number } | null> {
|
||||
const pathsToTry = IS_RUNNING_IN_CONTAINER
|
||||
? ["/app/server/uploads", "/app/server", "/app", "/"]
|
||||
const basePaths = IS_RUNNING_IN_CONTAINER
|
||||
? ["/app/server/uploads", "/app/server/temp-uploads", "/app/server/temp-chunks", "/app/server", "/app", "/"]
|
||||
: [".", "./uploads", process.cwd()];
|
||||
|
||||
const synologyPaths = await this._detectSynologyVolumes();
|
||||
|
||||
const pathsToTry = [...basePaths, ...synologyPaths];
|
||||
|
||||
for (const pathToCheck of pathsToTry) {
|
||||
if (pathToCheck.includes("uploads")) {
|
||||
if (pathToCheck.includes("uploads") || pathToCheck.includes("temp-")) {
|
||||
try {
|
||||
if (!fs.existsSync(pathToCheck)) {
|
||||
fs.mkdirSync(pathToCheck, { recursive: true });
|
||||
}
|
||||
} catch (err) {
|
||||
console.warn(`Could not create path ${pathToCheck}:`, err);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
@@ -214,12 +305,8 @@ export class StorageService {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Use the new filesystem detection method
|
||||
const result = await this._getFileSystemInfo(pathToCheck);
|
||||
if (result) {
|
||||
if (result.mountPoint) {
|
||||
console.log(`✅ Storage resolved via bind mount: ${result.mountPoint}`);
|
||||
}
|
||||
return { total: result.total, available: result.available };
|
||||
}
|
||||
}
|
||||
@@ -241,7 +328,6 @@ export class StorageService {
|
||||
const diskInfo = await this._getDiskSpaceMultiplePaths();
|
||||
|
||||
if (!diskInfo) {
|
||||
console.error("❌ Could not determine disk space - system configuration issue");
|
||||
throw new Error("Unable to determine actual disk space - system configuration issue");
|
||||
}
|
||||
|
||||
@@ -256,7 +342,7 @@ export class StorageService {
|
||||
diskSizeGB: Number(diskSizeGB.toFixed(2)),
|
||||
diskUsedGB: Number(diskUsedGB.toFixed(2)),
|
||||
diskAvailableGB: Number(diskAvailableGB.toFixed(2)),
|
||||
uploadAllowed: diskAvailableGB > 0.1, // At least 100MB free
|
||||
uploadAllowed: diskAvailableGB > 0.1,
|
||||
};
|
||||
} else if (userId) {
|
||||
const maxTotalStorage = BigInt(await this.configService.getValue("maxTotalStoragePerUser"));
|
||||
@@ -268,7 +354,6 @@ export class StorageService {
|
||||
});
|
||||
|
||||
const totalUsedStorage = userFiles.reduce((acc, file) => acc + file.size, BigInt(0));
|
||||
|
||||
const usedStorageGB = this._ensureNumber(Number(totalUsedStorage) / (1024 * 1024 * 1024), 0);
|
||||
const availableStorageGB = this._ensureNumber(maxStorageGB - usedStorageGB, 0);
|
||||
|
||||
@@ -282,7 +367,7 @@ export class StorageService {
|
||||
|
||||
throw new Error("User ID is required for non-admin users");
|
||||
} catch (error) {
|
||||
console.error("❌ Error getting disk space:", error);
|
||||
console.error("Error getting disk space:", error);
|
||||
throw new Error(
|
||||
`Failed to get disk space information: ${error instanceof Error ? error.message : String(error)}`
|
||||
);
|
||||
|
159
apps/server/src/modules/two-factor/controller.ts
Normal file
@@ -0,0 +1,159 @@
|
||||
import { FastifyReply, FastifyRequest } from "fastify";
|
||||
import { z } from "zod";
|
||||
|
||||
import { prisma } from "../../shared/prisma";
|
||||
import { ConfigService } from "../config/service";
|
||||
import { TwoFactorService } from "./service";
|
||||
|
||||
const SetupSchema = z
|
||||
.object({
|
||||
appName: z.string().optional(),
|
||||
})
|
||||
.optional()
|
||||
.default({});
|
||||
|
||||
const VerifySetupSchema = z.object({
|
||||
token: z.string().min(6, "Token must be at least 6 characters"),
|
||||
secret: z.string().min(1, "Secret is required"),
|
||||
});
|
||||
|
||||
const VerifyTokenSchema = z.object({
|
||||
token: z.string().min(6, "Token must be at least 6 characters"),
|
||||
});
|
||||
|
||||
const DisableSchema = z.object({
|
||||
password: z.string().min(1, "Password is required"),
|
||||
});
|
||||
|
||||
export class TwoFactorController {
|
||||
private twoFactorService = new TwoFactorService();
|
||||
private configService = new ConfigService();
|
||||
|
||||
/**
|
||||
* Generate 2FA setup (QR code and secret)
|
||||
*/
|
||||
async generateSetup(request: FastifyRequest, reply: FastifyReply) {
|
||||
try {
|
||||
const userId = (request as any).user?.userId;
|
||||
if (!userId) {
|
||||
return reply.status(401).send({ error: "Unauthorized" });
|
||||
}
|
||||
|
||||
const body = SetupSchema.parse(request.body || {});
|
||||
|
||||
const user = await prisma.user.findUnique({
|
||||
where: { id: userId },
|
||||
select: { email: true },
|
||||
});
|
||||
|
||||
if (!user) {
|
||||
return reply.status(404).send({ error: "User not found" });
|
||||
}
|
||||
|
||||
const appName = body?.appName || (await this.configService.getValue("appName")) || "Palmr";
|
||||
|
||||
const setupData = await this.twoFactorService.generateSetup(userId, user.email, appName);
|
||||
|
||||
return reply.send(setupData);
|
||||
} catch (error: any) {
|
||||
return reply.status(400).send({ error: error.message });
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Verify setup token and enable 2FA
|
||||
*/
|
||||
async verifySetup(request: FastifyRequest, reply: FastifyReply) {
|
||||
try {
|
||||
const userId = (request as any).user?.userId;
|
||||
if (!userId) {
|
||||
return reply.status(401).send({ error: "Unauthorized" });
|
||||
}
|
||||
|
||||
const body = VerifySetupSchema.parse(request.body);
|
||||
|
||||
const result = await this.twoFactorService.verifySetup(userId, body.token, body.secret);
|
||||
|
||||
return reply.send(result);
|
||||
} catch (error: any) {
|
||||
return reply.status(400).send({ error: error.message });
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Verify 2FA token during login
|
||||
*/
|
||||
async verifyToken(request: FastifyRequest, reply: FastifyReply) {
|
||||
try {
|
||||
const userId = (request as any).user?.userId;
|
||||
if (!userId) {
|
||||
return reply.status(401).send({ error: "Unauthorized" });
|
||||
}
|
||||
|
||||
const body = VerifyTokenSchema.parse(request.body);
|
||||
|
||||
const result = await this.twoFactorService.verifyToken(userId, body.token);
|
||||
|
||||
return reply.send(result);
|
||||
} catch (error: any) {
|
||||
return reply.status(400).send({ error: error.message });
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Disable 2FA
|
||||
*/
|
||||
async disable2FA(request: FastifyRequest, reply: FastifyReply) {
|
||||
try {
|
||||
const userId = (request as any).user?.userId;
|
||||
if (!userId) {
|
||||
return reply.status(401).send({ error: "Unauthorized" });
|
||||
}
|
||||
|
||||
const body = DisableSchema.parse(request.body);
|
||||
|
||||
const result = await this.twoFactorService.disable2FA(userId, body.password);
|
||||
|
||||
return reply.send(result);
|
||||
} catch (error: any) {
|
||||
console.error("2FA Disable Error:", error.message);
|
||||
return reply.status(400).send({ error: error.message });
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate new backup codes
|
||||
*/
|
||||
async generateBackupCodes(request: FastifyRequest, reply: FastifyReply) {
|
||||
try {
|
||||
const userId = (request as any).user?.userId;
|
||||
if (!userId) {
|
||||
return reply.status(401).send({ error: "Unauthorized" });
|
||||
}
|
||||
|
||||
const codes = await this.twoFactorService.generateNewBackupCodes(userId);
|
||||
|
||||
return reply.send({ backupCodes: codes });
|
||||
} catch (error: any) {
|
||||
return reply.status(400).send({ error: error.message });
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get 2FA status
|
||||
*/
|
||||
async getStatus(request: FastifyRequest, reply: FastifyReply) {
|
||||
try {
|
||||
const userId = (request as any).user?.userId;
|
||||
if (!userId) {
|
||||
return reply.status(401).send({ error: "Unauthorized" });
|
||||
}
|
||||
|
||||
const status = await this.twoFactorService.getStatus(userId);
|
||||
|
||||
return reply.send(status);
|
||||
} catch (error: any) {
|
||||
return reply.status(400).send({ error: error.message });
|
||||
}
|
||||
}
|
||||
}
|
170
apps/server/src/modules/two-factor/routes.ts
Normal file
@@ -0,0 +1,170 @@
|
||||
import { FastifyInstance, FastifyReply, FastifyRequest } from "fastify";
|
||||
import { z } from "zod";
|
||||
|
||||
import { TwoFactorController } from "./controller";
|
||||
|
||||
export async function twoFactorRoutes(app: FastifyInstance) {
|
||||
const twoFactorController = new TwoFactorController();
|
||||
|
||||
const preValidation = async (request: FastifyRequest, reply: FastifyReply) => {
|
||||
try {
|
||||
await request.jwtVerify();
|
||||
} catch (err) {
|
||||
console.error(err);
|
||||
reply.status(401).send({ error: "Unauthorized: a valid token is required to access this resource." });
|
||||
}
|
||||
};
|
||||
|
||||
app.post(
|
||||
"/2fa/setup",
|
||||
{
|
||||
preValidation,
|
||||
schema: {
|
||||
tags: ["Two-Factor Authentication"],
|
||||
operationId: "generate2FASetup",
|
||||
summary: "Generate 2FA Setup",
|
||||
description: "Generate QR code and secret for 2FA setup",
|
||||
body: z.object({
|
||||
appName: z.string().optional().describe("Application name for QR code"),
|
||||
}),
|
||||
response: {
|
||||
200: z.object({
|
||||
secret: z.string().describe("Base32 encoded secret"),
|
||||
qrCode: z.string().describe("QR code as data URL"),
|
||||
manualEntryKey: z.string().describe("Manual entry key"),
|
||||
backupCodes: z
|
||||
.array(
|
||||
z.object({
|
||||
code: z.string().describe("Backup code"),
|
||||
used: z.boolean().describe("Whether backup code is used"),
|
||||
})
|
||||
)
|
||||
.describe("Backup codes"),
|
||||
}),
|
||||
400: z.object({ error: z.string().describe("Error message") }),
|
||||
401: z.object({ error: z.string().describe("Error message") }),
|
||||
},
|
||||
},
|
||||
},
|
||||
twoFactorController.generateSetup.bind(twoFactorController)
|
||||
);
|
||||
|
||||
app.post(
|
||||
"/2fa/verify-setup",
|
||||
{
|
||||
preValidation,
|
||||
schema: {
|
||||
tags: ["Two-Factor Authentication"],
|
||||
operationId: "verify2FASetup",
|
||||
summary: "Verify 2FA Setup",
|
||||
description: "Verify the setup token and enable 2FA",
|
||||
body: z.object({
|
||||
token: z.string().min(6).describe("TOTP token"),
|
||||
secret: z.string().min(1).describe("Base32 encoded secret"),
|
||||
}),
|
||||
response: {
|
||||
200: z.object({
|
||||
success: z.boolean().describe("Setup success"),
|
||||
backupCodes: z.array(z.string()).describe("Backup codes"),
|
||||
}),
|
||||
400: z.object({ error: z.string().describe("Error message") }),
|
||||
401: z.object({ error: z.string().describe("Error message") }),
|
||||
},
|
||||
},
|
||||
},
|
||||
twoFactorController.verifySetup.bind(twoFactorController)
|
||||
);
|
||||
|
||||
app.post(
|
||||
"/2fa/verify",
|
||||
{
|
||||
preValidation,
|
||||
schema: {
|
||||
tags: ["Two-Factor Authentication"],
|
||||
operationId: "verify2FAToken",
|
||||
summary: "Verify 2FA Token",
|
||||
description: "Verify a 2FA token during authentication",
|
||||
body: z.object({
|
||||
token: z.string().min(6).describe("TOTP token or backup code"),
|
||||
}),
|
||||
response: {
|
||||
200: z.object({
|
||||
success: z.boolean().describe("Verification success"),
|
||||
method: z.enum(["totp", "backup"]).describe("Verification method used"),
|
||||
}),
|
||||
400: z.object({ error: z.string().describe("Error message") }),
|
||||
401: z.object({ error: z.string().describe("Error message") }),
|
||||
},
|
||||
},
|
||||
},
|
||||
twoFactorController.verifyToken.bind(twoFactorController)
|
||||
);
|
||||
|
||||
app.post(
|
||||
"/2fa/disable",
|
||||
{
|
||||
preValidation,
|
||||
schema: {
|
||||
tags: ["Two-Factor Authentication"],
|
||||
operationId: "disable2FA",
|
||||
summary: "Disable 2FA",
|
||||
description: "Disable two-factor authentication",
|
||||
body: z.object({
|
||||
password: z.string().min(1).describe("User password for confirmation"),
|
||||
}),
|
||||
response: {
|
||||
200: z.object({
|
||||
success: z.boolean().describe("Disable success"),
|
||||
}),
|
||||
400: z.object({ error: z.string().describe("Error message") }),
|
||||
401: z.object({ error: z.string().describe("Error message") }),
|
||||
},
|
||||
},
|
||||
},
|
||||
twoFactorController.disable2FA.bind(twoFactorController)
|
||||
);
|
||||
|
||||
app.post(
|
||||
"/2fa/backup-codes",
|
||||
{
|
||||
preValidation,
|
||||
schema: {
|
||||
tags: ["Two-Factor Authentication"],
|
||||
operationId: "generateBackupCodes",
|
||||
summary: "Generate Backup Codes",
|
||||
description: "Generate new backup codes for 2FA",
|
||||
response: {
|
||||
200: z.object({
|
||||
backupCodes: z.array(z.string()).describe("New backup codes"),
|
||||
}),
|
||||
400: z.object({ error: z.string().describe("Error message") }),
|
||||
401: z.object({ error: z.string().describe("Error message") }),
|
||||
},
|
||||
},
|
||||
},
|
||||
twoFactorController.generateBackupCodes.bind(twoFactorController)
|
||||
);
|
||||
|
||||
app.get(
|
||||
"/2fa/status",
|
||||
{
|
||||
preValidation,
|
||||
schema: {
|
||||
tags: ["Two-Factor Authentication"],
|
||||
operationId: "get2FAStatus",
|
||||
summary: "Get 2FA Status",
|
||||
description: "Get current 2FA status for the user",
|
||||
response: {
|
||||
200: z.object({
|
||||
enabled: z.boolean().describe("Whether 2FA is enabled"),
|
||||
verified: z.boolean().describe("Whether 2FA is verified"),
|
||||
availableBackupCodes: z.number().describe("Number of available backup codes"),
|
||||
}),
|
||||
400: z.object({ error: z.string().describe("Error message") }),
|
||||
401: z.object({ error: z.string().describe("Error message") }),
|
||||
},
|
||||
},
|
||||
},
|
||||
twoFactorController.getStatus.bind(twoFactorController)
|
||||
);
|
||||
}
|
287
apps/server/src/modules/two-factor/service.ts
Normal file
@@ -0,0 +1,287 @@
|
||||
import crypto from "node:crypto";
|
||||
import bcrypt from "bcryptjs";
|
||||
import QRCode from "qrcode";
|
||||
import speakeasy from "speakeasy";
|
||||
|
||||
import { prisma } from "../../shared/prisma";
|
||||
import { ConfigService } from "../config/service";
|
||||
|
||||
interface BackupCode {
|
||||
code: string;
|
||||
used: boolean;
|
||||
}
|
||||
|
||||
export class TwoFactorService {
|
||||
private configService = new ConfigService();
|
||||
|
||||
/**
|
||||
* Generate a new 2FA secret and QR code for setup
|
||||
*/
|
||||
async generateSetup(userId: string, userEmail: string, appName?: string) {
|
||||
const user = await prisma.user.findUnique({
|
||||
where: { id: userId },
|
||||
select: { id: true, email: true, twoFactorEnabled: true },
|
||||
});
|
||||
|
||||
if (!user) {
|
||||
throw new Error("User not found");
|
||||
}
|
||||
|
||||
if (user.twoFactorEnabled) {
|
||||
throw new Error("Two-factor authentication is already enabled");
|
||||
}
|
||||
|
||||
const secret = speakeasy.generateSecret({
|
||||
name: `${appName || "Palmr"}:${userEmail}`,
|
||||
issuer: appName || "Palmr",
|
||||
length: 32,
|
||||
});
|
||||
|
||||
const qrCodeUrl = await QRCode.toDataURL(secret.otpauth_url || "");
|
||||
|
||||
return {
|
||||
secret: secret.base32,
|
||||
qrCode: qrCodeUrl,
|
||||
manualEntryKey: secret.base32,
|
||||
backupCodes: await this.generateBackupCodes(),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Verify setup token and enable 2FA
|
||||
*/
|
||||
async verifySetup(userId: string, token: string, secret: string) {
|
||||
const user = await prisma.user.findUnique({
|
||||
where: { id: userId },
|
||||
select: { id: true, twoFactorEnabled: true },
|
||||
});
|
||||
|
||||
if (!user) {
|
||||
throw new Error("User not found");
|
||||
}
|
||||
|
||||
if (user.twoFactorEnabled) {
|
||||
throw new Error("Two-factor authentication is already enabled");
|
||||
}
|
||||
|
||||
const verified = speakeasy.totp.verify({
|
||||
secret: secret,
|
||||
encoding: "base32",
|
||||
token: token,
|
||||
window: 1,
|
||||
});
|
||||
|
||||
if (!verified) {
|
||||
throw new Error("Invalid verification code");
|
||||
}
|
||||
|
||||
const backupCodes = await this.generateBackupCodes();
|
||||
|
||||
await prisma.user.update({
|
||||
where: { id: userId },
|
||||
data: {
|
||||
twoFactorEnabled: true,
|
||||
twoFactorSecret: secret,
|
||||
twoFactorBackupCodes: JSON.stringify(backupCodes),
|
||||
twoFactorVerified: true,
|
||||
},
|
||||
});
|
||||
|
||||
return {
|
||||
success: true,
|
||||
backupCodes: backupCodes.map((bc) => bc.code),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Verify a 2FA token during login
|
||||
*/
|
||||
async verifyToken(userId: string, token: string) {
|
||||
const user = await prisma.user.findUnique({
|
||||
where: { id: userId },
|
||||
select: {
|
||||
id: true,
|
||||
twoFactorEnabled: true,
|
||||
twoFactorSecret: true,
|
||||
twoFactorBackupCodes: true,
|
||||
},
|
||||
});
|
||||
|
||||
if (!user) {
|
||||
throw new Error("User not found");
|
||||
}
|
||||
|
||||
if (!user.twoFactorEnabled || !user.twoFactorSecret) {
|
||||
throw new Error("Two-factor authentication is not enabled");
|
||||
}
|
||||
|
||||
const verified = speakeasy.totp.verify({
|
||||
secret: user.twoFactorSecret,
|
||||
encoding: "base32",
|
||||
token: token,
|
||||
window: 1,
|
||||
});
|
||||
|
||||
if (verified) {
|
||||
return { success: true, method: "totp" };
|
||||
}
|
||||
|
||||
if (user.twoFactorBackupCodes) {
|
||||
const backupCodes: BackupCode[] = JSON.parse(user.twoFactorBackupCodes);
|
||||
const backupCodeIndex = backupCodes.findIndex((bc) => bc.code === token && !bc.used);
|
||||
|
||||
if (backupCodeIndex !== -1) {
|
||||
backupCodes[backupCodeIndex].used = true;
|
||||
|
||||
await prisma.user.update({
|
||||
where: { id: userId },
|
||||
data: {
|
||||
twoFactorBackupCodes: JSON.stringify(backupCodes),
|
||||
},
|
||||
});
|
||||
|
||||
return { success: true, method: "backup" };
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error("Invalid verification code");
|
||||
}
|
||||
|
||||
/**
|
||||
* Disable 2FA for a user
|
||||
*/
|
||||
async disable2FA(userId: string, password: string) {
|
||||
const user = await prisma.user.findUnique({
|
||||
where: { id: userId },
|
||||
select: {
|
||||
id: true,
|
||||
password: true,
|
||||
twoFactorEnabled: true,
|
||||
},
|
||||
});
|
||||
|
||||
if (!user) {
|
||||
throw new Error("User not found");
|
||||
}
|
||||
|
||||
if (!user.twoFactorEnabled) {
|
||||
throw new Error("Two-factor authentication is not enabled");
|
||||
}
|
||||
|
||||
if (!user.password) {
|
||||
throw new Error("Password verification required");
|
||||
}
|
||||
|
||||
let isValidPassword = false;
|
||||
try {
|
||||
isValidPassword = await bcrypt.compare(password, user.password);
|
||||
} catch (error) {
|
||||
console.error("bcrypt.compare error:", error);
|
||||
throw new Error("Password verification failed");
|
||||
}
|
||||
if (!isValidPassword) {
|
||||
throw new Error("Invalid password");
|
||||
}
|
||||
|
||||
await prisma.user.update({
|
||||
where: { id: userId },
|
||||
data: {
|
||||
twoFactorEnabled: false,
|
||||
twoFactorSecret: null,
|
||||
twoFactorBackupCodes: null,
|
||||
twoFactorVerified: false,
|
||||
},
|
||||
});
|
||||
|
||||
return { success: true };
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate new backup codes
|
||||
*/
|
||||
async generateNewBackupCodes(userId: string) {
|
||||
const user = await prisma.user.findUnique({
|
||||
where: { id: userId },
|
||||
select: { id: true, twoFactorEnabled: true },
|
||||
});
|
||||
|
||||
if (!user) {
|
||||
throw new Error("User not found");
|
||||
}
|
||||
|
||||
if (!user.twoFactorEnabled) {
|
||||
throw new Error("Two-factor authentication is not enabled");
|
||||
}
|
||||
|
||||
const backupCodes = await this.generateBackupCodes();
|
||||
|
||||
await prisma.user.update({
|
||||
where: { id: userId },
|
||||
data: {
|
||||
twoFactorBackupCodes: JSON.stringify(backupCodes),
|
||||
},
|
||||
});
|
||||
|
||||
return backupCodes.map((bc) => bc.code);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get 2FA status for a user
|
||||
*/
|
||||
async getStatus(userId: string) {
|
||||
const user = await prisma.user.findUnique({
|
||||
where: { id: userId },
|
||||
select: {
|
||||
id: true,
|
||||
twoFactorEnabled: true,
|
||||
twoFactorVerified: true,
|
||||
twoFactorBackupCodes: true,
|
||||
},
|
||||
});
|
||||
|
||||
if (!user) {
|
||||
throw new Error("User not found");
|
||||
}
|
||||
|
||||
let availableBackupCodes = 0;
|
||||
if (user.twoFactorBackupCodes) {
|
||||
const backupCodes: BackupCode[] = JSON.parse(user.twoFactorBackupCodes);
|
||||
availableBackupCodes = backupCodes.filter((bc) => !bc.used).length;
|
||||
}
|
||||
|
||||
return {
|
||||
enabled: user.twoFactorEnabled,
|
||||
verified: user.twoFactorVerified,
|
||||
availableBackupCodes,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate backup codes
|
||||
*/
|
||||
private async generateBackupCodes(): Promise<BackupCode[]> {
|
||||
const codes: BackupCode[] = [];
|
||||
|
||||
for (let i = 0; i < 10; i++) {
|
||||
const code = crypto.randomBytes(4).toString("hex").toUpperCase();
|
||||
codes.push({
|
||||
code: code.match(/.{1,4}/g)?.join("-") || code,
|
||||
used: false,
|
||||
});
|
||||
}
|
||||
|
||||
return codes;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if user has 2FA enabled
|
||||
*/
|
||||
async isEnabled(userId: string): Promise<boolean> {
|
||||
const user = await prisma.user.findUnique({
|
||||
where: { id: userId },
|
||||
select: { twoFactorEnabled: true },
|
||||
});
|
||||
|
||||
return user?.twoFactorEnabled ?? false;
|
||||
}
|
||||
}
|
@@ -8,21 +8,28 @@ import { pipeline } from "stream/promises";
|
||||
import { directoriesConfig, getTempFilePath } from "../config/directories.config";
|
||||
import { env } from "../env";
|
||||
import { StorageProvider } from "../types/storage";
|
||||
import { IS_RUNNING_IN_CONTAINER } from "../utils/container-detection";
|
||||
|
||||
export class FilesystemStorageProvider implements StorageProvider {
|
||||
private static instance: FilesystemStorageProvider;
|
||||
private uploadsDir: string;
|
||||
private encryptionKey = env.ENCRYPTION_KEY;
|
||||
private isEncryptionDisabled = env.DISABLE_FILESYSTEM_ENCRYPTION === "true";
|
||||
private uploadTokens = new Map<string, { objectName: string; expiresAt: number }>();
|
||||
private downloadTokens = new Map<string, { objectName: string; expiresAt: number; fileName?: string }>();
|
||||
|
||||
private constructor() {
|
||||
this.uploadsDir = directoriesConfig.uploads;
|
||||
|
||||
if (!this.isEncryptionDisabled && !this.encryptionKey) {
|
||||
throw new Error(
|
||||
"Encryption is enabled but ENCRYPTION_KEY is not provided. " +
|
||||
"Please set ENCRYPTION_KEY environment variable or set DISABLE_FILESYSTEM_ENCRYPTION=true to disable encryption."
|
||||
);
|
||||
}
|
||||
|
||||
this.ensureUploadsDir();
|
||||
setInterval(() => this.cleanExpiredTokens(), 5 * 60 * 1000);
|
||||
setInterval(() => this.cleanupEmptyTempDirs(), 10 * 60 * 1000); // Every 10 minutes
|
||||
setInterval(() => this.cleanupEmptyTempDirs(), 10 * 60 * 1000);
|
||||
}
|
||||
|
||||
public static getInstance(): FilesystemStorageProvider {
|
||||
@@ -62,10 +69,25 @@ export class FilesystemStorageProvider implements StorageProvider {
|
||||
}
|
||||
|
||||
private createEncryptionKey(): Buffer {
|
||||
if (!this.encryptionKey) {
|
||||
throw new Error(
|
||||
"Encryption key is required when encryption is enabled. Please set ENCRYPTION_KEY environment variable."
|
||||
);
|
||||
}
|
||||
return crypto.scryptSync(this.encryptionKey, "salt", 32);
|
||||
}
|
||||
|
||||
public createEncryptStream(): Transform {
|
||||
if (this.isEncryptionDisabled) {
|
||||
return new Transform({
|
||||
highWaterMark: 64 * 1024,
|
||||
transform(chunk, _encoding, callback) {
|
||||
this.push(chunk);
|
||||
callback();
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
const key = this.createEncryptionKey();
|
||||
const iv = crypto.randomBytes(16);
|
||||
const cipher = crypto.createCipheriv("aes-256-cbc", key, iv);
|
||||
@@ -73,7 +95,8 @@ export class FilesystemStorageProvider implements StorageProvider {
|
||||
let isFirstChunk = true;
|
||||
|
||||
return new Transform({
|
||||
transform(chunk, encoding, callback) {
|
||||
highWaterMark: 64 * 1024,
|
||||
transform(chunk, _encoding, callback) {
|
||||
try {
|
||||
if (isFirstChunk) {
|
||||
this.push(iv);
|
||||
@@ -101,21 +124,32 @@ export class FilesystemStorageProvider implements StorageProvider {
|
||||
}
|
||||
|
||||
public createDecryptStream(): Transform {
|
||||
if (this.isEncryptionDisabled) {
|
||||
return new Transform({
|
||||
highWaterMark: 64 * 1024,
|
||||
transform(chunk, _encoding, callback) {
|
||||
this.push(chunk);
|
||||
callback();
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
const key = this.createEncryptionKey();
|
||||
let iv: Buffer | null = null;
|
||||
let decipher: crypto.Decipher | null = null;
|
||||
let ivBuffer = Buffer.alloc(0);
|
||||
|
||||
return new Transform({
|
||||
transform(chunk, encoding, callback) {
|
||||
highWaterMark: 64 * 1024,
|
||||
transform(chunk, _encoding, callback) {
|
||||
try {
|
||||
if (!iv) {
|
||||
ivBuffer = Buffer.concat([ivBuffer, chunk]);
|
||||
|
||||
if (ivBuffer.length >= 16) {
|
||||
iv = ivBuffer.slice(0, 16);
|
||||
iv = ivBuffer.subarray(0, 16);
|
||||
decipher = crypto.createDecipheriv("aes-256-cbc", key, iv);
|
||||
const remainingData = ivBuffer.slice(16);
|
||||
const remainingData = ivBuffer.subarray(16);
|
||||
if (remainingData.length > 0) {
|
||||
const decrypted = decipher.update(remainingData);
|
||||
this.push(decrypted);
|
||||
@@ -179,7 +213,6 @@ export class FilesystemStorageProvider implements StorageProvider {
|
||||
}
|
||||
|
||||
async uploadFile(objectName: string, buffer: Buffer): Promise<void> {
|
||||
// For backward compatibility, convert buffer to stream and use streaming upload
|
||||
const filePath = this.getFilePath(objectName);
|
||||
const dir = path.dirname(filePath);
|
||||
|
||||
@@ -197,7 +230,6 @@ export class FilesystemStorageProvider implements StorageProvider {
|
||||
|
||||
await fs.mkdir(dir, { recursive: true });
|
||||
|
||||
// Use the new temp file system for better organization
|
||||
const tempPath = getTempFilePath(objectName);
|
||||
const tempDir = path.dirname(tempPath);
|
||||
|
||||
@@ -215,32 +247,283 @@ export class FilesystemStorageProvider implements StorageProvider {
|
||||
}
|
||||
}
|
||||
|
||||
private encryptFileBuffer(buffer: Buffer): Buffer {
|
||||
const key = this.createEncryptionKey();
|
||||
const iv = crypto.randomBytes(16);
|
||||
const cipher = crypto.createCipheriv("aes-256-cbc", key, iv);
|
||||
|
||||
const encrypted = Buffer.concat([iv, cipher.update(buffer), cipher.final()]);
|
||||
|
||||
return encrypted;
|
||||
}
|
||||
|
||||
async downloadFile(objectName: string): Promise<Buffer> {
|
||||
const filePath = this.getFilePath(objectName);
|
||||
const encryptedBuffer = await fs.readFile(filePath);
|
||||
const fileBuffer = await fs.readFile(filePath);
|
||||
|
||||
if (encryptedBuffer.length > 16) {
|
||||
if (this.isEncryptionDisabled) {
|
||||
return fileBuffer;
|
||||
}
|
||||
|
||||
if (fileBuffer.length > 16) {
|
||||
try {
|
||||
return this.decryptFileBuffer(encryptedBuffer);
|
||||
return this.decryptFileBuffer(fileBuffer);
|
||||
} catch (error: unknown) {
|
||||
if (error instanceof Error) {
|
||||
console.warn("Failed to decrypt with new method, trying legacy format", error.message);
|
||||
}
|
||||
return this.decryptFileLegacy(encryptedBuffer);
|
||||
return this.decryptFileLegacy(fileBuffer);
|
||||
}
|
||||
}
|
||||
|
||||
return this.decryptFileLegacy(encryptedBuffer);
|
||||
return this.decryptFileLegacy(fileBuffer);
|
||||
}
|
||||
|
||||
createDownloadStream(objectName: string): NodeJS.ReadableStream {
|
||||
const filePath = this.getFilePath(objectName);
|
||||
|
||||
const streamOptions = {
|
||||
highWaterMark: 64 * 1024,
|
||||
autoDestroy: true,
|
||||
emitClose: true,
|
||||
};
|
||||
|
||||
const fileStream = fsSync.createReadStream(filePath, streamOptions);
|
||||
|
||||
if (this.isEncryptionDisabled) {
|
||||
this.setupStreamMemoryManagement(fileStream, objectName);
|
||||
return fileStream;
|
||||
}
|
||||
|
||||
const decryptStream = this.createDecryptStream();
|
||||
const { PassThrough } = require("stream");
|
||||
const outputStream = new PassThrough(streamOptions);
|
||||
|
||||
let isDestroyed = false;
|
||||
let memoryCheckInterval: NodeJS.Timeout;
|
||||
|
||||
const cleanup = () => {
|
||||
if (isDestroyed) return;
|
||||
isDestroyed = true;
|
||||
|
||||
if (memoryCheckInterval) {
|
||||
clearInterval(memoryCheckInterval);
|
||||
}
|
||||
|
||||
try {
|
||||
if (fileStream && !fileStream.destroyed) {
|
||||
fileStream.destroy();
|
||||
}
|
||||
if (decryptStream && !decryptStream.destroyed) {
|
||||
decryptStream.destroy();
|
||||
}
|
||||
if (outputStream && !outputStream.destroyed) {
|
||||
outputStream.destroy();
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn("Error during download stream cleanup:", error);
|
||||
}
|
||||
|
||||
setImmediate(() => {
|
||||
if (global.gc) {
|
||||
global.gc();
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
memoryCheckInterval = setInterval(() => {
|
||||
const memUsage = process.memoryUsage();
|
||||
const memoryUsageMB = memUsage.heapUsed / 1024 / 1024;
|
||||
|
||||
if (memoryUsageMB > 1024) {
|
||||
if (!fileStream.readableFlowing) return;
|
||||
|
||||
console.warn(
|
||||
`[MEMORY THROTTLE] ${objectName} - Pausing stream due to high memory usage: ${memoryUsageMB.toFixed(2)}MB`
|
||||
);
|
||||
fileStream.pause();
|
||||
|
||||
if (global.gc) {
|
||||
global.gc();
|
||||
}
|
||||
|
||||
setTimeout(() => {
|
||||
if (!isDestroyed && fileStream && !fileStream.destroyed) {
|
||||
fileStream.resume();
|
||||
console.log(`[MEMORY THROTTLE] ${objectName} - Stream resumed`);
|
||||
}
|
||||
}, 100);
|
||||
}
|
||||
}, 1000);
|
||||
|
||||
fileStream.on("error", (error: any) => {
|
||||
console.error("File stream error:", error);
|
||||
cleanup();
|
||||
});
|
||||
|
||||
decryptStream.on("error", (error: any) => {
|
||||
console.error("Decrypt stream error:", error);
|
||||
cleanup();
|
||||
});
|
||||
|
||||
outputStream.on("error", (error: any) => {
|
||||
console.error("Output stream error:", error);
|
||||
cleanup();
|
||||
});
|
||||
|
||||
outputStream.on("close", cleanup);
|
||||
outputStream.on("finish", cleanup);
|
||||
|
||||
outputStream.on("pipe", (src: any) => {
|
||||
if (src && src.on) {
|
||||
src.on("close", cleanup);
|
||||
src.on("error", cleanup);
|
||||
}
|
||||
});
|
||||
|
||||
pipeline(fileStream, decryptStream, outputStream)
|
||||
.then(() => {})
|
||||
.catch((error: any) => {
|
||||
console.error("Pipeline error during download:", error);
|
||||
cleanup();
|
||||
});
|
||||
|
||||
this.setupStreamMemoryManagement(outputStream, objectName);
|
||||
return outputStream;
|
||||
}
|
||||
|
||||
private setupStreamMemoryManagement(stream: NodeJS.ReadableStream, objectName: string): void {
|
||||
let lastMemoryLog = 0;
|
||||
|
||||
stream.on("data", () => {
|
||||
const now = Date.now();
|
||||
if (now - lastMemoryLog > 30000) {
|
||||
FilesystemStorageProvider.logMemoryUsage(`Active download: ${objectName}`);
|
||||
lastMemoryLog = now;
|
||||
}
|
||||
});
|
||||
|
||||
stream.on("end", () => {
|
||||
FilesystemStorageProvider.logMemoryUsage(`Download completed: ${objectName}`);
|
||||
setImmediate(() => {
|
||||
if (global.gc) {
|
||||
global.gc();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
stream.on("close", () => {
|
||||
FilesystemStorageProvider.logMemoryUsage(`Download closed: ${objectName}`);
|
||||
});
|
||||
}
|
||||
|
||||
async createDownloadRangeStream(objectName: string, start: number, end: number): Promise<NodeJS.ReadableStream> {
|
||||
if (!this.isEncryptionDisabled) {
|
||||
return this.createRangeStreamFromDecrypted(objectName, start, end);
|
||||
}
|
||||
|
||||
const filePath = this.getFilePath(objectName);
|
||||
return fsSync.createReadStream(filePath, { start, end });
|
||||
}
|
||||
|
||||
private createRangeStreamFromDecrypted(objectName: string, start: number, end: number): NodeJS.ReadableStream {
|
||||
const { Transform, PassThrough } = require("stream");
|
||||
const filePath = this.getFilePath(objectName);
|
||||
const fileStream = fsSync.createReadStream(filePath);
|
||||
const decryptStream = this.createDecryptStream();
|
||||
const rangeStream = new PassThrough();
|
||||
|
||||
let bytesRead = 0;
|
||||
let rangeEnded = false;
|
||||
let isDestroyed = false;
|
||||
|
||||
const rangeTransform = new Transform({
|
||||
transform(chunk: Buffer, encoding: any, callback: any) {
|
||||
if (rangeEnded || isDestroyed) {
|
||||
callback();
|
||||
return;
|
||||
}
|
||||
|
||||
const chunkStart = bytesRead;
|
||||
const chunkEnd = bytesRead + chunk.length - 1;
|
||||
bytesRead += chunk.length;
|
||||
|
||||
if (chunkEnd < start) {
|
||||
callback();
|
||||
return;
|
||||
}
|
||||
|
||||
if (chunkStart > end) {
|
||||
rangeEnded = true;
|
||||
this.end();
|
||||
callback();
|
||||
return;
|
||||
}
|
||||
|
||||
let sliceStart = 0;
|
||||
let sliceEnd = chunk.length;
|
||||
|
||||
if (chunkStart < start) {
|
||||
sliceStart = start - chunkStart;
|
||||
}
|
||||
|
||||
if (chunkEnd > end) {
|
||||
sliceEnd = end - chunkStart + 1;
|
||||
rangeEnded = true;
|
||||
}
|
||||
|
||||
const slicedChunk = chunk.slice(sliceStart, sliceEnd);
|
||||
this.push(slicedChunk);
|
||||
|
||||
if (rangeEnded) {
|
||||
this.end();
|
||||
}
|
||||
|
||||
callback();
|
||||
},
|
||||
|
||||
flush(callback: any) {
|
||||
if (global.gc) {
|
||||
global.gc();
|
||||
}
|
||||
callback();
|
||||
},
|
||||
});
|
||||
|
||||
const cleanup = () => {
|
||||
if (isDestroyed) return;
|
||||
isDestroyed = true;
|
||||
|
||||
try {
|
||||
if (fileStream && !fileStream.destroyed) {
|
||||
fileStream.destroy();
|
||||
}
|
||||
if (decryptStream && !decryptStream.destroyed) {
|
||||
decryptStream.destroy();
|
||||
}
|
||||
if (rangeTransform && !rangeTransform.destroyed) {
|
||||
rangeTransform.destroy();
|
||||
}
|
||||
if (rangeStream && !rangeStream.destroyed) {
|
||||
rangeStream.destroy();
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn("Error during stream cleanup:", error);
|
||||
}
|
||||
|
||||
if (global.gc) {
|
||||
global.gc();
|
||||
}
|
||||
};
|
||||
|
||||
fileStream.on("error", cleanup);
|
||||
decryptStream.on("error", cleanup);
|
||||
rangeTransform.on("error", cleanup);
|
||||
rangeStream.on("error", cleanup);
|
||||
|
||||
rangeStream.on("close", cleanup);
|
||||
rangeStream.on("end", cleanup);
|
||||
|
||||
rangeStream.on("pipe", (src: any) => {
|
||||
if (src && src.on) {
|
||||
src.on("close", cleanup);
|
||||
src.on("error", cleanup);
|
||||
}
|
||||
});
|
||||
|
||||
fileStream.pipe(decryptStream).pipe(rangeTransform).pipe(rangeStream);
|
||||
|
||||
return rangeStream;
|
||||
}
|
||||
|
||||
private decryptFileBuffer(encryptedBuffer: Buffer): Buffer {
|
||||
@@ -254,11 +537,69 @@ export class FilesystemStorageProvider implements StorageProvider {
|
||||
}
|
||||
|
||||
private decryptFileLegacy(encryptedBuffer: Buffer): Buffer {
|
||||
if (!this.encryptionKey) {
|
||||
throw new Error(
|
||||
"Encryption key is required when encryption is enabled. Please set ENCRYPTION_KEY environment variable."
|
||||
);
|
||||
}
|
||||
const CryptoJS = require("crypto-js");
|
||||
const decrypted = CryptoJS.AES.decrypt(encryptedBuffer.toString("utf8"), this.encryptionKey);
|
||||
return Buffer.from(decrypted.toString(CryptoJS.enc.Utf8), "base64");
|
||||
}
|
||||
|
||||
static logMemoryUsage(context: string = "Unknown"): void {
|
||||
const memUsage = process.memoryUsage();
|
||||
const formatBytes = (bytes: number) => {
|
||||
const mb = bytes / 1024 / 1024;
|
||||
return `${mb.toFixed(2)} MB`;
|
||||
};
|
||||
|
||||
const rssInMB = memUsage.rss / 1024 / 1024;
|
||||
const heapUsedInMB = memUsage.heapUsed / 1024 / 1024;
|
||||
|
||||
if (rssInMB > 1024 || heapUsedInMB > 512) {
|
||||
console.warn(`[MEMORY WARNING] ${context} - High memory usage detected:`);
|
||||
console.warn(` RSS: ${formatBytes(memUsage.rss)}`);
|
||||
console.warn(` Heap Used: ${formatBytes(memUsage.heapUsed)}`);
|
||||
console.warn(` Heap Total: ${formatBytes(memUsage.heapTotal)}`);
|
||||
console.warn(` External: ${formatBytes(memUsage.external)}`);
|
||||
|
||||
if (global.gc) {
|
||||
console.warn(" Forcing garbage collection...");
|
||||
global.gc();
|
||||
|
||||
const afterGC = process.memoryUsage();
|
||||
console.warn(` After GC - RSS: ${formatBytes(afterGC.rss)}, Heap: ${formatBytes(afterGC.heapUsed)}`);
|
||||
}
|
||||
} else {
|
||||
console.log(
|
||||
`[MEMORY INFO] ${context} - RSS: ${formatBytes(memUsage.rss)}, Heap: ${formatBytes(memUsage.heapUsed)}`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
static forceGarbageCollection(context: string = "Manual"): void {
|
||||
if (global.gc) {
|
||||
const beforeGC = process.memoryUsage();
|
||||
global.gc();
|
||||
const afterGC = process.memoryUsage();
|
||||
|
||||
const formatBytes = (bytes: number) => `${(bytes / 1024 / 1024).toFixed(2)} MB`;
|
||||
|
||||
console.log(`[GC] ${context} - Before: RSS ${formatBytes(beforeGC.rss)}, Heap ${formatBytes(beforeGC.heapUsed)}`);
|
||||
console.log(`[GC] ${context} - After: RSS ${formatBytes(afterGC.rss)}, Heap ${formatBytes(afterGC.heapUsed)}`);
|
||||
|
||||
const rssSaved = beforeGC.rss - afterGC.rss;
|
||||
const heapSaved = beforeGC.heapUsed - afterGC.heapUsed;
|
||||
|
||||
if (rssSaved > 0 || heapSaved > 0) {
|
||||
console.log(`[GC] ${context} - Freed: RSS ${formatBytes(rssSaved)}, Heap ${formatBytes(heapSaved)}`);
|
||||
}
|
||||
} else {
|
||||
console.warn(`[GC] ${context} - Garbage collection not available. Start Node.js with --expose-gc flag.`);
|
||||
}
|
||||
}
|
||||
|
||||
async fileExists(objectName: string): Promise<boolean> {
|
||||
const filePath = this.getFilePath(objectName);
|
||||
try {
|
||||
@@ -303,15 +644,10 @@ export class FilesystemStorageProvider implements StorageProvider {
|
||||
this.downloadTokens.delete(token);
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up temporary file and its parent directory if empty
|
||||
*/
|
||||
private async cleanupTempFile(tempPath: string): Promise<void> {
|
||||
try {
|
||||
// Remove the temp file
|
||||
await fs.unlink(tempPath);
|
||||
|
||||
// Try to remove the parent directory if it's empty
|
||||
const tempDir = path.dirname(tempPath);
|
||||
try {
|
||||
const files = await fs.readdir(tempDir);
|
||||
@@ -319,7 +655,6 @@ export class FilesystemStorageProvider implements StorageProvider {
|
||||
await fs.rmdir(tempDir);
|
||||
}
|
||||
} catch (dirError: any) {
|
||||
// Ignore errors when trying to remove directory (might not be empty or might not exist)
|
||||
if (dirError.code !== "ENOTEMPTY" && dirError.code !== "ENOENT") {
|
||||
console.warn("Warning: Could not remove temp directory:", dirError.message);
|
||||
}
|
||||
@@ -331,18 +666,14 @@ export class FilesystemStorageProvider implements StorageProvider {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up empty temporary directories periodically
|
||||
*/
|
||||
private async cleanupEmptyTempDirs(): Promise<void> {
|
||||
try {
|
||||
const tempUploadsDir = directoriesConfig.tempUploads;
|
||||
|
||||
// Check if temp-uploads directory exists
|
||||
try {
|
||||
await fs.access(tempUploadsDir);
|
||||
} catch {
|
||||
return; // Directory doesn't exist, nothing to clean
|
||||
return;
|
||||
}
|
||||
|
||||
const items = await fs.readdir(tempUploadsDir);
|
||||
@@ -354,14 +685,12 @@ export class FilesystemStorageProvider implements StorageProvider {
|
||||
const stat = await fs.stat(itemPath);
|
||||
|
||||
if (stat.isDirectory()) {
|
||||
// Check if directory is empty
|
||||
const dirContents = await fs.readdir(itemPath);
|
||||
if (dirContents.length === 0) {
|
||||
await fs.rmdir(itemPath);
|
||||
console.log(`🧹 Cleaned up empty temp directory: ${itemPath}`);
|
||||
}
|
||||
} else if (stat.isFile()) {
|
||||
// Check if file is older than 1 hour (stale temp files)
|
||||
const oneHourAgo = Date.now() - 60 * 60 * 1000;
|
||||
if (stat.mtime.getTime() < oneHourAgo) {
|
||||
await fs.unlink(itemPath);
|
||||
@@ -369,7 +698,6 @@ export class FilesystemStorageProvider implements StorageProvider {
|
||||
}
|
||||
}
|
||||
} catch (error: any) {
|
||||
// Ignore errors for individual items
|
||||
if (error.code !== "ENOENT") {
|
||||
console.warn(`Warning: Could not process temp item ${itemPath}:`, error.message);
|
||||
}
|
||||
|
@@ -3,6 +3,7 @@ import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
|
||||
|
||||
import { bucketName, s3Client } from "../config/storage.config";
|
||||
import { StorageProvider } from "../types/storage";
|
||||
import { getContentType } from "../utils/mime-types";
|
||||
|
||||
export class S3StorageProvider implements StorageProvider {
|
||||
constructor() {
|
||||
@@ -91,6 +92,7 @@ export class S3StorageProvider implements StorageProvider {
|
||||
Bucket: bucketName,
|
||||
Key: objectName,
|
||||
ResponseContentDisposition: this.encodeFilenameForHeader(rcdFileName),
|
||||
ResponseContentType: getContentType(rcdFileName),
|
||||
});
|
||||
|
||||
return await getSignedUrl(s3Client, command, { expiresIn: expires });
|
||||
|
@@ -11,11 +11,14 @@ import { appRoutes } from "./modules/app/routes";
|
||||
import { authProvidersRoutes } from "./modules/auth-providers/routes";
|
||||
import { authRoutes } from "./modules/auth/routes";
|
||||
import { fileRoutes } from "./modules/file/routes";
|
||||
import { ChunkManager } from "./modules/filesystem/chunk-manager";
|
||||
import { downloadQueueRoutes } from "./modules/filesystem/download-queue-routes";
|
||||
import { filesystemRoutes } from "./modules/filesystem/routes";
|
||||
import { healthRoutes } from "./modules/health/routes";
|
||||
import { reverseShareRoutes } from "./modules/reverse-share/routes";
|
||||
import { shareRoutes } from "./modules/share/routes";
|
||||
import { storageRoutes } from "./modules/storage/routes";
|
||||
import { twoFactorRoutes } from "./modules/two-factor/routes";
|
||||
import { userRoutes } from "./modules/user/routes";
|
||||
import { IS_RUNNING_IN_CONTAINER } from "./utils/container-detection";
|
||||
|
||||
@@ -69,19 +72,20 @@ async function startServer() {
|
||||
|
||||
app.register(authRoutes);
|
||||
app.register(authProvidersRoutes, { prefix: "/auth" });
|
||||
app.register(twoFactorRoutes, { prefix: "/auth" });
|
||||
app.register(userRoutes);
|
||||
app.register(fileRoutes);
|
||||
|
||||
if (env.ENABLE_S3 !== "true") {
|
||||
app.register(filesystemRoutes);
|
||||
}
|
||||
|
||||
app.register(downloadQueueRoutes);
|
||||
app.register(shareRoutes);
|
||||
app.register(reverseShareRoutes);
|
||||
app.register(storageRoutes);
|
||||
app.register(appRoutes);
|
||||
app.register(healthRoutes);
|
||||
|
||||
if (env.ENABLE_S3 !== "true") {
|
||||
app.register(filesystemRoutes);
|
||||
}
|
||||
|
||||
await app.listen({
|
||||
port: 3333,
|
||||
host: "0.0.0.0",
|
||||
@@ -98,11 +102,25 @@ async function startServer() {
|
||||
}
|
||||
|
||||
console.log(`🌴 Palmr server running on port 3333 🌴`);
|
||||
console.log(`📦 Storage mode: ${env.ENABLE_S3 === "true" ? "S3" : "Local Filesystem (Encrypted)"}`);
|
||||
console.log(
|
||||
`📦 Storage mode: ${env.ENABLE_S3 === "true" ? "S3" : `Local Filesystem ${env.DISABLE_FILESYSTEM_ENCRYPTION === "true" ? "(Unencrypted)" : "(Encrypted)"}`}`
|
||||
);
|
||||
console.log(`🔐 Auth Providers: ${authProviders}`);
|
||||
|
||||
console.log("\n📚 API Documentation:");
|
||||
console.log(` - API Reference: http://localhost:3333/docs\n`);
|
||||
|
||||
process.on("SIGINT", async () => {
|
||||
const chunkManager = ChunkManager.getInstance();
|
||||
chunkManager.destroy();
|
||||
process.exit(0);
|
||||
});
|
||||
|
||||
process.on("SIGTERM", async () => {
|
||||
const chunkManager = ChunkManager.getInstance();
|
||||
chunkManager.destroy();
|
||||
process.exit(0);
|
||||
});
|
||||
}
|
||||
|
||||
startServer().catch((err) => {
|
||||
|
52
apps/server/src/types/download-queue.ts
Normal file
@@ -0,0 +1,52 @@
|
||||
/**
|
||||
* TypeScript interfaces for download queue management
|
||||
*/
|
||||
|
||||
export interface QueuedDownloadInfo {
|
||||
downloadId: string;
|
||||
position: number;
|
||||
waitTime: number;
|
||||
fileName?: string;
|
||||
fileSize?: number;
|
||||
}
|
||||
|
||||
export interface QueueStatus {
|
||||
queueLength: number;
|
||||
maxQueueSize: number;
|
||||
activeDownloads: number;
|
||||
maxConcurrent: number;
|
||||
queuedDownloads: QueuedDownloadInfo[];
|
||||
}
|
||||
|
||||
export interface DownloadCancelResponse {
|
||||
message: string;
|
||||
downloadId: string;
|
||||
}
|
||||
|
||||
export interface QueueClearResponse {
|
||||
message: string;
|
||||
clearedCount: number;
|
||||
}
|
||||
|
||||
export interface ApiResponse<T = any> {
|
||||
status: "success" | "error";
|
||||
data?: T;
|
||||
error?: string;
|
||||
message?: string;
|
||||
}
|
||||
|
||||
export interface QueueStatusResponse extends ApiResponse<QueueStatus> {
|
||||
status: "success";
|
||||
data: QueueStatus;
|
||||
}
|
||||
|
||||
export interface DownloadSlotRequest {
|
||||
fileName?: string;
|
||||
fileSize?: number;
|
||||
objectName: string;
|
||||
}
|
||||
|
||||
export interface ActiveDownloadInfo {
|
||||
startTime: number;
|
||||
memoryAtStart: number;
|
||||
}
|
423
apps/server/src/utils/download-memory-manager.ts
Normal file
@@ -0,0 +1,423 @@
|
||||
import { ActiveDownloadInfo, DownloadSlotRequest, QueuedDownloadInfo, QueueStatus } from "../types/download-queue";
|
||||
|
||||
interface QueuedDownload {
|
||||
downloadId: string;
|
||||
queuedAt: number;
|
||||
resolve: () => void;
|
||||
reject: (error: Error) => void;
|
||||
metadata?: DownloadSlotRequest;
|
||||
}
|
||||
|
||||
export class DownloadMemoryManager {
|
||||
private static instance: DownloadMemoryManager;
|
||||
private activeDownloads = new Map<string, ActiveDownloadInfo>();
|
||||
private downloadQueue: QueuedDownload[] = [];
|
||||
private maxConcurrentDownloads: number;
|
||||
private memoryThresholdMB: number;
|
||||
private maxQueueSize: number;
|
||||
private cleanupInterval: NodeJS.Timeout;
|
||||
private isAutoScalingEnabled: boolean;
|
||||
private minFileSizeGB: number;
|
||||
|
||||
private constructor() {
|
||||
const { env } = require("../env");
|
||||
|
||||
const totalMemoryGB = require("os").totalmem() / 1024 ** 3;
|
||||
this.isAutoScalingEnabled = env.DOWNLOAD_AUTO_SCALE === "true";
|
||||
|
||||
if (env.DOWNLOAD_MAX_CONCURRENT !== undefined) {
|
||||
this.maxConcurrentDownloads = env.DOWNLOAD_MAX_CONCURRENT;
|
||||
} else if (this.isAutoScalingEnabled) {
|
||||
this.maxConcurrentDownloads = this.calculateDefaultConcurrentDownloads(totalMemoryGB);
|
||||
} else {
|
||||
this.maxConcurrentDownloads = 3;
|
||||
}
|
||||
|
||||
if (env.DOWNLOAD_MEMORY_THRESHOLD_MB !== undefined) {
|
||||
this.memoryThresholdMB = env.DOWNLOAD_MEMORY_THRESHOLD_MB;
|
||||
} else if (this.isAutoScalingEnabled) {
|
||||
this.memoryThresholdMB = this.calculateDefaultMemoryThreshold(totalMemoryGB);
|
||||
} else {
|
||||
this.memoryThresholdMB = 1024;
|
||||
}
|
||||
|
||||
if (env.DOWNLOAD_QUEUE_SIZE !== undefined) {
|
||||
this.maxQueueSize = env.DOWNLOAD_QUEUE_SIZE;
|
||||
} else if (this.isAutoScalingEnabled) {
|
||||
this.maxQueueSize = this.calculateDefaultQueueSize(totalMemoryGB);
|
||||
} else {
|
||||
this.maxQueueSize = 15;
|
||||
}
|
||||
|
||||
if (env.DOWNLOAD_MIN_FILE_SIZE_GB !== undefined) {
|
||||
this.minFileSizeGB = env.DOWNLOAD_MIN_FILE_SIZE_GB;
|
||||
} else {
|
||||
this.minFileSizeGB = 3.0;
|
||||
}
|
||||
|
||||
this.validateConfiguration();
|
||||
|
||||
console.log(`[DOWNLOAD MANAGER] Configuration loaded:`);
|
||||
console.log(` System Memory: ${totalMemoryGB.toFixed(1)}GB`);
|
||||
console.log(
|
||||
` Max Concurrent: ${this.maxConcurrentDownloads} ${env.DOWNLOAD_MAX_CONCURRENT !== undefined ? "(ENV)" : "(AUTO)"}`
|
||||
);
|
||||
console.log(
|
||||
` Memory Threshold: ${this.memoryThresholdMB}MB ${env.DOWNLOAD_MEMORY_THRESHOLD_MB !== undefined ? "(ENV)" : "(AUTO)"}`
|
||||
);
|
||||
console.log(` Queue Size: ${this.maxQueueSize} ${env.DOWNLOAD_QUEUE_SIZE !== undefined ? "(ENV)" : "(AUTO)"}`);
|
||||
console.log(
|
||||
` Min File Size: ${this.minFileSizeGB}GB ${env.DOWNLOAD_MIN_FILE_SIZE_GB !== undefined ? "(ENV)" : "(DEFAULT)"}`
|
||||
);
|
||||
console.log(` Auto-scaling: ${this.isAutoScalingEnabled ? "enabled" : "disabled"}`);
|
||||
|
||||
this.cleanupInterval = setInterval(() => {
|
||||
this.cleanupStaleDownloads();
|
||||
}, 30000);
|
||||
}
|
||||
|
||||
public static getInstance(): DownloadMemoryManager {
|
||||
if (!DownloadMemoryManager.instance) {
|
||||
DownloadMemoryManager.instance = new DownloadMemoryManager();
|
||||
}
|
||||
return DownloadMemoryManager.instance;
|
||||
}
|
||||
|
||||
private calculateDefaultConcurrentDownloads(totalMemoryGB: number): number {
|
||||
if (totalMemoryGB > 16) return 10;
|
||||
if (totalMemoryGB > 8) return 5;
|
||||
if (totalMemoryGB > 4) return 3;
|
||||
if (totalMemoryGB > 2) return 2;
|
||||
return 1;
|
||||
}
|
||||
|
||||
private calculateDefaultMemoryThreshold(totalMemoryGB: number): number {
|
||||
if (totalMemoryGB > 16) return 4096; // 4GB
|
||||
if (totalMemoryGB > 8) return 2048; // 2GB
|
||||
if (totalMemoryGB > 4) return 1024; // 1GB
|
||||
if (totalMemoryGB > 2) return 512; // 512MB
|
||||
return 256; // 256MB
|
||||
}
|
||||
|
||||
private calculateDefaultQueueSize(totalMemoryGB: number): number {
|
||||
if (totalMemoryGB > 16) return 50; // Large queue for powerful servers
|
||||
if (totalMemoryGB > 8) return 25; // Medium queue
|
||||
if (totalMemoryGB > 4) return 15; // Small queue
|
||||
if (totalMemoryGB > 2) return 10; // Very small queue
|
||||
return 5; // Minimal queue
|
||||
}
|
||||
|
||||
private validateConfiguration(): void {
|
||||
const warnings: string[] = [];
|
||||
const errors: string[] = [];
|
||||
|
||||
if (this.maxConcurrentDownloads < 1) {
|
||||
errors.push(`DOWNLOAD_MAX_CONCURRENT must be >= 1, got: ${this.maxConcurrentDownloads}`);
|
||||
}
|
||||
if (this.maxConcurrentDownloads > 50) {
|
||||
warnings.push(
|
||||
`DOWNLOAD_MAX_CONCURRENT is very high (${this.maxConcurrentDownloads}), this may cause performance issues`
|
||||
);
|
||||
}
|
||||
|
||||
if (this.memoryThresholdMB < 128) {
|
||||
warnings.push(
|
||||
`DOWNLOAD_MEMORY_THRESHOLD_MB is very low (${this.memoryThresholdMB}MB), downloads may be throttled frequently`
|
||||
);
|
||||
}
|
||||
if (this.memoryThresholdMB > 16384) {
|
||||
warnings.push(
|
||||
`DOWNLOAD_MEMORY_THRESHOLD_MB is very high (${this.memoryThresholdMB}MB), system may run out of memory`
|
||||
);
|
||||
}
|
||||
|
||||
if (this.maxQueueSize < 1) {
|
||||
errors.push(`DOWNLOAD_QUEUE_SIZE must be >= 1, got: ${this.maxQueueSize}`);
|
||||
}
|
||||
if (this.maxQueueSize > 1000) {
|
||||
warnings.push(`DOWNLOAD_QUEUE_SIZE is very high (${this.maxQueueSize}), this may consume significant memory`);
|
||||
}
|
||||
|
||||
if (this.minFileSizeGB < 0.1) {
|
||||
warnings.push(
|
||||
`DOWNLOAD_MIN_FILE_SIZE_GB is very low (${this.minFileSizeGB}GB), most downloads will use memory management`
|
||||
);
|
||||
}
|
||||
if (this.minFileSizeGB > 50) {
|
||||
warnings.push(
|
||||
`DOWNLOAD_MIN_FILE_SIZE_GB is very high (${this.minFileSizeGB}GB), memory management may rarely activate`
|
||||
);
|
||||
}
|
||||
|
||||
const recommendedQueueSize = this.maxConcurrentDownloads * 5;
|
||||
if (this.maxQueueSize < this.maxConcurrentDownloads) {
|
||||
warnings.push(
|
||||
`DOWNLOAD_QUEUE_SIZE (${this.maxQueueSize}) is smaller than DOWNLOAD_MAX_CONCURRENT (${this.maxConcurrentDownloads})`
|
||||
);
|
||||
} else if (this.maxQueueSize < recommendedQueueSize) {
|
||||
warnings.push(
|
||||
`DOWNLOAD_QUEUE_SIZE (${this.maxQueueSize}) might be too small. Recommended: ${recommendedQueueSize} (5x concurrent downloads)`
|
||||
);
|
||||
}
|
||||
|
||||
if (warnings.length > 0) {
|
||||
console.warn(`[DOWNLOAD MANAGER] Configuration warnings:`);
|
||||
warnings.forEach((warning) => console.warn(` - ${warning}`));
|
||||
}
|
||||
|
||||
if (errors.length > 0) {
|
||||
console.error(`[DOWNLOAD MANAGER] Configuration errors:`);
|
||||
errors.forEach((error) => console.error(` - ${error}`));
|
||||
throw new Error(`Invalid download manager configuration: ${errors.join(", ")}`);
|
||||
}
|
||||
}
|
||||
|
||||
public async requestDownloadSlot(downloadId: string, metadata?: DownloadSlotRequest): Promise<void> {
|
||||
if (metadata?.fileSize) {
|
||||
const fileSizeGB = metadata.fileSize / 1024 ** 3;
|
||||
if (fileSizeGB < this.minFileSizeGB) {
|
||||
console.log(
|
||||
`[DOWNLOAD MANAGER] File ${metadata.fileName || "unknown"} (${fileSizeGB.toFixed(2)}GB) below threshold (${this.minFileSizeGB}GB), bypassing queue`
|
||||
);
|
||||
return Promise.resolve();
|
||||
}
|
||||
}
|
||||
|
||||
if (this.canStartImmediately()) {
|
||||
console.log(`[DOWNLOAD MANAGER] Immediate start: ${downloadId}`);
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
if (this.downloadQueue.length >= this.maxQueueSize) {
|
||||
const error = new Error(`Download queue is full: ${this.downloadQueue.length}/${this.maxQueueSize}`);
|
||||
throw error;
|
||||
}
|
||||
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
const queuedDownload: QueuedDownload = {
|
||||
downloadId,
|
||||
queuedAt: Date.now(),
|
||||
resolve,
|
||||
reject,
|
||||
metadata,
|
||||
};
|
||||
|
||||
this.downloadQueue.push(queuedDownload);
|
||||
|
||||
const position = this.downloadQueue.length;
|
||||
console.log(`[DOWNLOAD MANAGER] Queued: ${downloadId} (Position: ${position}/${this.maxQueueSize})`);
|
||||
|
||||
if (metadata?.fileName && metadata?.fileSize) {
|
||||
const sizeMB = (metadata.fileSize / (1024 * 1024)).toFixed(1);
|
||||
console.log(`[DOWNLOAD MANAGER] Queued file: ${metadata.fileName} (${sizeMB}MB)`);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private canStartImmediately(): boolean {
|
||||
const currentMemoryMB = this.getCurrentMemoryUsage();
|
||||
|
||||
if (currentMemoryMB > this.memoryThresholdMB) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (this.activeDownloads.size >= this.maxConcurrentDownloads) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
public canStartDownload(): { allowed: boolean; reason?: string } {
|
||||
if (this.canStartImmediately()) {
|
||||
return { allowed: true };
|
||||
}
|
||||
|
||||
const currentMemoryMB = this.getCurrentMemoryUsage();
|
||||
|
||||
if (currentMemoryMB > this.memoryThresholdMB) {
|
||||
return {
|
||||
allowed: false,
|
||||
reason: `Memory usage too high: ${currentMemoryMB.toFixed(0)}MB > ${this.memoryThresholdMB}MB`,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
allowed: false,
|
||||
reason: `Too many concurrent downloads: ${this.activeDownloads.size}/${this.maxConcurrentDownloads}`,
|
||||
};
|
||||
}
|
||||
|
||||
public startDownload(downloadId: string): void {
|
||||
const memUsage = process.memoryUsage();
|
||||
this.activeDownloads.set(downloadId, {
|
||||
startTime: Date.now(),
|
||||
memoryAtStart: memUsage.rss + memUsage.external,
|
||||
});
|
||||
|
||||
console.log(
|
||||
`[DOWNLOAD MANAGER] Started: ${downloadId} (${this.activeDownloads.size}/${this.maxConcurrentDownloads} active)`
|
||||
);
|
||||
}
|
||||
|
||||
public endDownload(downloadId: string): void {
|
||||
const downloadInfo = this.activeDownloads.get(downloadId);
|
||||
this.activeDownloads.delete(downloadId);
|
||||
|
||||
if (downloadInfo) {
|
||||
const duration = Date.now() - downloadInfo.startTime;
|
||||
const memUsage = process.memoryUsage();
|
||||
const currentMemory = memUsage.rss + memUsage.external;
|
||||
const memoryDiff = currentMemory - downloadInfo.memoryAtStart;
|
||||
|
||||
console.log(
|
||||
`[DOWNLOAD MANAGER] Ended: ${downloadId} (Duration: ${(duration / 1000).toFixed(1)}s, Memory delta: ${(memoryDiff / 1024 / 1024).toFixed(1)}MB)`
|
||||
);
|
||||
|
||||
if (memoryDiff > 100 * 1024 * 1024 && global.gc) {
|
||||
setImmediate(() => {
|
||||
global.gc!();
|
||||
console.log(`[DOWNLOAD MANAGER] Forced GC after download ${downloadId}`);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
this.processQueue();
|
||||
}
|
||||
|
||||
private processQueue(): void {
|
||||
if (this.downloadQueue.length === 0 || !this.canStartImmediately()) {
|
||||
return;
|
||||
}
|
||||
|
||||
const nextDownload = this.downloadQueue.shift();
|
||||
if (!nextDownload) {
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(
|
||||
`[DOWNLOAD MANAGER] Processing queue: ${nextDownload.downloadId} (${this.downloadQueue.length} remaining)`
|
||||
);
|
||||
|
||||
if (nextDownload.metadata?.fileName && nextDownload.metadata?.fileSize) {
|
||||
const sizeMB = (nextDownload.metadata.fileSize / (1024 * 1024)).toFixed(1);
|
||||
console.log(`[DOWNLOAD MANAGER] Starting queued file: ${nextDownload.metadata.fileName} (${sizeMB}MB)`);
|
||||
}
|
||||
|
||||
nextDownload.resolve();
|
||||
}
|
||||
|
||||
public getActiveDownloadsCount(): number {
|
||||
return this.activeDownloads.size;
|
||||
}
|
||||
|
||||
private getCurrentMemoryUsage(): number {
|
||||
const usage = process.memoryUsage();
|
||||
return (usage.rss + usage.external) / (1024 * 1024);
|
||||
}
|
||||
|
||||
public getCurrentMemoryUsageMB(): number {
|
||||
return this.getCurrentMemoryUsage();
|
||||
}
|
||||
|
||||
public getQueueStatus(): QueueStatus {
|
||||
return {
|
||||
queueLength: this.downloadQueue.length,
|
||||
maxQueueSize: this.maxQueueSize,
|
||||
activeDownloads: this.activeDownloads.size,
|
||||
maxConcurrent: this.maxConcurrentDownloads,
|
||||
queuedDownloads: this.downloadQueue.map((download, index) => ({
|
||||
downloadId: download.downloadId,
|
||||
position: index + 1,
|
||||
waitTime: Date.now() - download.queuedAt,
|
||||
fileName: download.metadata?.fileName,
|
||||
fileSize: download.metadata?.fileSize,
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
public cancelQueuedDownload(downloadId: string): boolean {
|
||||
const index = this.downloadQueue.findIndex((item) => item.downloadId === downloadId);
|
||||
|
||||
if (index === -1) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const canceledDownload = this.downloadQueue.splice(index, 1)[0];
|
||||
canceledDownload.reject(new Error(`Download ${downloadId} was cancelled`));
|
||||
|
||||
console.log(`[DOWNLOAD MANAGER] Cancelled queued download: ${downloadId} (was at position ${index + 1})`);
|
||||
return true;
|
||||
}
|
||||
|
||||
private cleanupStaleDownloads(): void {
|
||||
const now = Date.now();
|
||||
const staleThreshold = 10 * 60 * 1000; // 10 minutes
|
||||
const queueStaleThreshold = 30 * 60 * 1000;
|
||||
|
||||
for (const [downloadId, info] of this.activeDownloads.entries()) {
|
||||
if (now - info.startTime > staleThreshold) {
|
||||
console.warn(`[DOWNLOAD MANAGER] Cleaning up stale active download: ${downloadId}`);
|
||||
this.activeDownloads.delete(downloadId);
|
||||
}
|
||||
}
|
||||
|
||||
const initialQueueLength = this.downloadQueue.length;
|
||||
this.downloadQueue = this.downloadQueue.filter((download) => {
|
||||
if (now - download.queuedAt > queueStaleThreshold) {
|
||||
console.warn(`[DOWNLOAD MANAGER] Cleaning up stale queued download: ${download.downloadId}`);
|
||||
download.reject(new Error(`Download ${download.downloadId} timed out in queue`));
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
});
|
||||
|
||||
if (this.downloadQueue.length < initialQueueLength) {
|
||||
console.log(
|
||||
`[DOWNLOAD MANAGER] Cleaned up ${initialQueueLength - this.downloadQueue.length} stale queued downloads`
|
||||
);
|
||||
}
|
||||
|
||||
this.processQueue();
|
||||
}
|
||||
|
||||
public shouldThrottleStream(): boolean {
|
||||
const currentMemoryMB = this.getCurrentMemoryUsageMB();
|
||||
return currentMemoryMB > this.memoryThresholdMB * 0.8;
|
||||
}
|
||||
|
||||
public getThrottleDelay(): number {
|
||||
const currentMemoryMB = this.getCurrentMemoryUsageMB();
|
||||
const thresholdRatio = currentMemoryMB / this.memoryThresholdMB;
|
||||
|
||||
if (thresholdRatio > 0.9) return 200;
|
||||
if (thresholdRatio > 0.8) return 100;
|
||||
return 50;
|
||||
}
|
||||
|
||||
public destroy(): void {
|
||||
if (this.cleanupInterval) {
|
||||
clearInterval(this.cleanupInterval);
|
||||
}
|
||||
|
||||
this.downloadQueue.forEach((download) => {
|
||||
download.reject(new Error("Download manager is shutting down"));
|
||||
});
|
||||
|
||||
this.activeDownloads.clear();
|
||||
this.downloadQueue = [];
|
||||
console.log("[DOWNLOAD MANAGER] Shutdown completed");
|
||||
}
|
||||
|
||||
public clearQueue(): number {
|
||||
const clearedCount = this.downloadQueue.length;
|
||||
|
||||
this.downloadQueue.forEach((download) => {
|
||||
download.reject(new Error("Queue was cleared by administrator"));
|
||||
});
|
||||
|
||||
this.downloadQueue = [];
|
||||
console.log(`[DOWNLOAD MANAGER] Cleared queue: ${clearedCount} downloads cancelled`);
|
||||
return clearedCount;
|
||||
}
|
||||
}
|