Files
CrawlFlix-API/index.js
Joris Bertomeu b008ff411d
All checks were successful
ci / Image build (push) Successful in 2m18s
ci / Deployment (push) Successful in 24s
Add missing wantedRemux opt
2025-08-25 15:40:00 +02:00

685 lines
21 KiB
JavaScript

const express = require('express');
const Queue = require('bull');
const { spawn } = require('child_process');
const fs = require('fs');
const path = require('path');
const cors = require('cors');
const EventEmitter = require('events');
const axios = require('axios');
const mpdParser = require('mpd-parser');
const softwareService = require('./services/softwares');
const BASE_PATH = process.env.DATA_PATH || `./data`;
const OUTPUT_PATH = process.env.OUTPUT_PATH || `${BASE_PATH}/output`;
const TMP_PATH = `${BASE_PATH}/tmp`;
const app = express();
const port = 3000;
app.use(cors());
app.use(express.json());
const videoQueue = new Queue('crawlflix_queue', 'redis://192.168.1.230:6379');
videoQueue.on('error', (e) => {
console.log('An error occured', e);
});
const runCommand = (command) => {
console.log('⚙️ Will execute: ' + command);
return new Promise((resolve, reject) => {
const cmd = spawn(command, { shell: true });
let lastLog = '';
cmd.stdout.on('data', (data) => {
lastLog = data.toString();
process.stdout.write(data.toString());
});
cmd.stderr.on('data', (data) => {
lastLog = data.toString();
process.stderr.write(`stderr: ${data.toString()}`);
});
cmd.on('close', (code) => {
if (code === 0) {
resolve('Command executed successfully.');
} else {
reject(lastLog || 'Command failed with status code ' + code);
}
});
});
};
const runProgressCommand = (command) => {
console.log('⚙️ Will execute: ' + command);
const emitter = new EventEmitter();
const executeCommand = new Promise((resolve, reject) => {
const cmd = spawn(command, { shell: true });
let lastLog = '';
cmd.stdout.on('data', (data) => {
lastLog = data.toString();
const perc = extractPercentage(data.toString());
if (!perc) {
process.stdout.write(data.toString());
} else {
emitter.emit('percentage', perc);
}
});
cmd.stderr.on('data', (data) => {
lastLog = data.toString();
process.stderr.write(`stderr: ${data.toString()}`);
});
cmd.on('close', (code) => {
if (code === 0) {
resolve('Command executed successfully.');
} else {
reject(lastLog || 'Command failed with status code ' + code);
}
});
});
return { executeCommand, emitter };
};
app.use((req, res, next) => {
const timestamp = new Date().toISOString();
console.log(`[${timestamp}] ${req.method} ${req.url} - ${req.ip}`);
next();
});
app.post('/start-process', async (req, res, next) => {
try {
const { mp4Filename, mpdUrl, keys, wantedResolution, wantedAudioTracks, wantedSubtitles, wantedRemux } = req.body;
console.log(JSON.stringify(req.body, null, 2));
const job = await videoQueue.add({
mp4Filename,
mpdUrl,
keys,
wantedResolution,
wantedAudioTracks,
wantedSubtitles,
wantedRemux
});
res.json({ jobId: job.id });
} catch(e) {
console.log(e);
next();
}
});
app.get('/job-status/:jobId', async (req, res) => {
const job = await videoQueue.getJob(req.params.jobId);
if (job === null) {
res.status(404).json({ error: 'Job not found' });
} else {
const state = await job.getState();
const progress = job._progress;
res.json({ jobId: job.id, state, progress });
}
});
app.delete('/job/:jobId', async (req, res) => {
try {
const job = await videoQueue.getJob(req.params.jobId);
if (job === null) {
return res.status(404).json({ error: 'Job not found' });
}
const state = await job.getState();
if (state === 'active' || state === 'waiting') {
const { mp4Filename } = job.data;
const workdir = path.join(TMP_PATH, mp4Filename);
if (fs.existsSync(workdir)) {
await runCommand(`rm -rf ${workdir}`);
console.log(`Cleaned up temp files for cancelled job: ${workdir}`);
}
}
await job.remove();
res.json({
message: 'Job deleted successfully',
jobId: req.params.jobId,
previousState: state
});
} catch (error) {
console.error('Error deleting job:', error);
res.status(500).json({ error: error.message || 'Failed to delete job' });
}
});
app.delete('/jobs/completed', async (req, res) => {
try {
const completedJobs = await videoQueue.getJobs(['completed', 'failed'], 0, -1);
if (completedJobs.length === 0) {
return res.json({
message: 'No completed jobs to delete',
deletedCount: 0
});
}
const deletePromises = completedJobs.map(job => job.remove());
await Promise.all(deletePromises);
res.json({
message: `Successfully deleted ${completedJobs.length} completed jobs`,
deletedCount: completedJobs.length
});
} catch (error) {
console.error('Error deleting completed jobs:', error);
res.status(500).json({ error: error.message || 'Failed to delete completed jobs' });
}
});
app.delete('/jobs/all', async (req, res) => {
try {
const allJobs = await videoQueue.getJobs(['waiting', 'active', 'completed', 'failed', 'delayed'], 0, -1);
if (allJobs.length === 0) {
return res.json({
message: 'Queue is already empty',
deletedCount: 0
});
}
let cleanedDirs = 0;
for (const job of allJobs) {
const state = await job.getState();
if (state === 'active' || state === 'waiting') {
const { mp4Filename } = job.data;
const workdir = path.join(TMP_PATH, mp4Filename);
if (fs.existsSync(workdir)) {
await runCommand(`rm -rf ${workdir}`);
cleanedDirs++;
}
}
}
const deletePromises = allJobs.map(job => job.remove());
await Promise.all(deletePromises);
res.json({
message: `Successfully deleted all ${allJobs.length} jobs`,
deletedCount: allJobs.length,
cleanedTempDirs: cleanedDirs
});
} catch (error) {
console.error('Error deleting all jobs:', error);
res.status(500).json({ error: error.message || 'Failed to delete all jobs' });
}
});
app.get('/jobs/stats', async (req, res) => {
try {
const waiting = await videoQueue.getWaiting();
const active = await videoQueue.getActive();
const completed = await videoQueue.getCompleted();
const failed = await videoQueue.getFailed();
res.json({
waiting: waiting.length,
active: active.length,
completed: completed.length,
failed: failed.length,
total: waiting.length + active.length + completed.length + failed.length
});
} catch (error) {
console.error('Error getting queue stats:', error);
res.status(500).json({ error: error.message || 'Failed to get queue stats' });
}
});
app.get('/jobs-status', async (req, res) => {
const jobs = await videoQueue.getJobs();
res.json(await Promise.all(jobs.splice(0, 25).map(async job => ({
id: job.id,
state: await job.getState(),
progress: job._progress,
addedOn: job.timestamp,
processedOn: job.processedOn,
finishedOn: job.finishedOn,
returnValue: job.returnvalue,
failedReason: job.failedReason,
data: job.data
}))));
});
app.get('/download/:filename', async (req, res) => {
try {
const { filename } = req.params;
if (filename.includes('..') || filename.includes('/') || filename.includes('\\')) {
return res.status(400).json({ error: 'Invalid filename' });
}
const filePath = path.join(OUTPUT_PATH, filename);
if (!fs.existsSync(filePath)) {
return res.status(404).json({ error: 'File not found' });
}
const stat = fs.statSync(filePath);
if (!stat.isFile()) {
return res.status(400).json({ error: 'Not a file' });
}
const allowedExtensions = ['.mp4', '.mkv', '.srt', '.m4a'];
const fileExt = path.extname(filename).toLowerCase();
if (!allowedExtensions.includes(fileExt)) {
return res.status(403).json({ error: 'File type not allowed' });
}
console.log(`📥 Download requested: ${filename} (${stat.size} bytes)`);
res.download(filePath, filename, (err) => {
if (err) {
console.error('Download error:', err);
if (!res.headersSent) {
res.status(500).json({ error: 'Download failed' });
}
} else {
console.log(`✅ Download completed: ${filename}`);
}
});
} catch (error) {
console.error('Download route error:', error);
if (!res.headersSent) {
res.status(500).json({ error: 'Internal server error' });
}
}
});
app.get('/hello', async (req, res, next) => {
try {
res.json({
downloader: await softwareService.checkDownloaderUpdate(),
mp4decrypt: await softwareService.checkMp4decryptUpdate()
})
} catch(e) {
console.log(e);
next(e);
}
});
app.post('/processUpdate', async (req, res, next) => {
try {
console.log(req.body);
await softwareService.processUpdate(req.body);
res.end();
} catch(e) {
console.log(e);
next(e);
}
});
const checkFilesExistance = (pattern, path) => {
return new Promise(async (resolve, reject) => {
try {
const files = fs.readdirSync(path);
resolve(files.filter(file => file.includes(pattern)));
} catch(e) {
reject(e);
}
});
};
const extractPercentage = (line) => {
const s = line.split('%');
const percentages = [];
for (let i = 0; i < s.length; i++) {
if (s[i].length === 0) continue;
const ss = s[i].split(' ');
if (ss.length > 0 && !isNaN(ss[ss.length - 1]))
percentages.push(parseFloat(ss[ss.length - 1]));
}
if (percentages.length === 0) return null;
return Math.max(...percentages.filter((p) => p < 100));
}
const parseMPDStream = async (mpdUrl) => {
const mpdResponse = await axios({
url: mpdUrl,
method: 'GET',
responseType: 'text'
});
const eventHandler = ({ type, message }) => console.log(`${type}: ${message}`);
const parsedManifest = mpdParser.parse(mpdResponse.data , { mpdUrl, eventHandler });
const obj = {
audioTracks: [],
videoTracks: [],
subtitles: []
};
const toParse = [{
rootProp: 'AUDIO',
subProp: 'audio',
targetProp: 'audioTracks'
}, {
rootProp: 'SUBTITLES',
subProp: 'subs',
targetProp: 'subtitles'
}];
toParse.forEach(({ rootProp, subProp, targetProp }) => {
try {
for (const [key, value] of Object.entries(parsedManifest?.mediaGroups?.[rootProp]?.[subProp])) {
for (let i = 0; i < value.playlists.length; i++) {
obj[targetProp].push({
name: key,
language: value.language,
attributes: value.playlists[i].attributes
});
}
}
} catch(e) {
console.log(`No ${targetProp} found in manifest`);
}
});
for (let i = 0; i < parsedManifest.playlists.length; i++) {
obj.videoTracks.push({
name: `${parsedManifest.playlists?.[i]?.attributes?.RESOLUTION?.width || 'N/C'}x${parsedManifest.playlists?.[i]?.attributes?.RESOLUTION?.height || 'N/C'}`,
codec: parsedManifest.playlists?.[i]?.attributes?.CODECS,
bandwidth: parsedManifest.playlists?.[i]?.attributes?.BANDWIDTH,
fps: parsedManifest.playlists?.[i]?.attributes?.['FRAME-RATE'],
resolution: {
width: parsedManifest.playlists?.[i]?.attributes?.RESOLUTION?.width,
height: parsedManifest.playlists?.[i]?.attributes?.RESOLUTION?.height
}
});
}
obj.videoTracks = obj.videoTracks.sort((a, b) => b.resolution.width - a.resolution.width);
return obj;
};
app.post('/processMPD', async (req, res, next) => {
try {
res.json(await parseMPDStream(req.body.mpdUrl));
} catch(e) {
console.log(e);
next(e);
}
});
app.use((err, req, res, next) => {
res.status(500).json({ error: err.message || err.toString() || 'An error occured' });
});
const safeMove = async (source, destination) => {
try {
const destDir = path.dirname(destination);
if (!fs.existsSync(destDir)) {
fs.mkdirSync(destDir, { recursive: true });
}
fs.renameSync(source, destination);
console.log(`✓ Moved: ${path.basename(source)} -> ${destination}`);
} catch (error) {
if (error.code === 'EXDEV') {
console.log(`⚠️ Cross-device detected, copying: ${path.basename(source)}`);
fs.copyFileSync(source, destination);
fs.unlinkSync(source);
console.log(`✓ Copied: ${path.basename(source)} -> ${destination}`);
} else if (error.code === 'ENOENT') {
console.error(`❌ Source file not found: ${source}`);
throw new Error(`Source file not found: ${source}`);
} else {
console.error(`❌ Move failed:`, error);
throw error;
}
}
};
const remuxToMKV = async (options) => {
const { mp4FilePath, outputPath, filename, wantedAudioTracks, wantedSubtitles, videoInfo } = options;
try {
console.log('🎬 Starting MKV remux...');
const mkvFilePath = `${outputPath}/${filename}.mkv`;
let mkvCommand = `mkvmerge -o "${mkvFilePath}" --verbose`;
mkvCommand += ` --title "${filename}"`;
const resolution = videoInfo?.resolution || { width: 'Unknown', height: 'Unknown' };
mkvCommand += ` --language 0:und`;
mkvCommand += ` --track-name "0:Video ${resolution.width}x${resolution.height}"`;
mkvCommand += ` "${mp4FilePath}"`;
for (let i = 0; i < wantedAudioTracks.length; i++) {
const audioTrack = wantedAudioTracks[i];
const codec = audioTrack.attributes.CODECS?.split('.')[0] || 'Unknown';
const bitrate = Math.round(audioTrack.attributes.BANDWIDTH / 1000);
const trackIndex = i + 1;
mkvCommand += ` --language ${trackIndex}:${audioTrack.language}`;
mkvCommand += ` --track-name "${trackIndex}:${audioTrack.language.toUpperCase()} ${codec} ${bitrate}kbps"`;
mkvCommand += ` --default-track ${trackIndex}:${i === 0 ? 'yes' : 'no'}`;
}
const srtFiles = fs.readdirSync(outputPath).filter(file =>
file.startsWith(filename) && file.endsWith('.srt')
);
for (let i = 0; i < srtFiles.length; i++) {
const srtFile = srtFiles[i];
const srtPath = `${outputPath}/${srtFile}`;
const srtIndex = i;
const correspondingSubtitle = wantedSubtitles[srtIndex];
if (correspondingSubtitle) {
const language = correspondingSubtitle.language;
const isForced = srtFile.toLowerCase().includes('forced') || srtFile.toLowerCase().includes('sdh') === false;
const trackName = isForced ? `${language.toUpperCase()} Forced` : `${language.toUpperCase()}`;
mkvCommand += ` --language 0:${language}`;
mkvCommand += ` --track-name "0:${trackName}"`;
if (isForced) {
mkvCommand += ` --forced-track 0:yes`;
}
const defaultSub = (i === 0 && wantedAudioTracks.length > 0 && language === wantedAudioTracks[0].language) ? 'yes' : 'no';
mkvCommand += ` --default-track 0:${defaultSub}`;
mkvCommand += ` "${srtPath}"`;
}
}
console.log('🔧 MKV Command:', mkvCommand);
await runCommand(mkvCommand);
console.log(`✅ MKV remux completed: ${mkvFilePath}`);
fs.unlinkSync(mp4FilePath);
srtFiles.forEach(srt => fs.unlinkSync(`${outputPath}/${srt}`));
return mkvFilePath;
} catch (error) {
console.error('❌ MKV remux failed:', error);
throw new Error(`MKV remux failed: ${error.message}`);
}
};
// Processus de la file d'attente
videoQueue.process((job) => {
return new Promise(async (resolve, reject) => {
try {
console.log('Will launch job')
const { mp4Filename, mpdUrl, keys, wantedResolution, wantedAudioTracks, wantedSubtitles, wantedRemux } = job.data;
const downloaderPath = softwareService.getLocalBinFileInfo('downloader').path;
const mp4decryptPath = softwareService.getLocalBinFileInfo('mp4decrypt').path;
const workdir = path.join(TMP_PATH, mp4Filename);
if (!fs.existsSync(workdir))
fs.mkdirSync(workdir);
const mp4FilenameWithExt= `${mp4Filename}.mp4`;
const finalPath = path.join(OUTPUT_PATH, mp4Filename);
const filesExist = await checkFilesExistance('encrypted', workdir);
if (filesExist.length === 0) {
console.log('Encrypted files not found, downloading...');
let objectsDownloaded = -1, previousPercentage = -1;
const objectsToDownload = 1 + wantedAudioTracks.length + wantedSubtitles.length;
job.progress(10); // Début à 10%
const bwAudio = wantedAudioTracks.length === 1 ? `:bwMin=\"${wantedAudioTracks.map(elem => Math.floor(elem.attributes.BANDWIDTH / 1000 -1)).join('|')}\":bwMax=\"${wantedAudioTracks.map(elem => Math.round(elem.attributes.BANDWIDTH / 1000 + 1)).join('|')}\"` : '';
const bwSubs = wantedSubtitles.length === 1 ? `:bwMin=\"${wantedSubtitles.map(elem => Math.floor(elem.attributes.BANDWIDTH / 1000 -1)).join('|')}\":bwMax=\"${wantedSubtitles.map(elem => Math.round(elem.attributes.BANDWIDTH / 1000 + 1)).join('|')}\"` : '';
const subPart = wantedSubtitles.length > 0 ? `--select-subtitle lang=\"${wantedSubtitles.map(elem => elem.language).join('|')}\"${bwSubs}` : '--drop-subtitle lang=\".*\"';
const audioPart = wantedAudioTracks.length > 0 ? `--select-audio lang=\"${wantedAudioTracks.map(elem => elem.language).join('|')}\":codecs=\"${[...new Set(wantedAudioTracks.map(elem => elem.attributes.CODECS))].join('|')}\":for=all${bwAudio}` : '--drop-audio lang=\".*\"';
const { executeCommand, emitter } = runProgressCommand(`${downloaderPath} \"${mpdUrl}\" --save-dir ${workdir} --save-name ${mp4Filename}_encrypted --select-video res=\"${wantedResolution.resolution.width}*\" ${audioPart} ${subPart}`, true);
emitter.on('percentage', (percentage) => {
if (percentage < previousPercentage) {
objectsDownloaded++;
}
previousPercentage = percentage;
const subPercMax = 50 / objectsToDownload;
job.progress(Math.round((10 + (objectsDownloaded * subPercMax)) + (percentage * subPercMax / 100)));
});
await executeCommand;
} else {
console.log('Encrypted files already exist, bypassing download...')
}
job.progress(60);
// Decrypt video stream
await runCommand(`${mp4decryptPath} ${keys.map(k => `--key ${k.key}:${k.value}`).join(' ')} "${workdir}/${mp4Filename}_encrypted.mp4" "${workdir}/${mp4Filename}_decrypted.mp4"`);
job.progress(70);
// Decrypt audio streams
const audioFiles = fs.readdirSync(workdir);
const finalAudio = [];
for (const file of audioFiles) {
if (file.startsWith(`${mp4Filename}_encrypted`) && file.endsWith('.m4a')) {
const baseName = path.basename(file, '.m4a');
await runCommand(`${mp4decryptPath} ${keys.map(k => `--key ${k.key}:${k.value}`).join(' ')} "${workdir}/${file}" "${workdir}/${baseName}_decrypted.m4a"`);
finalAudio.push(`${workdir}/${baseName}_decrypted.m4a`);
}
}
job.progress(80);
// Combinaison avec ffmpeg
let ffmpegCommand = `ffmpeg -y -i ${workdir}/${mp4Filename}_decrypted.mp4`;
let mapCommand = ' -map 0:v';
let inputIndex = 1;
for (const file of finalAudio) {
ffmpegCommand += ` -i ${file}`;
mapCommand += ` -map ${inputIndex}:a`;
inputIndex++;
}
if (!fs.existsSync(finalPath))
fs.mkdirSync(finalPath);
ffmpegCommand += `${mapCommand} -c copy ${finalPath}/${mp4FilenameWithExt}`;
await runCommand(ffmpegCommand);
job.progress(90);
if (wantedRemux) {
console.log('🎬 Starting optional MKV remux...');
try {
await remuxToMKV({
mp4FilePath: `${finalPath}/${mp4FilenameWithExt}`,
outputPath: finalPath,
filename: mp4Filename,
wantedAudioTracks,
wantedSubtitles,
videoInfo: wantedResolution
});
job.progress(95);
console.log('✅ MKV remux completed successfully');
} catch (remuxError) {
console.error('⚠️ MKV remux failed, keeping MP4:', remuxError);
}
}
// Renommage des fichiers SRT
const subFiles = fs.readdirSync(workdir);
let counter = 1;
for (const file of subFiles) {
if (file.startsWith(`${mp4Filename}_encrypted`) && file.endsWith('.srt')) {
const sourcePath = `${workdir}/${file}`;
const destPath = `${finalPath}/${mp4Filename}_${counter}.srt`;
await safeMove(sourcePath, destPath);
counter++;
}
}
// Nettoyage (commenté pour correspondre au script original)
await runCommand(`rm -fr ${workdir}`);
job.progress(100);
resolve({ message: `File fetched and decrypted with success: ${mp4Filename}.mp4`, filePath: `${OUTPUT_PATH}/${mp4Filename}.mp4`, fileName: `${mp4Filename}.mp4` });
} catch (error) {
console.log('Error while processing task', error)
reject(new Error(`${error.toString() || error}`));
}
});
});
app.listen(port, () => {
console.log(`Server running at http://localhost:${port}`);
const dirsToCheck = [{
path: BASE_PATH,
name: 'data'
}, {
path: OUTPUT_PATH,
name: 'output'
}, {
path: TMP_PATH,
name: 'tmp'
}];
for (let i = 0; i < dirsToCheck.length; i++) {
const dir = dirsToCheck[i];
if (!fs.existsSync(dir.path)) {
console.log(`Creating ${dir.name} directory...`);
fs.mkdirSync(dir.path);
}
}
softwareService.init();
});