const express = require('express'); const Queue = require('bull'); const { spawn } = require('child_process'); const fs = require('fs'); const path = require('path'); const cors = require('cors'); const EventEmitter = require('events'); const axios = require('axios'); const mpdParser = require('mpd-parser'); const softwareService = require('./services/softwares'); const BASE_PATH = process.env.DATA_PATH || `./data`; const OUTPUT_PATH = process.env.OUTPUT_PATH || `${BASE_PATH}/output`; const TMP_PATH = `${BASE_PATH}/tmp`; const app = express(); const port = 3000; app.use(cors()); app.use(express.json()); const videoQueue = new Queue('crawlflix_queue', 'redis://192.168.1.230:6379'); videoQueue.on('error', (e) => { console.log('An error occured', e); }); const runCommand = (command) => { console.log('⚙️ Will execute: ' + command); return new Promise((resolve, reject) => { const cmd = spawn(command, { shell: true }); let lastLog = ''; cmd.stdout.on('data', (data) => { lastLog = data.toString(); process.stdout.write(data.toString()); }); cmd.stderr.on('data', (data) => { lastLog = data.toString(); process.stderr.write(`stderr: ${data.toString()}`); }); cmd.on('close', (code) => { if (code === 0) { resolve('Command executed successfully.'); } else { reject(lastLog || 'Command failed with status code ' + code); } }); }); }; const runProgressCommand = (command) => { console.log('⚙️ Will execute: ' + command); const emitter = new EventEmitter(); const executeCommand = new Promise((resolve, reject) => { const cmd = spawn(command, { shell: true }); let lastLog = ''; cmd.stdout.on('data', (data) => { lastLog = data.toString(); const perc = extractPercentage(data.toString()); if (!perc) { process.stdout.write(data.toString()); } else { emitter.emit('percentage', perc); } }); cmd.stderr.on('data', (data) => { lastLog = data.toString(); process.stderr.write(`stderr: ${data.toString()}`); }); cmd.on('close', (code) => { if (code === 0) { resolve('Command executed successfully.'); } else { reject(lastLog || 'Command failed with status code ' + code); } }); }); return { executeCommand, emitter }; }; app.use((req, res, next) => { const timestamp = new Date().toISOString(); console.log(`[${timestamp}] ${req.method} ${req.url} - ${req.ip}`); next(); }); app.post('/start-process', async (req, res, next) => { try { const { mp4Filename, mpdUrl, keys, wantedResolution, wantedAudioTracks, wantedSubtitles } = req.body; console.log(JSON.stringify(req.body, null, 2)); const job = await videoQueue.add({ mp4Filename, mpdUrl, keys, wantedResolution, wantedAudioTracks, wantedSubtitles }); res.json({ jobId: job.id }); } catch(e) { console.log(e); next(); } }); app.get('/job-status/:jobId', async (req, res) => { const job = await videoQueue.getJob(req.params.jobId); if (job === null) { res.status(404).json({ error: 'Job not found' }); } else { const state = await job.getState(); const progress = job._progress; res.json({ jobId: job.id, state, progress }); } }); app.delete('/job/:jobId', async (req, res) => { try { const job = await videoQueue.getJob(req.params.jobId); if (job === null) { return res.status(404).json({ error: 'Job not found' }); } const state = await job.getState(); if (state === 'active' || state === 'waiting') { const { mp4Filename } = job.data; const workdir = path.join(TMP_PATH, mp4Filename); if (fs.existsSync(workdir)) { await runCommand(`rm -rf ${workdir}`); console.log(`Cleaned up temp files for cancelled job: ${workdir}`); } } await job.remove(); res.json({ message: 'Job deleted successfully', jobId: req.params.jobId, previousState: state }); } catch (error) { console.error('Error deleting job:', error); res.status(500).json({ error: error.message || 'Failed to delete job' }); } }); app.delete('/jobs/completed', async (req, res) => { try { const completedJobs = await videoQueue.getJobs(['completed', 'failed'], 0, -1); if (completedJobs.length === 0) { return res.json({ message: 'No completed jobs to delete', deletedCount: 0 }); } const deletePromises = completedJobs.map(job => job.remove()); await Promise.all(deletePromises); res.json({ message: `Successfully deleted ${completedJobs.length} completed jobs`, deletedCount: completedJobs.length }); } catch (error) { console.error('Error deleting completed jobs:', error); res.status(500).json({ error: error.message || 'Failed to delete completed jobs' }); } }); app.delete('/jobs/all', async (req, res) => { try { const allJobs = await videoQueue.getJobs(['waiting', 'active', 'completed', 'failed', 'delayed'], 0, -1); if (allJobs.length === 0) { return res.json({ message: 'Queue is already empty', deletedCount: 0 }); } let cleanedDirs = 0; for (const job of allJobs) { const state = await job.getState(); if (state === 'active' || state === 'waiting') { const { mp4Filename } = job.data; const workdir = path.join(TMP_PATH, mp4Filename); if (fs.existsSync(workdir)) { await runCommand(`rm -rf ${workdir}`); cleanedDirs++; } } } const deletePromises = allJobs.map(job => job.remove()); await Promise.all(deletePromises); res.json({ message: `Successfully deleted all ${allJobs.length} jobs`, deletedCount: allJobs.length, cleanedTempDirs: cleanedDirs }); } catch (error) { console.error('Error deleting all jobs:', error); res.status(500).json({ error: error.message || 'Failed to delete all jobs' }); } }); app.get('/jobs/stats', async (req, res) => { try { const waiting = await videoQueue.getWaiting(); const active = await videoQueue.getActive(); const completed = await videoQueue.getCompleted(); const failed = await videoQueue.getFailed(); res.json({ waiting: waiting.length, active: active.length, completed: completed.length, failed: failed.length, total: waiting.length + active.length + completed.length + failed.length }); } catch (error) { console.error('Error getting queue stats:', error); res.status(500).json({ error: error.message || 'Failed to get queue stats' }); } }); app.get('/jobs-status', async (req, res) => { const jobs = await videoQueue.getJobs(); res.json(await Promise.all(jobs.splice(0, 25).map(async job => ({ id: job.id, state: await job.getState(), progress: job._progress, addedOn: job.timestamp, processedOn: job.processedOn, finishedOn: job.finishedOn, returnValue: job.returnvalue, failedReason: job.failedReason, data: job.data })))); }); app.get('/download/:filename', async (req, res) => { const { filename } = req.params; const file = path.join(process.cwd(), filename); res.download(file); }); app.get('/hello', async (req, res, next) => { try { res.json({ downloader: await softwareService.checkDownloaderUpdate(), mp4decrypt: await softwareService.checkMp4decryptUpdate() }) } catch(e) { console.log(e); next(e); } }); app.post('/processUpdate', async (req, res, next) => { try { console.log(req.body); await softwareService.processUpdate(req.body); res.end(); } catch(e) { console.log(e); next(e); } }); const checkFilesExistance = (pattern, path) => { return new Promise(async (resolve, reject) => { try { const files = fs.readdirSync(path); resolve(files.filter(file => file.includes(pattern))); } catch(e) { reject(e); } }); }; const extractPercentage = (line) => { const s = line.split('%'); const percentages = []; for (let i = 0; i < s.length; i++) { if (s[i].length === 0) continue; const ss = s[i].split(' '); if (ss.length > 0 && !isNaN(ss[ss.length - 1])) percentages.push(parseFloat(ss[ss.length - 1])); } if (percentages.length === 0) return null; return Math.max(...percentages.filter((p) => p < 100)); } const parseMPDStream = async (mpdUrl) => { const mpdResponse = await axios({ url: mpdUrl, method: 'GET', responseType: 'text' }); const eventHandler = ({ type, message }) => console.log(`${type}: ${message}`); const parsedManifest = mpdParser.parse(mpdResponse.data , { mpdUrl, eventHandler }); const obj = { audioTracks: [], videoTracks: [], subtitles: [] }; const toParse = [{ rootProp: 'AUDIO', subProp: 'audio', targetProp: 'audioTracks' }, { rootProp: 'SUBTITLES', subProp: 'subs', targetProp: 'subtitles' }]; toParse.forEach(({ rootProp, subProp, targetProp }) => { try { for (const [key, value] of Object.entries(parsedManifest?.mediaGroups?.[rootProp]?.[subProp])) { for (let i = 0; i < value.playlists.length; i++) { obj[targetProp].push({ name: key, language: value.language, attributes: value.playlists[i].attributes }); } } } catch(e) { console.log(`No ${targetProp} found in manifest`); } }); for (let i = 0; i < parsedManifest.playlists.length; i++) { obj.videoTracks.push({ name: `${parsedManifest.playlists?.[i]?.attributes?.RESOLUTION?.width || 'N/C'}x${parsedManifest.playlists?.[i]?.attributes?.RESOLUTION?.height || 'N/C'}`, codec: parsedManifest.playlists?.[i]?.attributes?.CODECS, bandwidth: parsedManifest.playlists?.[i]?.attributes?.BANDWIDTH, fps: parsedManifest.playlists?.[i]?.attributes?.['FRAME-RATE'], resolution: { width: parsedManifest.playlists?.[i]?.attributes?.RESOLUTION?.width, height: parsedManifest.playlists?.[i]?.attributes?.RESOLUTION?.height } }); } obj.videoTracks = obj.videoTracks.sort((a, b) => b.resolution.width - a.resolution.width); return obj; }; app.post('/processMPD', async (req, res, next) => { try { res.json(await parseMPDStream(req.body.mpdUrl)); } catch(e) { console.log(e); next(e); } }); app.use((err, req, res, next) => { res.status(500).json({ error: err.message || err.toString() || 'An error occured' }); }); const safeMove = async (source, destination) => { try { const destDir = path.dirname(destination); if (!fs.existsSync(destDir)) { fs.mkdirSync(destDir, { recursive: true }); } fs.renameSync(source, destination); console.log(`✓ Moved: ${path.basename(source)} -> ${destination}`); } catch (error) { if (error.code === 'EXDEV') { console.log(`⚠️ Cross-device detected, copying: ${path.basename(source)}`); fs.copyFileSync(source, destination); fs.unlinkSync(source); console.log(`✓ Copied: ${path.basename(source)} -> ${destination}`); } else if (error.code === 'ENOENT') { console.error(`❌ Source file not found: ${source}`); throw new Error(`Source file not found: ${source}`); } else { console.error(`❌ Move failed:`, error); throw error; } } }; // Processus de la file d'attente videoQueue.process((job) => { return new Promise(async (resolve, reject) => { try { console.log('Will launch job') const { mp4Filename, mpdUrl, keys, wantedResolution, wantedAudioTracks, wantedSubtitles } = job.data; const downloaderPath = softwareService.getLocalBinFileInfo('downloader').path; const mp4decryptPath = softwareService.getLocalBinFileInfo('mp4decrypt').path; const workdir = path.join(TMP_PATH, mp4Filename); if (!fs.existsSync(workdir)) fs.mkdirSync(workdir); const mp4FilenameWithExt= `${mp4Filename}.mp4`; const finalPath = path.join(OUTPUT_PATH, mp4Filename); const filesExist = await checkFilesExistance('encrypted', workdir); if (filesExist.length === 0) { console.log('Encrypted files not found, downloading...'); let objectsDownloaded = -1, previousPercentage = -1; const objectsToDownload = 1 + wantedAudioTracks.length + wantedSubtitles.length; job.progress(10); // Début à 10% const bwAudio = wantedAudioTracks.length === 1 ? `:bwMin=\"${wantedAudioTracks.map(elem => Math.floor(elem.attributes.BANDWIDTH / 1000 -1)).join('|')}\":bwMax=\"${wantedAudioTracks.map(elem => Math.round(elem.attributes.BANDWIDTH / 1000 + 1)).join('|')}\"` : ''; const bwSubs = wantedSubtitles.length === 1 ? `:bwMin=\"${wantedSubtitles.map(elem => Math.floor(elem.attributes.BANDWIDTH / 1000 -1)).join('|')}\":bwMax=\"${wantedSubtitles.map(elem => Math.round(elem.attributes.BANDWIDTH / 1000 + 1)).join('|')}\"` : ''; const subPart = wantedSubtitles.length > 0 ? `--select-subtitle lang=\"${wantedSubtitles.map(elem => elem.language).join('|')}\"${bwSubs}` : '--drop-subtitle lang=\".*\"'; const audioPart = wantedAudioTracks.length > 0 ? `--select-audio lang=\"${wantedAudioTracks.map(elem => elem.language).join('|')}\":codecs=\"${[...new Set(wantedAudioTracks.map(elem => elem.attributes.CODECS))].join('|')}\":for=all${bwAudio}` : '--drop-audio lang=\".*\"'; const { executeCommand, emitter } = runProgressCommand(`${downloaderPath} \"${mpdUrl}\" --save-dir ${workdir} --save-name ${mp4Filename}_encrypted --select-video res=\"${wantedResolution.resolution.width}*\" ${audioPart} ${subPart}`, true); emitter.on('percentage', (percentage) => { if (percentage < previousPercentage) { objectsDownloaded++; } previousPercentage = percentage; const subPercMax = 50 / objectsToDownload; job.progress(Math.round((10 + (objectsDownloaded * subPercMax)) + (percentage * subPercMax / 100))); }); await executeCommand; } else { console.log('Encrypted files already exist, bypassing download...') } job.progress(60); // Decrypt video stream await runCommand(`${mp4decryptPath} ${keys.map(k => `--key ${k.key}:${k.value}`).join(' ')} "${workdir}/${mp4Filename}_encrypted.mp4" "${workdir}/${mp4Filename}_decrypted.mp4"`); job.progress(70); // Decrypt audio streams const audioFiles = fs.readdirSync(workdir); const finalAudio = []; for (const file of audioFiles) { if (file.startsWith(`${mp4Filename}_encrypted`) && file.endsWith('.m4a')) { const baseName = path.basename(file, '.m4a'); await runCommand(`${mp4decryptPath} ${keys.map(k => `--key ${k.key}:${k.value}`).join(' ')} "${workdir}/${file}" "${workdir}/${baseName}_decrypted.m4a"`); finalAudio.push(`${workdir}/${baseName}_decrypted.m4a`); } } job.progress(80); // Combinaison avec ffmpeg let ffmpegCommand = `ffmpeg -y -i ${workdir}/${mp4Filename}_decrypted.mp4`; let mapCommand = ' -map 0:v'; let inputIndex = 1; for (const file of finalAudio) { ffmpegCommand += ` -i ${file}`; mapCommand += ` -map ${inputIndex}:a`; inputIndex++; } if (!fs.existsSync(finalPath)) fs.mkdirSync(finalPath); ffmpegCommand += `${mapCommand} -c copy ${finalPath}/${mp4FilenameWithExt}`; await runCommand(ffmpegCommand); job.progress(90); // Renommage des fichiers SRT const subFiles = fs.readdirSync(workdir); let counter = 1; for (const file of subFiles) { if (file.startsWith(`${mp4Filename}_encrypted`) && file.endsWith('.srt')) { const sourcePath = `${workdir}/${file}`; const destPath = `${finalPath}/${mp4Filename}_${counter}.srt`; await safeMove(sourcePath, destPath); counter++; } } // Nettoyage (commenté pour correspondre au script original) await runCommand(`rm -fr ${workdir}`); job.progress(100); resolve({ message: `File fetched and decrypted with success: ${mp4Filename}.mp4`, filePath: `${OUTPUT_PATH}/${mp4Filename}.mp4`, fileName: `${mp4Filename}.mp4` }); } catch (error) { console.log('Error while processing task', error) reject(new Error(`${error.toString() || error}`)); } }); }); app.listen(port, () => { console.log(`Server running at http://localhost:${port}`); const dirsToCheck = [{ path: BASE_PATH, name: 'data' }, { path: OUTPUT_PATH, name: 'output' }, { path: TMP_PATH, name: 'tmp' }]; for (let i = 0; i < dirsToCheck.length; i++) { const dir = dirsToCheck[i]; if (!fs.existsSync(dir.path)) { console.log(`Creating ${dir.name} directory...`); fs.mkdirSync(dir.path); } } softwareService.init(); });