"use strict"; // License: MIT import { EventEmitter } from "../events"; import { Notification } from "../notifications"; import { DB } from "../db"; import { QUEUED, CANCELED, RUNNING, RETRYING } from "./state"; // eslint-disable-next-line no-unused-vars import { Bus, Port } from "../bus"; import { sort } from "../sorting"; import { Prefs, PrefWatcher } from "../prefs"; import { _ } from "../i18n"; import { CoalescedUpdate, mapFilterInSitu, filterInSitu } from "../util"; import { PromiseSerializer } from "../pserializer"; import { Download } from "./download"; import { ManagerPort } from "./port"; import { Scheduler } from "./scheduler"; import { Limits } from "./limits"; import { downloads, runtime, webRequest, CHROME, OPERA } from "../browser"; import { browser } from "webextension-polyfill-ts"; const US = runtime.getURL(""); const AUTOSAVE_TIMEOUT = 2000; const DIRTY_TIMEOUT = 100; // eslint-disable-next-line no-magic-numbers const MISSING_TIMEOUT = 12 * 1000; const RELOAD_TIMEOUT = 10 * 1000; const setShelfEnabled = downloads.setShelfEnabled || function() { // ignored }; const FINISH_NOTIFICATION = new PrefWatcher("finish-notification", true); const SOUNDS = new PrefWatcher("sounds", false); export class Manager extends EventEmitter { private items: Download[]; public active: boolean; private notifiedFinished: boolean; private readonly saveQueue: CoalescedUpdate; private readonly dirty: CoalescedUpdate; private readonly sids: Map; private readonly manIds: Map; private readonly ports: Set; private readonly running: Set; private readonly retrying: Set; private scheduler: Scheduler | null; private shouldReload: boolean; private deadlineTimer: number; constructor() { if (!document.location.href.includes("background")) { throw new Error("Not on background"); } super(); this.active = true; this.shouldReload = false; this.notifiedFinished = true; this.items = []; this.saveQueue = new CoalescedUpdate( AUTOSAVE_TIMEOUT, this.save.bind(this)); this.dirty = new CoalescedUpdate( DIRTY_TIMEOUT, this.processDirty.bind(this)); this.processDeadlines = this.processDeadlines.bind(this); this.sids = new Map(); this.manIds = new Map(); this.ports = new Set(); this.scheduler = null; this.running = new Set(); this.retrying = new Set(); this.startNext = PromiseSerializer.wrapNew(1, this, this.startNext); downloads.onChanged.addListener(this.onChanged.bind(this)); downloads.onErased.addListener(this.onErased.bind(this)); if (CHROME && downloads.onDeterminingFilename) { downloads.onDeterminingFilename.addListener( this.onDeterminingFilename.bind(this)); } Bus.onPort("manager", (port: Port) => { const managerPort = new ManagerPort(this, port); port.on("disconnect", () => { this.ports.delete(managerPort); }); this.ports.add(managerPort); return true; }); Limits.on("changed", () => { this.resetScheduler(); }); if (CHROME) { webRequest.onBeforeSendHeaders.addListener( this.stuffReferrer.bind(this), {urls: [""]}, ["blocking", "requestHeaders", "extraHeaders"] ); } } async init() { const items = await DB.getAll(); items.forEach((i: any, idx: number) => { const rv = new Download(this, i); rv.position = idx; this.sids.set(rv.sessionId, rv); if (rv.manId) { this.manIds.set(rv.manId, rv); } this.items.push(rv); }); // Do not wait for the scheduler this.resetScheduler(); this.emit("initialized"); setTimeout(() => this.checkMissing(), MISSING_TIMEOUT); runtime.onUpdateAvailable.addListener(() => { if (this.running.size) { this.shouldReload = true; return; } runtime.reload(); }); return this; } async checkMissing() { const serializer = new PromiseSerializer(2); const missing = await Promise.all(this.items.map( item => serializer.scheduleWithContext(item, item.maybeMissing))); if (!(await Prefs.get("remove-missing-on-init"))) { return; } this.remove(filterInSitu(missing, e => !!e)); } onChanged(changes: {id: number}) { const item = this.manIds.get(changes.id); if (!item) { return; } item.updateStateFromBrowser(); } onErased(downloadId: number) { const item = this.manIds.get(downloadId); if (!item) { return; } item.setMissing(); this.manIds.delete(downloadId); } onDeterminingFilename(state: any, suggest: Function) { const download = this.manIds.get(state.id); if (!download) { return; } try { download.updateFromSuggestion(state); } finally { const suggestion = {filename: download.dest.full}; suggest(suggestion); } } async resetScheduler() { this.scheduler = null; await this.startNext(); } async startNext() { if (!this.active) { return; } while (this.running.size < Limits.concurrent) { if (!this.scheduler) { this.scheduler = new Scheduler(this.items); } const next = await this.scheduler.next(this.running); if (!next) { this.maybeRunFinishActions(); break; } if (this.running.has(next) || next.state !== QUEUED) { continue; } try { await this.startDownload(next); } catch (ex) { next.changeState(CANCELED); next.error = ex.toString(); console.error(ex.toString(), ex); } } } async startDownload(download: Download) { // Add to running first, so we don't confuse the scheduler and other parts this.running.add(download); setShelfEnabled(false); await download.start(); this.notifiedFinished = false; } maybeRunFinishActions() { if (this.running.size) { return; } this.maybeNotifyFinished(); if (this.shouldReload) { this.saveQueue.trigger(); setTimeout(() => { if (this.running.size) { return; } runtime.reload(); }, RELOAD_TIMEOUT); } setShelfEnabled(true); } maybeNotifyFinished() { if (this.notifiedFinished || this.running.size || this.retrying.size) { return; } if (SOUNDS.value && !OPERA) { const audio = new Audio(runtime.getURL("/style/done.opus")); audio.addEventListener("canplaythrough", () => audio.play()); audio.addEventListener("ended", () => document.body.removeChild(audio)); audio.addEventListener("error", () => document.body.removeChild(audio)); document.body.appendChild(audio); } if (FINISH_NOTIFICATION.value) { new Notification(null, _("queue-finished")); } this.notifiedFinished = true; } addManId(id: number, download: Download) { this.manIds.set(id, download); } removeManId(id: number) { this.manIds.delete(id); } async prepareItems(items: any[]) { var links = new Array(); for (var item of items) { var cookiesToSend = Array(); var cs = await browser.cookies.getAll({ url: item.url, firstPartyDomain: null, }); if (item.cookies) { for (var c of cs){ cookiesToSend.push({ name: c.name, value: c.value, domain: c.domain, // expires: new Date(c.expirationDate * 1000) ?? null, path: c.path, secure: c.secure, httponly: c.httpOnly, // samesite: c.sameSite, }); } } var status = item.paused ? "Paused":"Queue"; links.push({ description: item.description, filename: item.fileName, mask: item.mask, status: status, postData: item.postData, subdir: item.subfolder, title: item.title, url: item.usable, referrer: item.usableReferrer, cookies: cookiesToSend, }); } return links; } addNewDownloads(items: any[]) { if (!items || !items.length) { return; } this.prepareItems(items).then(links => { var xhr = new XMLHttpRequest(); xhr.open("POST", items[0].server); xhr.setRequestHeader("Content-Type", "application/javascript"); xhr.send(JSON.stringify(links)); }); } /* addNewDownloads(items: any[]) { if (!items || !items.length) { return; } items = items.map(i => { const dl = new Download(this, i); dl.position = this.items.push(dl) - 1; this.sids.set(dl.sessionId, dl); dl.markDirty(); return dl; }); Prefs.get("nagging", 0). then(v => { return Prefs.set("nagging", (v || 0) + items.length); }). catch(console.error); this.scheduler = null; this.save(items); this.startNext(); } */ setDirty(item: Download) { this.dirty.add(item); } removeDirty(item: Download) { this.dirty.delete(item); } processDirty(items: Download[]) { items = items.filter(i => !i.removed); items.forEach(item => this.saveQueue.add(item)); this.emit("dirty", items); } private save(items: Download[]) { DB.saveItems(items.filter(i => !i.removed)). catch(console.error); } setPositions() { const items = this.items.filter((e, idx) => { if (e.position === idx) { return false; } e.position = idx; e.markDirty(); return true; }); if (!items.length) { return; } this.save(items); this.resetScheduler(); } forEach(sids: number[], cb: (item: Download) => void) { sids.forEach(sid => { const download = this.sids.get(sid); if (!download) { return; } cb.call(this, download); }); } resumeDownloads(sids: number[], forced = false) { this.forEach(sids, download => download.resume(forced)); } pauseDownloads(sids: number[]) { this.forEach(sids, download => download.pause()); } cancelDownloads(sids: number[]) { this.forEach(sids, download => download.cancel()); } setMissing(sid: number) { this.forEach([sid], download => download.setMissing()); } changedState(download: Download, oldState: number, newState: number) { if (oldState === RUNNING) { this.running.delete(download); } else if (oldState === RETRYING) { this.retrying.delete(download); this.findDeadline(); } if (newState === QUEUED) { this.resetScheduler(); this.startNext().catch(console.error); } else if (newState === RUNNING) { // Usually we already added it. But if a user uses the built-in // download manager to restart // a download, we have not, so make sure it is added either way this.running.add(download); } else { if (newState === RETRYING) { this.addRetry(download); } this.startNext().catch(console.error); } } addRetry(download: Download) { this.retrying.add(download); this.findDeadline(); } private findDeadline() { let deadline = Array.from(this.retrying). reduce((deadline, item) => { if (deadline) { return item.deadline ? Math.min(deadline, item.deadline) : deadline; } return item.deadline; }, 0); if (deadline <= 0) { return; } deadline -= Date.now(); if (deadline <= 0) { return; } if (this.deadlineTimer) { window.clearTimeout(this.deadlineTimer); } this.deadlineTimer = window.setTimeout(this.processDeadlines, deadline); } private processDeadlines() { this.deadlineTimer = 0; try { const now = Date.now(); this.items.forEach(item => { if (item.deadline && Math.abs(item.deadline - now) < 1000) { this.retrying.delete(item); item.resume(false); } }); } finally { this.findDeadline(); } } sorted(sids: number[]) { try { // Construct new items const currentSids = new Map(this.sids); let items = mapFilterInSitu(sids, sid => { const item = currentSids.get(sid); if (!item) { return null; } currentSids.delete(sid); return item; }, e => !!e); if (currentSids.size) { items = items.concat( sort(Array.from(currentSids.values()), i => i.position)); } this.items = items; this.setPositions(); } catch (ex) { console.error("sorted", "sids", sids, "ex", ex.message, ex); } } remove(items: Download[]) { if (!items.length) { return; } items.forEach(item => { item.removed = true; if (!item.manId) { return; } this.removeManId(item.manId); item.cancel(); }); DB.deleteItems(items).then(() => { const sids = items.map(item => item.sessionId); sids.forEach(sid => this.sids.delete(sid)); sort(items.map(item => item.position)). reverse(). forEach(idx => this.items.splice(idx, 1)); this.emit("removed", sids); this.setPositions(); this.resetScheduler(); }).catch(console.error); } removeBySids(sids: number[]) { const items = mapFilterInSitu(sids, sid => this.sids.get(sid), e => !!e); return this.remove(items); } toggleActive() { this.active = !this.active; if (this.active) { this.startNext(); } this.emit("active", this.active); } getMsgItems() { return this.items.map(e => e.toMsg()); } stuffReferrer(details: any): any { if (details.tabId > 0 && !US.startsWith(details.initiator)) { return undefined; } const sidx = details.requestHeaders.findIndex( (e: any) => e.name.toLowerCase() === "x-dta-id"); if (sidx < 0) { return undefined; } const sid = parseInt(details.requestHeaders[sidx].value, 10); details.requestHeaders.splice(sidx, 1); const item = this.sids.get(sid); if (!item) { return undefined; } details.requestHeaders.push({ name: "Referer", value: (item.uReferrer || item.uURL).toString() }); const rv: any = { requestHeaders: details.requestHeaders }; return rv; } } let inited: Promise; export function getManager() { if (!inited) { const man = new Manager(); inited = man.init(); } return inited; }