From 4d953c373f41dd4cd7e514324220cd6439339354 Mon Sep 17 00:00:00 2001 From: Nils Maier Date: Sun, 15 Sep 2019 12:28:31 +0200 Subject: [PATCH] Basic import/export Closes #64 --- _locales/en/messages.json | 20 +++ lib/imex.ts | 247 ++++++++++++++++++++++++++++++++++++++ lib/select.ts | 2 +- windows/manager.html | 9 ++ windows/manager/table.ts | 53 ++++++++ 5 files changed, 330 insertions(+), 1 deletion(-) create mode 100644 lib/imex.ts diff --git a/_locales/en/messages.json b/_locales/en/messages.json index b798a33..bb4478b 100644 --- a/_locales/en/messages.json +++ b/_locales/en/messages.json @@ -323,6 +323,22 @@ "description": "Error Message; select/single window", "message": "Dots (.) in subfolders are not supported by browsers" }, + "export": { + "description": "menu text", + "message": "Export To File" + }, + "export_aria2": { + "description": "menu text", + "message": "Export As aria2 List" + }, + "export_metalink": { + "description": "menu text", + "message": "Export As Metalink" + }, + "export_text": { + "description": "menu text", + "message": "Export As Text" + }, "extensionDescription": { "description": "DownThemAll! tagline, displayed in about:addons; Please do NOT refer to a specific browser such as firefox, as we will probably support more than one", "message": "The Mass Downloader for your browser" @@ -371,6 +387,10 @@ "description": "Menu text", "message": "Force Start" }, + "import": { + "description": "menu text", + "message": "Import From File" + }, "information_title": { "description": "Used in message boxes", "message": "Information" diff --git a/lib/imex.ts b/lib/imex.ts new file mode 100644 index 0000000..1bd0c33 --- /dev/null +++ b/lib/imex.ts @@ -0,0 +1,247 @@ +"use strict"; +// License: MIT + +import { getTextLinks } from "./textlinks"; +// eslint-disable-next-line no-unused-vars +import { BaseItem } from "./item"; +import { ALLOWED_SCHEMES } from "./constants"; + +export const NS_METALINK_RFC5854 = "urn:ietf:params:xml:ns:metalink"; +export const NS_DTA = "http://www.downthemall.net/properties#"; + +function parseNum( + file: Element, + attr: string, + defaultValue: number, + ns = NS_METALINK_RFC5854) { + const val = file.getAttributeNS(ns, attr); + if (!val) { + return defaultValue + 1; + } + const num = parseInt(val, 10); + if (isFinite(num)) { + return num; + } + return defaultValue + 1; +} + +function importMeta4(data: string) { + const parser = new DOMParser(); + const document = parser.parseFromString(data, "text/xml"); + const {documentElement} = document; + const items: BaseItem[] = []; + let batch = 0; + for (const file of documentElement.querySelectorAll("file")) { + try { + const url = Array.from(file.querySelectorAll("url")).map(u => { + try { + const {textContent} = u; + if (!textContent) { + return null; + } + const url = new URL(textContent); + if (!ALLOWED_SCHEMES.has(url.protocol)) { + return null; + } + const prio = parseNum(u, "priority", 0); + return { + url, + prio + }; + } + catch { + return null; + } + }).filter(u => !!u).reduce((p, c) => { + if (!c) { + return null; + } + if (!p || p.prio < c.prio) { + return c; + } + return p; + }); + if (!url) { + continue; + } + batch = parseNum(file, "num", batch, NS_DTA); + const idx = parseNum(file, "idx", 0, NS_DTA); + const item: BaseItem = { + url: url.url.toString(), + usable: decodeURIComponent(url.url.toString()), + batch, + idx + }; + const ref = file.getAttributeNS(NS_DTA, "referrer"); + if (ref) { + item.referrer = ref; + item.usableReferrer = decodeURIComponent(ref); + } + const mask = file.getAttributeNS(NS_DTA, "mask"); + if (mask) { + item.mask = mask; + } + items.push(item); + } + catch (ex) { + console.error("Failed to import file", ex); + } + } + return items; +} + +function parseKV(current: BaseItem, line: string) { + const [k, v] = line.split("=", 2); + switch (k.toLocaleLowerCase().trim()) { + case "referer": { + const rurls = getTextLinks(v); + if (rurls && rurls.length) { + current.referrer = rurls.pop(); + current.usableReferrer = decodeURIComponent(current.referrer || ""); + } + break; + } + } +} + +export function importText(data: string) { + if (data.includes(NS_METALINK_RFC5854)) { + return importMeta4(data); + } + const splitter = /(.+)\n|(.+)$/g; + const spacer = /^\s+/; + let match; + let current: BaseItem | undefined = undefined; + let idx = 0; + const items = []; + while ((match = splitter.exec(data)) !== null) { + try { + const line = match[0].trimRight(); + if (!line) { + continue; + } + if (spacer.test(line)) { + if (!current) { + continue; + } + parseKV(current, line); + continue; + } + const urls = getTextLinks(line); + if (!urls || !urls.length) { + continue; + } + current = { + url: urls[0], + usable: decodeURIComponent(urls[0]), + idx: ++idx + }; + items.push(current); + } + catch (ex) { + current = undefined; + console.error("Failed to import", ex); + } + } + return items; +} + +export interface Exporter { + fileName: string; + getText(items: BaseItem[]): string; +} + +class TextExporter { + readonly fileName: string; + + constructor() { + this.fileName = "links.txt"; + } + + getText(items: BaseItem[]) { + const lines = []; + for (const item of items) { + lines.push(item.url); + } + return lines.join("\n"); + } +} + +class Aria2Exporter { + readonly fileName: string; + + constructor() { + this.fileName = "links.aria2.txt"; + } + + getText(items: BaseItem[]) { + const lines = []; + for (const item of items) { + lines.push(item.url); + if (item.referrer) { + lines.push(` referer=${item.referrer}`); + } + } + return lines.join("\n"); + } +} + +class MetalinkExporter { + readonly fileName: string; + + constructor() { + this.fileName = "links.meta4"; + } + + getText(items: BaseItem[]) { + const document = window.document.implementation. + createDocument(NS_METALINK_RFC5854, "metalink", null); + const root = document.documentElement; + root.setAttributeNS(NS_DTA, "generator", "DownThemAll!"); + root.appendChild(document.createComment( + "metalink as exported by DownThemAll!", + )); + + for (const item of items) { + const aitem = item as any; + const f = document.createElementNS(NS_METALINK_RFC5854, "file"); + f.setAttribute("name", aitem.currentName); + if (item.batch) { + f.setAttributeNS(NS_DTA, "num", item.batch.toString()); + } + if (item.idx) { + f.setAttributeNS(NS_DTA, "idx", item.idx.toString()); + } + if (item.referrer) { + f.setAttributeNS(NS_DTA, "referrer", item.referrer); + } + if (item.mask) { + f.setAttributeNS(NS_DTA, "mask", item.mask); + } + + if (item.description) { + const n = document.createElementNS(NS_METALINK_RFC5854, "description"); + n.textContent = item.description; + f.appendChild(n); + } + + const u = document.createElementNS(NS_METALINK_RFC5854, "url"); + u.textContent = item.url; + f.appendChild(u); + + if (aitem.totalSize > 0) { + const s = document.createElementNS(NS_METALINK_RFC5854, "size"); + s.textContent = aitem.totalSize.toString(); + f.appendChild(s); + } + root.appendChild(f); + } + let xml = "\n"; + xml += root.outerHTML; + return xml; + } +} + +export const textExporter = new TextExporter(); +export const aria2Exporter = new Aria2Exporter(); +export const metalinkExporter = new MetalinkExporter(); diff --git a/lib/select.ts b/lib/select.ts index de6cac3..77ffba6 100644 --- a/lib/select.ts +++ b/lib/select.ts @@ -28,7 +28,7 @@ function computeSelection( items: BaseMatchedItem[], onlyFast: boolean): ItemDelta[] { let ws = items.map((item, idx: number) => { - item.idx = idx; + item.idx = item.idx || idx; const {matched = null} = item; item.prevMatched = matched; item.matched = null; diff --git a/windows/manager.html b/windows/manager.html index 78089bf..57f13b2 100644 --- a/windows/manager.html +++ b/windows/manager.html @@ -110,6 +110,15 @@
  • Select All
  • Invert Selection
  • -
    • +
  • +
  • + +
    • +
  • -
  • Top
  • Up
  • down
    • diff --git a/windows/manager/table.ts b/windows/manager/table.ts index 649ec4a..d7d71ff 100644 --- a/windows/manager/table.ts +++ b/windows/manager/table.ts @@ -38,6 +38,10 @@ import { $ } from "../winutil"; // eslint-disable-next-line no-unused-vars import { TableConfig } from "../../uikit/lib/config"; import { IconCache } from "../../lib/iconcache"; +import * as imex from "../../lib/imex"; +// eslint-disable-next-line no-unused-vars +import { BaseItem } from "../../lib/item"; +import { API } from "../../lib/api"; const TREE_CONFIG_VERSION = 2; const RUNNING_TIMEOUT = 1000; @@ -556,6 +560,12 @@ export class DownloadTable extends VirtualTable { ctx.on("ctx-remove-paused", () => this.removePausedDownloads()); ctx.on("ctx-remove-batch", () => this.removeBatchDownloads()); + ctx.on("ctx-import", () => this.importDownloads()); + ctx.on("ctx-export-text", () => this.exportDownloads(imex.textExporter)); + ctx.on("ctx-export-aria2", () => this.exportDownloads(imex.aria2Exporter)); + ctx.on("ctx-export-metalink", + () => this.exportDownloads(imex.metalinkExporter)); + ctx.on("dismissed", () => this.table.focus()); this.on("contextmenu", (tree, event) => { @@ -1177,6 +1187,49 @@ export class DownloadTable extends VirtualTable { this.selection.toggle(0, this.rowCount - 1); } + importDownloads() { + const picker = document.createElement("input"); + picker.setAttribute("type", "file"); + picker.setAttribute("accept", "text/*,.txt,.lst,.metalink,.meta4"); + picker.onchange = () => { + if (!picker.files || !picker.files.length) { + return; + } + const reader = new FileReader(); + reader.onload = () => { + if (!reader.result) { + return; + } + const items = imex.importText(reader.result as string); + if (!items || !items.length) { + return; + } + API.regular(items, []); + }; + reader.readAsText(picker.files[0], "utf-8"); + }; + picker.click(); + } + + exportDownloads(exporter: imex.Exporter) { + const items = this.getSelectedItems(); + if (!items.length) { + return; + } + const text = exporter.getText(items as unknown as BaseItem[]); + const enc = new TextEncoder(); + const data = enc.encode(text); + const url = URL.createObjectURL(new Blob([data], {type: "text/plain"})); + const link = document.createElement("a"); + link.setAttribute("href", url); + link.setAttribute("download", exporter.fileName); + link.style.display = "none"; + document.body.appendChild(link); + link.click(); + document.body.removeChild(link); + URL.revokeObjectURL(url); + } + getRowClasses(rowid: number) { const item = this.downloads.filtered[rowid]; if (!item) {