Add sublime-project file

Switch to using bootstrap-table
Add DownThemAll plugin
Remove Firefox plugin

assets/bootstrap-table-sticky-header.css

@@ -0,0 +1,21 @@
+/**
+ * @author vincent loh <vincent.ml@gmail.com>
+ * @update zhixin wen <wenzhixin2010@gmail.com>
+ */
+.fix-sticky {
+  position: fixed !important;
+  overflow: hidden;
+  z-index: 100;
+}
+
+.fix-sticky table thead {
+  background: #fff;
+}
+
+.fix-sticky table thead.thead-light {
+  background: #e9ecef;
+}
+
+.fix-sticky table thead.thead-dark {
+  background: #212529;
+}

assets/downloads.js

@@ -0,0 +1,218 @@
+var myCheckedData = [];
+var mySavedData = [];
+
+function deleteDownload(data){
+	var xhr = new XMLHttpRequest();
+	xhr.open("DELETE", "/delete");
+	xhr.setRequestHeader("Content-Type", "application/javascript");
+	xhr.send(JSON.stringify(data));
+}
+
+function setDownload(data){
+	var xhr = new XMLHttpRequest();
+	xhr.open("POST", "/set");
+	xhr.setRequestHeader("Content-Type", "application/javascript");
+	xhr.send(JSON.stringify(data));
+}
+
+function addDownload(data) {
+	var xhr = new XMLHttpRequest();
+	xhr.open("POST", "/add");
+	xhr.setRequestHeader("Content-Type", "application/javascript");
+	xhr.send(JSON.stringify(data));
+}
+
+function filename(value,download) {
+	const path = (download.subdir + '/' + download.filename).replace(/^\//, "");
+	ret = $("<span></span>").text(path).attr("title", download.filepath);
+	if (download.status == "Complete") {
+		ret = $('<a></a>').attr("href", 'get/' + path).text(path).attr("title", download.filepath);
+	}
+	return ret.prop("outerHTML");
+}
+function status(value,download) {
+	return $('<span></span>').addClass(value).text(value).prop('outerHTML') + $("<span></span>").addClass(value).text(download.error).prop('outerHTML');
+}
+
+function actions(value,download) {
+	var action = $("<div></div>").addClass("dropdown");
+	var button = $("<button>Actions</button>").addClass("btn btn-secondary dropdown-toggle").attr("type", "button").attr("data-toggle", "dropdown");
+	var menu = $("<div></div>").addClass("dropdown-menu");
+
+	var del = $('<a href="#">Delete</a>').addClass("dropdown-item action-delete");
+	var resume = $('<a href="#">Resume</a>').addClass("dropdown-item action-resume");
+	var stop = $('<a href="#">Stop</a>').addClass("dropdown-item action-stop");
+	var retrieve = "";
+	if (download.status == "Complete") {
+		retrieve = $('<a>retrieve</a>').addClass("dropdown-item action-retrieve").attr("href", 'get/' + (download.subdir + '/' + download.filename).replace(/^\//, "")).attr("title", download.filepath);
+	}
+
+	menu.append(del, resume, stop, retrieve);
+	action.append(button, menu);
+	return action.prop('outerHTML');
+}
+
+function progress(value,download) {
+	if (download.status == "Complete") {
+		download.progress = 100;
+	}
+	var progress = $("<div></div>").addClass("progress");
+	var bar = $("<div></div>").addClass("progress-bar text-dark").css("width", (download.progress ?? "0") + '%')
+	.text((download.progress ?? "0") + '%');
+	progress.append(bar)
+	return progress.prop('outerHTML');
+}
+
+function actionsbulk() {
+	var action = $("<div></div>").addClass("dropdown").attr("title", "Actions");
+	var button = $("<button>Actions</button>").addClass("btn btn-secondary dropdown-toggle").attr("type", "button").attr("data-toggle", "dropdown");
+	var menu = $("<div></div>").addClass("dropdown-menu");
+
+	var del = $('<a href="#">Delete</a>').addClass("dropdown-item action-delete").attr("onclick", "bulkDelete();");
+	var resume = $('<a href="#">Resume</a>').addClass("dropdown-item action-resume").attr("onclick", "bulkResume();");
+	var stop = $('<a href="#">Stop</a>').addClass("dropdown-item action-stop").attr("onclick", "bulkStop();");
+	var retrieve = $('<a href="#">retrieve</a>').addClass("dropdown-item action-retrieve").attr("onclick", "bulkRetrieve();");
+
+	menu.append(del, resume, stop, retrieve);
+	action.append(button, menu);
+	return action.prop('outerHTML');
+}
+
+var actionEvents = {
+	'click .action-delete': function (e, value, download, index) {
+		if (confirm("confirm deletion of " + download.filename)) {
+			deleteDownload({
+				requests: [{
+					url: download.url
+				}],
+				history: IsHistory,
+			});
+		}
+	},
+	'click .action-resume': function(e, value, download, index) {
+		setDownload([{
+			url: download.url,
+			status: "Queue",
+			priority: download.priority
+		}]);
+	},
+	'click .action-stop': function(e, value, download, index) {
+		setDownload([{
+			url: download.url,
+			status: "Stop",
+			priority: download.priority
+		}]);
+	},
+}
+function buttons () {
+	return {
+		actions: {
+			html:actionsbulk,
+		},
+		checkAll: {
+			icon: "fa-check-square",
+			text: "Check All",
+			attributes: {title: "Check All"},
+			event:{
+				'click': function() {
+					var table = $("#table");
+					var downloads = table.bootstrapTable('getData', {useCurrentPage: false, includeHiddenRows: true, unfiltered: false});
+					for (var d of downloads){
+						d.checked = true;
+					}
+					table.bootstrapTable('prepend', {});
+					table.bootstrapTable('remove', { field: '$index', values: [0]});
+				}
+			}
+		},
+		refresh: {
+        event: function() {$('#table').bootstrapTable('refresh',{silent:true});},
+      },
+	};
+}
+function bulkResume() {
+	var selectedDownloads = $("#table").bootstrapTable('getSelections');
+	for (var d of selectedDownloads){
+		d.status = "Queue";
+	}
+	setDownload(selectedDownloads);
+}
+function bulkStop() {
+	var selectedDownloads = $("#table").bootstrapTable('getSelections');
+	for (var d of selectedDownloads){
+		d.status = "Stop";
+	}
+	setDownload(selectedDownloads);
+}
+function bulkDelete() {
+	var selectedDownloads = $("#table").bootstrapTable('getSelections');
+	if (confirm("confirm deletion of:\n" + selectedDownloads.map(d => d.filename).join('\n'))) {
+		deleteDownload({
+			requests: selectedDownloads,
+			history: IsHistory,
+		});
+	}
+}
+
+function bulkRetrieve(){
+	var selectedDownloads = $("#table").bootstrapTable('getSelections');
+	formPostJSON(selectedDownloads, "/get/", "discard")
+}
+
+// o must be an object or an array of objects
+// discardFieldName must be a valid json field name
+// endpoint must accept Content-Type: text/plain
+// awesome hack taken from https://systemoverlord.com/2016/08/24/posting-json-with-an-html-form.html
+function formPostJSON(o, url, discardFieldName='discard') {
+	var form = $("<form></form>").attr("method", "POST").attr("enctype", "text/plain").attr("action", url);
+	var json;
+	if (Array.isArray(o)) {
+		var name = JSON.stringify(o).slice(0,-2) +',"' + discardFieldName + '":"';
+		var value = '"}]';
+	} else {
+		var name = JSON.stringify(o).slice(0,-1) +',"'+ discardFieldName + '":"';
+		var value = '"}';
+	}
+	var input = $("<input></input>").attr("name", name).attr("value", value).attr("type", "hidden");
+	form.append(input);
+	$(document.body).append(form);
+	form.submit();
+	form.remove();
+}
+
+function SaveChecked() {
+	myCheckedData = $("#table").bootstrapTable('getSelections').map( d => d.url );
+	mySavedData = $("#table").bootstrapTable('getData', {useCurrentPage: false, includeHiddenRows: true, unfiltered: true});
+}
+
+function LoadChecked(downloads) {
+	// console.log("data loaded");
+	var loaded = [];
+	for (var i = downloads.length - 1; i >= 0; i--) {
+		if (myCheckedData.includes(downloads[i].url)) {
+			downloads[i].checked = true;
+			loaded.push(downloads[i]);
+		}
+	}
+	mySavedData = $("#table").bootstrapTable('getData', {useCurrentPage: false, includeHiddenRows: true, unfiltered: true});
+					$('#table').bootstrapTable('prepend', {});
+					$('#table').bootstrapTable('remove', { field: '$index', values: [0]});
+	// console.log(loaded);
+}
+
+function LoadSaved() {
+	$('#table').bootstrapTable('load', mySavedData);
+}
+
+document.addEventListener("DOMContentLoaded", function(event) {
+	$('#table').bootstrapTable({
+		onCheck: SaveChecked,
+		onCheckAll: SaveChecked,
+		onCheckSome: SaveChecked,
+		onUncheck: SaveChecked,
+		onUncheckAll: SaveChecked,
+		onUncheckSome: SaveChecked,
+		onLoadSuccess: LoadChecked,
+		onLoadError: LoadSaved,
+	});
+});

assets/footer.tmpl

@@ -0,0 +1,11 @@
+
+</div>
+
+<script src="/jquery-3.2.1.min.js"></script>
+<script src="/popper.min.js"></script>
+<script src="/bootstrap.min.js"></script>
+<script src="/bootstrap-table.min.js"></script>
+<script src="/bootstrap-table-auto-refresh.min.js"></script>
+<script src="/bootstrap-table-sticky-header.min.js"></script>
+
+</html>

assets/header.tmpl

@@ -0,0 +1,102 @@
+<!DOCTYPE html>
+<html lang="en">
+<title>{{ .Hostname }} — gloader</title>
+<link rel="stylesheet" href="/bootstrap.min.css" />
+<link rel="stylesheet" href="/bootstrap-table.min.css" />
+<link rel="stylesheet" href="/bootstrap-table-sticky-header.css" />
+<link rel="stylesheet" href="/fontawesome.css" />
+<style type="text/css">
+.progress-bar:nth-child(5n) {
+        background-color: #337ab7;
+}
+.progress-bar:nth-child(5n+1) {
+        background-color: #5cb85c;
+}
+.progress-bar:nth-child(5n+2) {
+        background-color: #5bc0de;
+}
+.progress-bar:nth-child(5n+3) {
+        background-color: #f0ad4e;
+}
+.progress-bar:nth-child(5n+4) {
+        background-color: #d9534f;
+}
+.lastlog {
+        text-overflow: ellipsis;
+        white-space: nowrap;
+        overflow: hidden;
+}
+.Error {
+  color: red;
+}
+
+.container-fluid {
+  padding-left: 45px;
+}
+
+.navbar-default .navbar-text {
+  color: #777;
+}
+
+.navbar-default {
+  background-color: #f8f8f8;
+  border-color: #e7e7e7;
+}
+
+.Queued {
+
+}
+.Complete {
+
+}
+.Stopped {
+
+}
+.Paused {
+
+}
+.Downloading {
+
+}
+.Error {
+
+}
+.Canceled {
+
+}
+
+body {
+  font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
+}
+</style>
+
+  <nav class="navbar navbar-default">
+        <p style="margin-top: 0.25em; font-size: 18px"><a href="/">gokrazy</a><br>
+        <!-- <small style="font-size: 11px" class="text-muted">version { .BuildTimestamp }}</small> --></p>
+
+       <table class="navbar-text navbar-right" style="border-spacing: 10px 0; border-collapse: separate">
+    <tr>
+      <th>host</th>
+      <td>{{ .Hostname }}</td>
+    </tr>
+    <!-- <tr>
+      <th>kernel</th>
+      <td>{ .Kernel }}</td>
+    </tr>
+    { if (ne .Model "") }}
+    <tr>
+      <th>model</th>
+      <td>{ .Model }}</td>
+    </tr>
+    { end }}
+    { if .EEPROM }}
+    <tr>
+      <th>EEPROM<br>(SHA256)</th>
+      <td>{ shortenSHA256 .EEPROM.PieepromSHA256 }}<br>{ shortenSHA256 .EEPROM.VL805SHA256 }}</td>
+    </tr>
+    { end }} -->
+  </table>
+
+  </nav>
+
+  <div class="container-fluid">

assets/queue.tmpl

@@ -0,0 +1,64 @@
+{{ template "header" . }}
+
+<div >
+<script type="text/javascript">
+	var IsHistory = {{ .IsHistory }};
+</script>
+	<!-- <h1>services</h1> -->
+
+	<input type="text" id="addurl"/><button onclick="addDownload([{url:document.getElementById('addurl').value}]);">Add URL</button>
+
+{{ if .IsHistory }}
+	<a href="../">Goto Queue</a>
+{{ else }}
+	<a href="history">Goto History</a>
+{{ end }}
+
+	<table id="table" data-toggle="table" class="table"
+{{ if .IsHistory }}
+		data-url="../completed"
+		data-auto-refresh-interval="30"
+{{ else }}
+		data-url="queued"
+		data-auto-refresh-interval="5"
+{{ end }}
+		data-auto-refresh="true"
+		data-buttons-order="['actions', 'checkAll', 'columns', 'paginationSwitch', 'refresh']"
+		data-buttons="buttons"
+		data-cache="false"
+		data-click-to-select="true"
+		data-maintain-meta-data="true"
+		data-multiple-select-row="true"
+		data-page-list="[10, 25, 50, 100, 200, 300, 400, 500, all]"
+		data-pagination-loop="false"
+		data-pagination="true"
+		data-search="true"
+		data-show-columns="true"
+		data-show-pagination-switch="true"
+		data-show-refresh="true"
+		data-sort-reset="true"
+		data-sort-stable="true"
+		data-sticky-header="true"
+		>
+	<thead>
+		<tr>
+			<th data-field="checked" data-checkbox="true"></th>
+			<th data-sortable="true" scope="col" data-field="status" data-formatter="status">Status</th>
+			<th data-sortable="true" scope="col" data-field="priority">Priority</th>
+			<th data-sortable="true" scope="col" data-field="filename" data-formatter="filename">Filename</th>
+			<th data-sortable="true" scope="col" data-field="url">URL</th>
+			<th scope="col" data-field="actions" data-formatter="actions" data-events="actionEvents">Actions</th>
+{{ if .IsHistory }}
+				<th data-visible="false" scope="col" data-field="progress" data-formatter="progress">Progress</th>
+{{ else }}
+				<th scope="col" data-field="progress" data-formatter="progress">Progress</th>
+{{ end }}
+		</tr>
+	</thead>
+</table>
+
+</div>
+
+<script src="/downloads.js"></script>
+
+{{ template "footer" . }}

assets/status.tmpl

@@ -0,0 +1,47 @@
+{{ template "header" . }}
+
+<div class="row">
+<div class="col-md-12">
+<table>
+<tr>
+<th>Name</th>
+<th>Started</th>
+<th>Actions</th>
+</tr>
+<tr>
+<td><a href="#{{ .Service.Name }}">{{ .Service.Name }}</a></td>
+<td>{{ .Service.Started }}</td>
+<td>
+  <form method="POST" action="/restart">
+    <input type="hidden" name="xsrftoken" value="{{ .XsrfToken }}">
+    <input type="hidden" name="path" value="{{ .Service.Name }}">
+    <input type="submit" value="restart">
+  </form>
+  <form method="POST" action="/stop">
+    <input type="hidden" name="xsrftoken" value="{{ .XsrfToken }}">
+    <input type="hidden" name="path" value="{{ .Service.Name }}">
+    <input type="submit" value="stop">
+  </form></td>
+</tr>
+</table>
+
+  <h3>module info</h3>
+  <pre>{{ .Service.ModuleInfo }}</pre>
+
+  <h3>stdout</h3>
+  <pre>
+  {{ range $idx, $line := .Service.Stdout.Lines -}}
+    {{ $line }}
+  {{ end }}
+  </pre>
+
+  <h3>stderr</h3>
+  <pre>
+  {{ range $idx, $line := .Service.Stderr.Lines -}}
+    {{ $line }}
+  {{ end }}
+  </pre>
+</div>
+</div>
+
+{{ template "footer" . }}

bundle.go

@@ -0,0 +1,3 @@
+package main
+
+//go:generate sh -c "go run goembed.go -package bundled -var assets assets/fontawesome.css assets/fa-solid-900.woff2 assets/bootstrap.min.css assets/bootstrap.min.js assets/bootstrap-table-auto-refresh.min.js assets/bootstrap-table.min.css assets/bootstrap-table.min.js assets/bootstrap-table-sticky-header.css assets/bootstrap-table-sticky-header.min.js assets/downloads.js assets/favicon.ico assets/footer.tmpl assets/header.tmpl assets/jquery-3.2.1.min.js assets/popper.min.js assets/queue.tmpl assets/status.tmpl > bundled/GENERATED_bundled.go && gofmt -w bundled/GENERATED_bundled.go"

bundled/bundled.go

@@ -0,0 +1,18 @@
+package bundled
+
+import (
+	"bytes"
+	"net/http"
+	"time"
+)
+
+func Asset(basename string) string {
+	return string(assets["assets/"+basename])
+}
+
+func HTTPHandlerFunc(basename string) http.Handler {
+	modTime := time.Now()
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.ServeContent(w, r, basename, modTime, bytes.NewReader(assets["assets/"+basename]))
+	})
+}

downloader.go

@@ -1,11 +1,17 @@
 package main
 
 import (
+	"archive/tar"
+	"bytes"
+	"context"
 	"encoding/json"
 	"errors"
 	"fmt"
+	"html/template"
+	"io"
 	"io/ioutil"
 	"log"
+	"math"
 	"mime"
 	"net"
 	"net/http"
@@ -19,21 +25,23 @@ import (
 	"strings"
 	"time"
 
+	"git.narnian.us/lordwelch/gloader/bundled"
+
 	"github.com/cavaliercoder/grab"
 	"github.com/lordwelch/pathvalidate"
+	"github.com/wI2L/jettison"
+
 	"golang.org/x/net/publicsuffix"
 )
 
 var (
-	DefaultCookieJar          = newCookieJar()
-	DefaultGrabClient         = grab.NewClient()
 	DefaultMaxActiveDownloads = 4
 
 	ErrUnsupportedScheme = errors.New("unsupported scheme")
 )
 
-type Priority uint8
-type Status uint8
+type Priority int
+type Status int
 
 const (
 	Highest Priority = iota
@@ -57,31 +65,35 @@ type Downloader struct {
 	DownloadDir        string
 	CompleteDir        string
 	InfoDir            string
-	Grab               *grab.Client
-	Jar                http.CookieJar
 	MaxActiveDownloads int
 	Server             *http.Server
 	Downloads          RequestQueue
 	History            RequestQueue
 	NewRequest         chan Request
 	requestDone        chan *Request
-	OnComplete         func(r Request)
-	OnAdd              func(r Request)
+	OnComplete         func(d *Downloader, r Request)
+	OnAdd              func(d *Downloader, r Request)
+	OnDelete           func(d *Downloader, r Request)
 }
 
 type Request struct {
 	URL           string         `json:"url"`
 	Cookies       []http.Cookie  `json:"cookies"`
 	ForceDownload bool           `json:"forceDownload"`
-	Status        Status         `json:"Status"`
+	Status        Status         `json:"status"`
 	Priority      Priority       `json:"priority"`
 	FilePath      string         `json:"filepath"`
 	Filename      string         `json:"filename"`
 	Subdir        string         `json:"subdir"`
 	TempPath      string         `json:"tempPath"`
 	Response      *grab.Response `json:"-"`
-	Error         error          `json:"-"`
-	CompletedDate time.Time      `json:"completedDate"`
+	Error         string         `json:"error"`
+	Err           error          `json:"-"`
+	Completed     time.Time      `json:"completed"`
+	Started       time.Time      `json:"started"`
+	Jar           http.CookieJar `json:"-"`
+	Progress      string         `json:"progress,omitempty"`
+	grab          *grab.Client
 }
 
 type RequestQueue struct {
@@ -90,6 +102,93 @@ type RequestQueue struct {
 	DateSort bool
 }
 
+func (p Priority) MarshalJSON() ([]byte, error) {
+	var v string
+	switch p {
+	default:
+		v = "Medium"
+	case Medium:
+		v = "Medium"
+	case Low:
+		v = "Low"
+	case High:
+		v = "High"
+	case Highest:
+		v = "Highest"
+	}
+	return json.Marshal(v)
+}
+
+func (p *Priority) UnmarshalJSON(b []byte) error {
+	var (
+		v int
+		s string
+	)
+	if err := json.Unmarshal(b, &v); err == nil {
+		*p = Priority(v)
+		return nil
+	}
+	json.Unmarshal(b, &s)
+	switch strings.ToLower(s) {
+	default:
+		*p = Medium
+	case "medium":
+		*p = Medium
+	case "low":
+		*p = Low
+	case "high":
+		*p = High
+	case "highest":
+		*p = Highest
+	}
+	return nil
+}
+
+func (s *Status) UnmarshalJSON(b []byte) error {
+	var v string
+	json.Unmarshal(b, &v)
+	switch strings.ToLower(v) {
+	default:
+		*s = Queued
+	case "queue", "queued":
+		*s = Queued
+	case "complete", "completed":
+		*s = Complete
+	case "stop", "stopped":
+		*s = Stopped
+	case "download", "downloading":
+		*s = Downloading
+	case "error":
+		*s = Error
+	case "cancel", "canceled":
+		*s = Canceled
+	}
+	return nil
+}
+
+func (s Status) MarshalJSON() ([]byte, error) {
+	return json.Marshal(s.String())
+}
+
+func (s Status) String() string {
+	switch s {
+	default:
+		return "Queued"
+	case Queued:
+		return "Queued"
+	case Complete:
+		return "Complete"
+	case Stopped:
+		return "Stopped"
+	case Downloading:
+		return "Downloading"
+	case Error:
+		return "Error"
+	case Canceled:
+		return "Canceled"
+	}
+}
+
 func (rq RequestQueue) Less(i, j int) bool {
 	ii := 0
 	jj := 0
@@ -107,7 +206,7 @@ func (rq RequestQueue) Less(i, j int) bool {
 		return true
 	}
 
-	if rq.DateSort && rq.Queue[i].CompletedDate.Before(rq.Queue[j].CompletedDate) {
+	if rq.DateSort && rq.Queue[i].Completed.Before(rq.Queue[j].Completed) {
 		return true
 	}
 
@@ -134,15 +233,61 @@ func (rq *RequestQueue) Pop(i int) *Request {
 	return r
 }
 
-func (rq *RequestQueue) remove(r *Request) {
+func (rq *RequestQueue) find(url string) *Request {
+	for _, req := range rq.Queue {
+		if req.URL == url {
+			return req
+		}
+	}
+	return nil
+}
+
+func (rq *RequestQueue) remove(url string) *Request {
 	for i, req := range rq.Queue {
-		if req == r {
+		if req.URL == url {
 			copy(rq.Queue[i:], rq.Queue[i+1:])
 			rq.Queue[len(rq.Queue)-1] = nil
 			rq.Queue = rq.Queue[:len(rq.Queue)-1]
-			break
+			return req
 		}
 	}
+	return nil
+}
+
+func (rq *RequestQueue) updateStatus() {
+	for _, req := range rq.Queue {
+		if req.Response != nil {
+			req.Progress = fmt.Sprintf("%.2f", math.Abs(req.Response.Progress()*100))
+		}
+	}
+}
+
+func (r *Request) setError(err error) {
+	if err != nil {
+		r.Status = Error
+		r.Err = err
+		r.Error = err.Error()
+	} else {
+		r.Status = Paused
+		r.Err = nil
+		r.Error = ""
+	}
+}
+
+func (r Request) Delete() error {
+	var err, ret error
+	if r.Response != nil {
+		err = r.Response.Cancel()
+		if err != nil && err != context.Canceled {
+			ret = err
+		}
+	}
+	_ = os.Remove(r.TempPath)
+	err = os.Remove(r.FilePath)
+	if err != nil && err != os.ErrNotExist {
+		ret = err
+	}
+	return ret
 }
 
 func newCookieJar() http.CookieJar {
@@ -150,13 +295,6 @@ func newCookieJar() http.CookieJar {
 	return c
 }
 
-func newDownloader() *Downloader {
-	return &Downloader{
-		Jar:  DefaultCookieJar,
-		Grab: DefaultGrabClient,
-	}
-}
-
 func (d *Downloader) Start(network, address string) {
 	var (
 		listener net.Listener
@@ -183,7 +321,6 @@ func (d *Downloader) Start(network, address string) {
 			ReadTimeout:  2 * time.Minute,
 			WriteTimeout: 2 * time.Minute,
 		}
-
 	}
 
 	if d.DataDir == "" {
@@ -209,26 +346,17 @@ func (d *Downloader) Start(network, address string) {
 	if err != nil {
 		panic(err)
 	}
-	log.Println("adding /add handler")
-	// mux.HandleFunc("/", d.UI)
+	log.Println("adding http handlers")
+
+	d.initStatus(mux)
 	mux.HandleFunc("/add", d.restAddDownload)
-	mux.HandleFunc("/queue", d.restQueueStatus)
-	mux.HandleFunc("/history", d.restHistoryStatus)
-	mux.HandleFunc("/start", d.restStartDownload)
+	mux.HandleFunc("/queued", d.restStatus(true))
+	mux.HandleFunc("/completed", d.restStatus(false))
+	mux.HandleFunc("/set", d.restSetDownloadStatus)
+	mux.HandleFunc("/delete", d.restDelete)
+	mux.Handle("/get/", http.StripPrefix("/get", http.HandlerFunc(d.get)))
 
 	log.Println("starting main go routine")
-	d.Grab.HTTPClient = &http.Client{
-		Jar: d.Jar,
-		Transport: &http.Transport{
-			Dial: (&net.Dialer{
-				Timeout:   10 * time.Second,
-				KeepAlive: 30 * time.Second,
-			}).Dial,
-			TLSHandshakeTimeout:   5 * time.Second,
-			ResponseHeaderTimeout: 5 * time.Second,
-			ExpectContinueTimeout: 1 * time.Second,
-		},
-	}
 
 	go d.download()
 
@@ -236,64 +364,235 @@ func (d *Downloader) Start(network, address string) {
 	_ = d.Server.Serve(listener)
 }
 
-func (d *Downloader) restStartDownload(w http.ResponseWriter, r *http.Request) {
-	var (
-		err   error
-		index struct {
-			index int
-		}
-	)
-	if r.Method != http.MethodPost {
-		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
-		w.Header().Set("X-Content-Type-Options", "nosniff")
-		w.Header().Add("Allow", http.MethodPost)
-		w.WriteHeader(http.StatusMethodNotAllowed)
-		fmt.Fprintln(w, "HTTP Error 405 – Method Not Allowed\nOnly POST method is allowed")
-		log.Println("HTTP Error 405 – Method Not Allowed\nOnly POST method is allowed")
+func httpMethodNotAllowed(w http.ResponseWriter, method ...string) {
+	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+	w.Header().Set("X-Content-Type-Options", "nosniff")
+	w.Header().Add("Allow", strings.Join(method, ", "))
+	w.WriteHeader(http.StatusMethodNotAllowed)
+	fmt.Fprintf(w, "HTTP Error 405 – Method Not Allowed\nOnly %s method(s) is allowed\n", strings.Join(method, ", "))
+	log.Printf("HTTP Error 405 – Method Not Allowed\nOnly %s method(s) is allowed\n", strings.Join(method, ", "))
+}
+
+func (d *Downloader) get(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodGet && r.Method != http.MethodPost {
+		httpMethodNotAllowed(w, http.MethodPost, http.MethodGet)
 		return
 	}
-	err = json.NewDecoder(r.Body).Decode(index)
+	if r.Method == http.MethodGet {
+		var file = filepath.Join(d.CompleteDir, strings.TrimLeft(filepath.Clean(r.URL.Path), "./"))
+		http.ServeFile(w, r, file)
+		return
+	}
+	if r.Method == http.MethodPost {
+		var (
+			downloads []struct {
+				Subdir   string
+				Filename string
+			}
+			filenames []string
+			err       error
+		)
+		err = json.NewDecoder(r.Body).Decode(&downloads)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusBadRequest)
+			return
+		}
+
+		filenames = make([]string, 0, len(downloads))
+
+		for _, name := range downloads {
+			var (
+				subdir   = strings.TrimLeft(filepath.Clean(name.Subdir), "./")
+				filename = strings.TrimLeft(filepath.Clean(name.Filename), "./")
+			)
+			filenames = append(filenames, filepath.Join(d.CompleteDir, subdir, filename))
+		}
+		tarFiles(w, filenames...)
+		return
+	}
+}
+func tarFiles(w http.ResponseWriter, files ...string) {
+	var (
+		err  error
+		size int64
+	)
+
+	size, err = DirSize(files...)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	w.Header().Add("content-length", fmt.Sprintf("%v", size))
+	w.Header().Add("content-disposition", fmt.Sprintf("attachment; filename=\"downloads_%v.tar\"", time.Now().Format("01-02-2006_15-04-05")))
+	w.Header().Add("content-type", "application/x-tar")
+
+	tw := tar.NewWriter(w)
+	defer tw.Close()
+	var tarLength int64
+	for _, filename := range files {
+		var (
+			info os.FileInfo
+			file io.ReadCloser
+		)
+		info, err = os.Stat(filename)
+		if err != nil {
+			break
+		}
+		err = tw.WriteHeader(&tar.Header{
+			Name: info.Name(),
+			Mode: 0777,
+			Size: info.Size(),
+		})
+		if err != nil {
+			break
+		}
+		file, err = os.OpenFile(filename, os.O_RDONLY, 0o666)
+		if err != nil {
+			break
+		}
+		var data int64
+		data, err = io.Copy(tw, file)
+		tarLength += data
+		file.Close()
+		if err != nil {
+			break
+		}
+	}
+}
+
+func DirSize(files ...string) (int64, error) {
+	var size int64
+	var err error
+
+	for _, filename := range files {
+		var (
+			info os.FileInfo
+		)
+		info, err = os.Stat(filename)
+		if err != nil {
+			break
+		}
+		size += info.Size() + 512
+		remainder := size % 512
+		if remainder > 0 {
+			size += 512 - remainder
+		}
+	}
+	size += 1024
+	return size, err
+}
+
+func (d *Downloader) restDelete(w http.ResponseWriter, r *http.Request) {
+	var (
+		err       error
+		downloads struct {
+			Requests []struct {
+				URL string
+			}
+			History bool
+		}
+		ret []*Request
+	)
+	if r.Method != http.MethodDelete {
+		httpMethodNotAllowed(w, http.MethodDelete)
+		return
+	}
+	err = json.NewDecoder(r.Body).Decode(&downloads)
 	if err != nil {
 		http.Error(w, err.Error(), http.StatusBadRequest)
 		return
 	}
-	if index.index >= d.Downloads.Len() || index.index < 0 {
-		http.Error(w, fmt.Sprintf("slice index out of bounds. index: %d length of slice: %d", index.index, d.Downloads.Len()), http.StatusBadRequest)
-		return
+	sort.Slice(downloads.Requests, func(i, j int) bool {
+		return downloads.Requests[i].URL < downloads.Requests[j].URL
+	})
+	if downloads.History {
+		for _, i := range downloads.Requests {
+			ret = append(ret, d.History.remove(i.URL))
+			ret[len(ret)-1].Delete()
+			if d.OnDelete != nil {
+				d.OnDelete(d, *ret[len(ret)-1])
+			}
+		}
+	} else {
+		for _, i := range downloads.Requests {
+			ret = append(ret, d.Downloads.remove(i.URL))
+			ret[len(ret)-1].Delete()
+			if d.OnDelete != nil {
+				d.OnDelete(d, *ret[len(ret)-1])
+			}
+		}
 	}
-	d.startDownload(index.index)
-}
+	d.syncDownloads()
 
-func (d *Downloader) restHistoryStatus(w http.ResponseWriter, r *http.Request) {
-	if r.Method != http.MethodGet {
-		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
-		w.Header().Set("X-Content-Type-Options", "nosniff")
-		w.Header().Add("Allow", http.MethodGet)
-		w.WriteHeader(http.StatusMethodNotAllowed)
-		fmt.Fprintln(w, "HTTP Error 405 – Method Not Allowed\nOnly GET method is allowed")
-		log.Println("HTTP Error 405 – Method Not Allowed\nOnly GET method is allowed")
+	v, err := jettison.MarshalOpts(ret, jettison.DenyList([]string{"cookies"}))
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
 	}
-	j := json.NewEncoder(w)
 	w.Header().Add("Content-Type", "application/json; charset=utf-8")
 	w.WriteHeader(http.StatusOK)
-	j.Encode(d.History.Queue)
+	w.Write(v)
 }
 
-func (d *Downloader) restQueueStatus(w http.ResponseWriter, r *http.Request) {
-	if r.Method != http.MethodGet {
-		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
-		w.Header().Set("X-Content-Type-Options", "nosniff")
-		w.Header().Add("Allow", http.MethodGet)
-		w.WriteHeader(http.StatusMethodNotAllowed)
-		fmt.Fprintln(w, "HTTP Error 405 – Method Not Allowed\nOnly GET method is allowed")
-		log.Println("HTTP Error 405 – Method Not Allowed\nOnly GET method is allowed")
+func (d *Downloader) restSetDownloadStatus(w http.ResponseWriter, r *http.Request) {
+	var (
+		err       error
+		downloads []struct {
+			URL      string
+			Status   Status
+			Priority Priority
+		}
+		req *Request
+	)
+	if r.Method != http.MethodPost {
+		httpMethodNotAllowed(w, http.MethodPost)
 		return
 	}
-	j := json.NewEncoder(w)
-	w.Header().Add("Content-Type", "application/json; charset=utf-8")
-	w.WriteHeader(http.StatusOK)
-	j.Encode(d.Downloads.Queue)
+	err = json.NewDecoder(r.Body).Decode(&downloads)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+	for _, i := range downloads {
+		req = d.Downloads.find(i.URL)
+		req.Priority = i.Priority
+		req.Status = i.Status
+		req.Err = nil
+		req.Error = ""
+	}
+	d.syncDownloads()
+}
+
+func (d *Downloader) restStatus(q bool) func(http.ResponseWriter, *http.Request) {
+	return func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Add("Access-Control-Allow-Origin", "*")
+		w.Header().Add("Access-Control-Allow-Methods", "GET")
+		w.Header().Add("Access-Control-Allow-Headers", "X-PINGOTHER, Content-Type")
+		if r.Method == http.MethodOptions {
+			w.Header().Add("Allow", "GET")
+			w.WriteHeader(http.StatusNoContent)
+			return
+		}
+		var queue = &d.History
+		if q {
+			d.Downloads.updateStatus()
+			queue = &d.Downloads
+		}
+
+		if r.Method != http.MethodGet {
+			httpMethodNotAllowed(w, http.MethodGet)
+			return
+		}
+		queue.updateStatus()
+		v, err := jettison.MarshalOpts(queue.Queue, jettison.DenyList([]string{"cookies"}))
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+		w.Header().Add("Content-Type", "application/json; charset=utf-8")
+		w.WriteHeader(http.StatusOK)
+		w.Write(v)
+	}
 }
 
 func (d *Downloader) restAddDownload(w http.ResponseWriter, r *http.Request) {
@@ -302,12 +601,7 @@ func (d *Downloader) restAddDownload(w http.ResponseWriter, r *http.Request) {
 		err      error
 	)
 	if r.Method != http.MethodPost {
-		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
-		w.Header().Set("X-Content-Type-Options", "nosniff")
-		w.Header().Add("Allow", http.MethodPost)
-		w.WriteHeader(http.StatusMethodNotAllowed)
-		fmt.Fprintln(w, "HTTP Error 405 – Method Not Allowed\nOnly POST method is allowed")
-		log.Println("HTTP Error 405 – Method Not Allowed\nOnly POST method is allowed")
+		httpMethodNotAllowed(w, http.MethodPost)
 		return
 	}
 	// TODO fail only on individual requests
@@ -334,8 +628,10 @@ func (d Downloader) getNameFromHEAD(r Request) string {
 		re  *http.Response
 		p   map[string]string
 	)
+
+	r.insertCookies()
 	ht := &http.Client{
-		Jar:     d.Jar,
+		Jar:     r.Jar,
 		Timeout: 30 * time.Second,
 		Transport: &http.Transport{
 			Dial: (&net.Dialer{
@@ -464,14 +760,6 @@ func (d Downloader) FindRequest(u string) *Request {
 func (d *Downloader) addRequest(r *Request) {
 	log.Println("adding download for", r.URL)
 	req := d.FindRequest(r.URL)
-	u, _ := url.Parse(r.URL)
-	for i, v := range r.Cookies {
-		d.Jar.SetCookies(&url.URL{
-			Scheme: u.Scheme,
-			Path:   v.Path,
-			Host:   v.Domain,
-		}, []*http.Cookie{&r.Cookies[i]})
-	}
 	d.getFilename(r)
 
 	if req != nil { // url alread added
@@ -502,20 +790,38 @@ func (d *Downloader) startDownload(i int) {
 		err error
 	)
 	r = d.Downloads.Queue[i]
+	r.insertCookies()
 	d.getTempFilename(r)
 	log.Println("starting download for", r.URL, "to", r.TempPath)
 	// d.Downloads.Queue = append(d.Downloads.Queue, r)
 	if r.Response == nil || r.Response.Err() != nil {
 		req, err = grab.NewRequest(r.TempPath, r.URL)
 		if err != nil {
-			r.Status = Error
-			r.Error = err
+			r.setError(err)
 			return
 		}
 	}
 	r.Status = Downloading
 
-	r.Response = d.Grab.Do(req)
+	if r.grab == nil {
+		r.grab = &grab.Client{
+			HTTPClient: &http.Client{
+				Jar: r.Jar,
+				Transport: &http.Transport{
+					Dial: (&net.Dialer{
+						Timeout:   10 * time.Second,
+						KeepAlive: 30 * time.Second,
+					}).Dial,
+					TLSHandshakeTimeout:   5 * time.Second,
+					ResponseHeaderTimeout: 5 * time.Second,
+					ExpectContinueTimeout: 1 * time.Second,
+				},
+			},
+		}
+	}
+
+	r.Response = r.grab.Do(req)
+	r.Started = time.Now()
 	go func(r *Request) {
 		log.Println("wait for download")
 		log.Println(r.Response.IsComplete())
@@ -542,12 +848,43 @@ func (d *Downloader) syncDownloads() {
 		return
 	}
 	sort.Stable(d.Downloads)
+	var downloadsMaxed bool
 	// Start new downloads
 	for i, req := range d.Downloads.Queue {
-		if d.MaxActiveDownloads >= len(d.getRunningDownloads()) {
-			if req.Status == Queued {
+		switch req.Status {
+		case Queued:
+			if !downloadsMaxed && d.MaxActiveDownloads >= len(d.getRunningDownloads()) {
 				d.startDownload(i)
 			}
+		case Stopped:
+			if req.Response != nil {
+				var err = req.Response.Cancel()
+				if err != nil && err != context.Canceled {
+					req.setError(err)
+				}
+				req.Response = nil
+			}
+		case Downloading:
+			if req.Response == nil {
+				d.startDownload(i)
+			}
+		case Error:
+			if req.Response != nil && req.Err == nil {
+				var err = req.Response.Err()
+				var err2 = req.Response.Cancel()
+				if err != nil && err2 != context.Canceled {
+					err = err2
+				}
+				req.setError(err)
+			}
+		case Canceled:
+			if req.Response != nil {
+				err := req.Response.Cancel()
+				if err != nil && err != context.Canceled {
+					req.setError(err)
+				}
+				req.Response = nil
+			}
 		}
 	}
 
@@ -558,12 +895,13 @@ func (d *Downloader) syncDownloads() {
 			i--
 		}
 	}
+	d.Downloads.updateStatus()
 }
 
 func (d *Downloader) requestCompleted(r *Request) {
 	if r.Response.Err() == nil {
 		log.Println("removing from downloads")
-		d.Downloads.remove(r)
+		d.Downloads.remove(r.URL)
 		r.Status = Complete
 		log.Println(r.TempPath, "!=", r.FilePath)
 		if r.TempPath != r.FilePath {
@@ -571,10 +909,10 @@ func (d *Downloader) requestCompleted(r *Request) {
 			os.Rename(r.TempPath, r.FilePath)
 		}
 		d.History.Queue = append(d.History.Queue, r)
+		r.Completed = time.Now()
 	} else {
-		r.Status = Error
-		r.Error = r.Response.Err()
-		log.Println("fucking error:", r.Error)
+		r.setError(r.Response.Err())
+		log.Println("fucking error:", r.Err)
 	}
 }
 
@@ -585,17 +923,79 @@ func (d *Downloader) download() {
 			d.syncDownloads()
 
 		case r := <-d.NewRequest:
+			r.Progress = "" // not allowed when adding
 			d.addRequest(&r)
 			if d.OnAdd != nil {
-				d.OnAdd(r)
+				d.OnAdd(d, r)
 			}
 
 		case r := <-d.requestDone:
 			log.Println("finishing request for", r.URL)
 			d.requestCompleted(r)
 			if d.OnComplete != nil {
-				d.OnComplete(*r)
+				d.OnComplete(d, *r)
 			}
 		}
 	}
 }
+
+func (r *Request) insertCookies() {
+	if r.Jar == nil {
+		r.Jar = newCookieJar()
+	}
+	u, _ := url.Parse(r.URL)
+	for i, v := range r.Cookies {
+		r.Jar.SetCookies(&url.URL{
+			Scheme: u.Scheme,
+			Path:   v.Path,
+			Host:   v.Domain,
+		}, []*http.Cookie{&r.Cookies[i]})
+	}
+}
+
+func (d *Downloader) initStatus(mux *http.ServeMux) {
+	for _, fn := range []string{
+		"bootstrap-table-auto-refresh.min.js",
+		"bootstrap-table-sticky-header.css",
+		"bootstrap-table-sticky-header.min.js",
+		"bootstrap-table.min.css",
+		"bootstrap-table.min.js",
+		"bootstrap.min.css",
+		"bootstrap.min.js",
+		"fontawesome.css",
+		"jquery-3.2.1.min.js",
+		"popper.min.js",
+		"fa-solid-900.woff2",
+		"downloads.js",
+		"favicon.ico",
+	} {
+		mux.Handle("/"+fn, bundled.HTTPHandlerFunc(fn))
+	}
+	mux.HandleFunc("/", d.httpQueue(false))
+	mux.HandleFunc("/history", d.httpQueue(true))
+}
+func (d *Downloader) httpQueue(history bool) func(w http.ResponseWriter, r *http.Request) {
+	commonTmpls := template.New("root")
+	commonTmpls = template.Must(commonTmpls.New("header").Parse(bundled.Asset("header.tmpl")))
+	commonTmpls = template.Must(commonTmpls.New("footer").Parse(bundled.Asset("footer.tmpl")))
+
+	queueTmpl := template.Must(template.Must(commonTmpls.Clone()).New("queueTmpl").Parse(bundled.Asset("queue.tmpl")))
+	return func(w http.ResponseWriter, r *http.Request) {
+		var buf bytes.Buffer
+		d.Downloads.updateStatus()
+		if err := queueTmpl.Execute(&buf, struct {
+			Queue     []*Request
+			History   []*Request
+			Hostname  string
+			IsHistory bool
+		}{
+			Queue:     d.Downloads.Queue,
+			History:   d.History.Queue,
+			Hostname:  "gloader",
+			IsHistory: history,
+		}); err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+		}
+		io.Copy(w, &buf)
+	}
+}

extensions/downthemall/.eslintignore

@@ -0,0 +1 @@
+bundles/

extensions/downthemall/.eslintrc.js

@@ -0,0 +1,145 @@
+"use strict";
+// License: MIT
+
+module.exports = {
+    "parser": "@typescript-eslint/parser",
+    "plugins": ["@typescript-eslint"],
+    "env": {
+        "es6": true,
+        "browser": true,
+        "commonjs": true,
+        "webextensions": true,
+    },
+    "extends": ["eslint:recommended", "plugin:@typescript-eslint/recommended"],
+    "parserOptions": {
+        "sourceType": "module",
+        "ecmaVersion": 8,
+    },
+    "rules": {
+        "max-len": ["error", {
+                "ignoreTemplateLiterals": true,
+                "ignoreRegExpLiterals": true
+        }],
+        "max-depth": ["error", 4],
+        "curly": ["error", "all"],
+        "brace-style": ["error", "stroustrup"],
+        "no-console": 0,
+        "no-unused-vars": ["error", {
+            "vars": "local",
+            "varsIgnorePattern": "^_+|^dummy"
+        }],
+        "indent": [
+            "error", 2, {
+                "SwitchCase": 0,
+                "flatTernaryExpressions": true,
+                "VariableDeclarator": 2,
+                "outerIIFEBody": 0,
+                "MemberExpression": 1,
+                "FunctionDeclaration": {"body": 1, "parameters": 2},
+                "FunctionExpression": {"body": 1, "parameters": 2},
+                "CallExpression": {"arguments": 1},
+                "ArrayExpression": 1,
+                "ObjectExpression": 1
+            }
+        ],
+        "no-trailing-spaces": "error",
+        "no-multi-spaces": "error",
+        "key-spacing": ["error", {
+                "mode": "strict",
+                "beforeColon": false,
+                "afterColon": true,
+            }
+        ],
+        "keyword-spacing": ["error", { "before": true }],
+        "space-infix-ops": "error",
+        "space-unary-ops": "error",
+        "no-mixed-spaces-and-tabs": "error",
+        "accessor-pairs": "error",
+        "no-template-curly-in-string": "error",
+        "array-callback-return": "error",
+        "block-scoped-var": "error",
+        "consistent-return": "error",
+        "dot-location": ["error", "object"],
+        "dot-notation": "error",
+        "eqeqeq": "error",
+        "no-else-return": "error",
+        "no-eval": "error",
+        "no-floating-decimal": "error",
+        "no-implied-eval": "error",
+        "no-lone-blocks": "error",
+        "no-magic-numbers": ["error", { "ignore": [-1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 100, 500, 1000, 20, 25] }],
+        "no-new-func": "error",
+        "no-self-assign": "error",
+        "no-self-compare": "error",
+        "no-useless-call": "error",
+        "no-useless-concat": "error",
+        "no-with": "error",
+        "radix": "error",
+        "require-await": "error",
+        "block-spacing": "error",
+        "eol-last": ["error", "always"],
+        "comma-dangle": ["error", "only-multiline"],
+        "comma-spacing": ["error", { "before": false, "after": true }],
+        "comma-style": ["error", "last"],
+        "func-name-matching": "error",
+        "func-call-spacing": ["error", "never"],
+        "func-call-spacing": ["error", "never"],
+        "computed-property-spacing": ["error", "never"],
+        "yoda": "error",
+        "new-cap": "error",
+        "new-parens": "error",
+        "no-lonely-if": "error",
+        "no-tabs": "error",
+        "one-var": ["error", "never"],
+        "operator-assignment": ["error", "always"],
+        "operator-linebreak": ["error", "after"],
+        "padded-blocks": ["error", "never"],
+        "quote-props": ["error", "consistent-as-needed"],
+        "linebreak-style": [ "error", "unix" ],
+        "quotes": [ "error", "double" ],
+        "semi": [ "error", "always" ],
+        "semi-spacing": "error",
+        "semi-style": ["error", "last"],
+        "space-before-blocks": "error",
+        "space-in-parens": ["error", "never"],
+        "arrow-parens": ["error", "as-needed"],
+        "arrow-spacing": "error",
+        "generator-star-spacing": ["error", {"before": true, "after": false}],
+        "no-useless-computed-key": "error",
+        "no-useless-constructor": "error",
+        "no-var": "error",
+        "object-shorthand": "error",
+        "prefer-const": "error",
+        "prefer-destructuring": "error",
+        "prefer-numeric-literals": "error",
+        "prefer-rest-params": "error",
+        "prefer-template": "error",
+        "rest-spread-spacing": ["error", "never"],
+        "template-curly-spacing": "error",
+        "yield-star-spacing": ["error", {"before": true, "after": false}],
+        "valid-jsdoc": ["error", {
+            "requireReturn": false,
+            "requireParamDescription": false,
+            "requireReturnDescription": false,
+            "prefer": {
+                "arg": "param",
+                "argument": "param",
+                "return": "returns",
+                "virtual": "abstract"
+            }
+        }],
+        "padding-line-between-statements": [
+            "error",
+            { "blankLine": "always", "prev": "*", "next": ["class", "function", "case", "directive"] },
+            { "blankLine": "never", "prev": "directive", "next": "directive" },
+            { "blankLine": "always", "prev": ["class", "function", "directive", "cjs-import"], "next": "*" },
+            { "blankLine": "never", "prev": "directive", "next": "directive" },
+            { "blankLine": "never", "prev": "cjs-import", "next": "cjs-import" },
+        ],
+        "lines-between-class-members": "error",
+        "padded-blocks": ["error", "never"],
+        "@typescript-eslint/explicit-function-return-type": 0,
+        "@typescript-eslint/no-explicit-any": 0,
+        "@typescript-eslint/no-unused-vars": 0,
+    }
+};

extensions/downthemall/.github/FUNDING.yml

@@ -0,0 +1,12 @@
+# These are supported funding model platforms
+
+github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
+patreon: nmaier
+open_collective: # Replace with a single Open Collective username
+ko_fi: # Replace with a single Ko-fi username
+tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
+community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
+liberapay: # Replace with a single Liberapay username
+issuehunt: # Replace with a single IssueHunt username
+otechie: # Replace with a single Otechie username
+custom: https://www.downthemall.org/howto/donate/

extensions/downthemall/.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,33 @@
+---
+name: Bug report
+about: Create a report to help us improve DownThemAll!
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Desktop (please complete the following information):**
+ - OS: [e.g. Windows 10, macOS, Linux (distribution, desktop environment)]
+ - Browser and version: [e.g. Firefox 42, Chrome 70, Opera 15, Seamonkey 2.16]
+ - DownThemAll! version: [e.g. 4.2, 3.0, latest]
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+
+**Additional context**
+Add any other context about the problem here.

extensions/downthemall/.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest a feature or an idea for DownThemAll!
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

extensions/downthemall/.gitignore

@@ -0,0 +1,9 @@
+*.bundle.js*
+bundles/
+package-lock.json
+node_modules
+.DS_Store
+Thumbs.db
+web-ext-artifacts/
+uikit/modal.html
+.vscode

extensions/downthemall/LICENSE.gpl-2.0.txt

@@ -0,0 +1,339 @@
+                    GNU GENERAL PUBLIC LICENSE
+                       Version 2, June 1991
+
+ Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+                            Preamble
+
+  The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+License is intended to guarantee your freedom to share and change free
+software--to make sure the software is free for all its users.  This
+General Public License applies to most of the Free Software
+Foundation's software and to any other program whose authors commit to
+using it.  (Some other Free Software Foundation software is covered by
+the GNU Lesser General Public License instead.)  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+this service if you wish), that you receive source code or can get it
+if you want it, that you can change the software or use pieces of it
+in new free programs; and that you know you can do these things.
+
+  To protect your rights, we need to make restrictions that forbid
+anyone to deny you these rights or to ask you to surrender the rights.
+These restrictions translate to certain responsibilities for you if you
+distribute copies of the software, or if you modify it.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must give the recipients all the rights that
+you have.  You must make sure that they, too, receive or can get the
+source code.  And you must show them these terms so they know their
+rights.
+
+  We protect your rights with two steps: (1) copyright the software, and
+(2) offer you this license which gives you legal permission to copy,
+distribute and/or modify the software.
+
+  Also, for each author's protection and ours, we want to make certain
+that everyone understands that there is no warranty for this free
+software.  If the software is modified by someone else and passed on, we
+want its recipients to know that what they have is not the original, so
+that any problems introduced by others will not reflect on the original
+authors' reputations.
+
+  Finally, any free program is threatened constantly by software
+patents.  We wish to avoid the danger that redistributors of a free
+program will individually obtain patent licenses, in effect making the
+program proprietary.  To prevent this, we have made it clear that any
+patent must be licensed for everyone's free use or not licensed at all.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+                    GNU GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License applies to any program or other work which contains
+a notice placed by the copyright holder saying it may be distributed
+under the terms of this General Public License.  The "Program", below,
+refers to any such program or work, and a "work based on the Program"
+means either the Program or any derivative work under copyright law:
+that is to say, a work containing the Program or a portion of it,
+either verbatim or with modifications and/or translated into another
+language.  (Hereinafter, translation is included without limitation in
+the term "modification".)  Each licensee is addressed as "you".
+
+Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running the Program is not restricted, and the output from the Program
+is covered only if its contents constitute a work based on the
+Program (independent of having been made by running the Program).
+Whether that is true depends on what the Program does.
+
+  1. You may copy and distribute verbatim copies of the Program's
+source code as you receive it, in any medium, provided that you
+conspicuously and appropriately publish on each copy an appropriate
+copyright notice and disclaimer of warranty; keep intact all the
+notices that refer to this License and to the absence of any warranty;
+and give any other recipients of the Program a copy of this License
+along with the Program.
+
+You may charge a fee for the physical act of transferring a copy, and
+you may at your option offer warranty protection in exchange for a fee.
+
+  2. You may modify your copy or copies of the Program or any portion
+of it, thus forming a work based on the Program, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) You must cause the modified files to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    b) You must cause any work that you distribute or publish, that in
+    whole or in part contains or is derived from the Program or any
+    part thereof, to be licensed as a whole at no charge to all third
+    parties under the terms of this License.
+
+    c) If the modified program normally reads commands interactively
+    when run, you must cause it, when started running for such
+    interactive use in the most ordinary way, to print or display an
+    announcement including an appropriate copyright notice and a
+    notice that there is no warranty (or else, saying that you provide
+    a warranty) and that users may redistribute the program under
+    these conditions, and telling the user how to view a copy of this
+    License.  (Exception: if the Program itself is interactive but
+    does not normally print such an announcement, your work based on
+    the Program is not required to print an announcement.)
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Program,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Program, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Program.
+
+In addition, mere aggregation of another work not based on the Program
+with the Program (or with a work based on the Program) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may copy and distribute the Program (or a work based on it,
+under Section 2) in object code or executable form under the terms of
+Sections 1 and 2 above provided that you also do one of the following:
+
+    a) Accompany it with the complete corresponding machine-readable
+    source code, which must be distributed under the terms of Sections
+    1 and 2 above on a medium customarily used for software interchange; or,
+
+    b) Accompany it with a written offer, valid for at least three
+    years, to give any third party, for a charge no more than your
+    cost of physically performing source distribution, a complete
+    machine-readable copy of the corresponding source code, to be
+    distributed under the terms of Sections 1 and 2 above on a medium
+    customarily used for software interchange; or,
+
+    c) Accompany it with the information you received as to the offer
+    to distribute corresponding source code.  (This alternative is
+    allowed only for noncommercial distribution and only if you
+    received the program in object code or executable form with such
+    an offer, in accord with Subsection b above.)
+
+The source code for a work means the preferred form of the work for
+making modifications to it.  For an executable work, complete source
+code means all the source code for all modules it contains, plus any
+associated interface definition files, plus the scripts used to
+control compilation and installation of the executable.  However, as a
+special exception, the source code distributed need not include
+anything that is normally distributed (in either source or binary
+form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component
+itself accompanies the executable.
+
+If distribution of executable or object code is made by offering
+access to copy from a designated place, then offering equivalent
+access to copy the source code from the same place counts as
+distribution of the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+  4. You may not copy, modify, sublicense, or distribute the Program
+except as expressly provided under this License.  Any attempt
+otherwise to copy, modify, sublicense or distribute the Program is
+void, and will automatically terminate your rights under this License.
+However, parties who have received copies, or rights, from you under
+this License will not have their licenses terminated so long as such
+parties remain in full compliance.
+
+  5. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Program or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Program (or any work based on the
+Program), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+  6. Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the
+original licensor to copy, distribute or modify the Program subject to
+these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties to
+this License.
+
+  7. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Program at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Program by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system, which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+  8. If the distribution and/or use of the Program is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License
+may add an explicit geographical distribution limitation excluding
+those countries, so that distribution is permitted only in or among
+countries not thus excluded.  In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+  9. The Free Software Foundation may publish revised and/or new versions
+of the General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Program
+specifies a version number of this License which applies to it and "any
+later version", you have the option of following the terms and conditions
+either of that version or of any later version published by the Free
+Software Foundation.  If the Program does not specify a version number of
+this License, you may choose any version ever published by the Free Software
+Foundation.
+
+  10. If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the author
+to ask for permission.  For software which is copyrighted by the Free
+Software Foundation, write to the Free Software Foundation; we sometimes
+make exceptions for this.  Our decision will be guided by the two goals
+of preserving the free status of all derivatives of our free software and
+of promoting the sharing and reuse of software generally.
+
+                            NO WARRANTY
+
+  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+                     END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+convey the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+Also add information on how to contact you by electronic and paper mail.
+
+If the program is interactive, make it output a short notice like this
+when it starts in an interactive mode:
+
+    Gnomovision version 69, Copyright (C) year name of author
+    Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, the commands you use may
+be called something other than `show w' and `show c'; they could even be
+mouse-clicks or menu items--whatever suits your program.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the program, if
+necessary.  Here is a sample; alter the names:
+
+  Yoyodyne, Inc., hereby disclaims all copyright interest in the program
+  `Gnomovision' (which makes passes at compilers) written by James Hacker.
+
+  <signature of Ty Coon>, 1 April 1989
+  Ty Coon, President of Vice
+
+This General Public License does not permit incorporating your program into
+proprietary programs.  If your program is a subroutine library, you may
+consider it more useful to permit linking proprietary applications with the
+library.  If this is what you want to do, use the GNU Lesser General
+Public License instead of this License.

extensions/downthemall/LICENSE.md

@@ -0,0 +1,89 @@
+## License
+
+DownThemAll! is comprised of a different parts.
+The core is open source, licensed under MIT with the interface licensed under GPL-2.0.
+This means that you can reuse code almost everywhere, while forks of DownThemAll! need
+to be licensed under the GPL-2.0. Forks need to use their own name and logo however.
+
+## DownThemAll! source code
+
+Copyright © 2017-2019 by Nils Maier
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the “Software”), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
+## DownThemAll! uikit
+
+Copyright © 2016-2019 by Nils Maier
+The uikit libraries and assets are licensed under the MIT license.
+
+## DownThemAll! interface (.html, .css)
+
+Copyright © 2017-2019 by Nils Maier
+Licensed under GPL2.0; see [LICENSE.gpl-2.0.txt](LICENSE.gpl-2.0.txt).
+
+
+## DownThemAll! icons, icon-font and graphic assets
+
+Copyright (C) 2012-2019 by Nils Maier
+Licensed under Creative Commons Attribution-ShareAlike 4.0 International.
+
+The icon font contains icons from Font Awesome.
+
+See: https://creativecommons.org/licenses/by-sa/4.0/legalcode
+
+
+## DownThemAll! logo and name
+
+Copyright © 2010-2019 by Nils Maier, Stefano Verna.
+
+The DownThemAll! name and logo cannot be used without explicit permission
+in any derivative work, except in credits and license-related notices.
+
+Using the DownThemAll! logo in personal non-distributed non-commercial
+modifications of the software and forks is permitted without explicit
+permission.
+
+Distributing official DownThemAll! releases without any modifications is allowed without explicit permission.
+
+## Font Awesome
+
+Copyright (C) 2016 by Dave Gandy
+
+License:   SIL ()
+
+Homepage:  http://fortawesome.github.com/Font-Awesome/
+
+## webextension-polyfill
+
+Licensed under the Mozilla Public License 2.0.
+
+## PSL (public-suffix-list)
+
+The list itself is licensed under the Mozilla Public License 2.0.
+The javascript library accessing it is licensed under the MIT license.
+
+## whatwg-mimetype
+
+Licensed under MIT
+
+
+## CDHeaderParser
+
+Licensed under MPL-2
+(c) 2017 Rob Wu <rob@robwu.nl> (https://robwu.nl)

extensions/downthemall/Readme.md

@@ -0,0 +1,103 @@
+
+![DownThemAll!](https://raw.githubusercontent.com/downthemall/downthemall/master/style/icon128.png)
+
+
+# DownThemAll! WE
+
+The DownThemAll! WebExtension.
+
+For those still on supported browser: [Non-WebExtension legacy code](https://github.com/downthemall/downthemall-legacy).
+
+## About
+
+This is the WebExtension version of DownThemAll!, a complete re-development from scratch.
+Being a WebExtension it lacks a ton of features the original DownThemAll! had. Sorry, but there is no way around it since Mozilla decided to adopt WebExtensions as the *only* extension type and WebExtensions are extremely limited in what they can do.
+
+For what is planned (and not planned because impossible to do in WebExtensions), see [TODO.md](TODO.md).
+
+What this furthermore means is that some bugs we fixed in the original DownThemAll! are back, as we cannot do our own downloads any longer but have to go through the browser download manager always, which is notoriously bad at handling certain "quirks" real web servers in the wild show. It doesn't even handle regular 404 errors.
+
+I spent countless hours evaluating various workarounds to enable us to do our own downloads instead of relying on the downloads API (the browser built-in downloader). From using `IndexedDB` to store retrieved chunks via `XHR`, to doing nasty service-worker tricks to fake a download that the backend would retrieve with `XHR`. The last one looks promising but I have yet to get it to work in a manner that is reliable, performs well enough and doesn't eat all the system memory for breakfast. Maybe in the future...
+
+What this also means is that we have to write our user interface in HTML, which never looks "native" and cannot offer deep OS integration.
+
+But it is what it is...
+
+**What we *can* do and did do is bring the mass selection, organizing (renaming masks, etc) and queueing tools of DownThemAll! over to the WebExtension, so you can easily queue up hundreds or thousands files at once without the downloads going up in flames because the browser tried to download them all at once.**
+
+## Translations
+
+If you would like to help out translating DTA, please see our [translation guide](_locales/Readme.md).
+
+## Development
+
+### Requirements
+
+ - [node](https://nodejs.org/en/)
+ - [yarn](https://yarnpkg.com/)
+ - [python3](https://www.python.org/) >= 3.6 (to build zips)
+ - [web-ext](https://www.npmjs.com/package/web-ext) (for development ease)
+ 
+### Setup
+
+You will want to run `yarn` to install the development dependencies such as webpack first.
+
+### Making changes
+
+Just use your favorite text editor to edit the files.
+
+You will want to run`yarn watch`.
+This will run the webpack bundler in watch mode, transpiling the TypeScript to Javascript and updating bundles as you change the source.
+
+Please note: You have to run `yarn watch` or `yarn build` (at least once) as it builds the actual script bundles.
+
+### Running in Firefox
+
+I recommend you install the [`web-ext`](https://www.npmjs.com/package/web-ext) tools from mozilla. It is not listed as a dependency by design at it causes problems with dependency resolution in yarn right now if installed in the same location as the rest of the dependencies.
+
+If you did, then running `yarn webext` (additionally to `yarn watch`) will run the WebExtension in a development profile. This will use the directory `../dtalite.p` to keep a development profile. You might need to create this directory before you use this command. Furthermore `yarn webext` will watch for changes to the sources and automatically reload the extension.
+  
+Alternatively, you can also `yarn build`, which then builds an *unsigned* zip that you can then install permanently in a browser that does not enforce signing (i.e. Nightly or the Unbranded Firefox with the right about:config preferences).
+
+### Running in Chrome/Chromium/etc
+
+You have to build the bundles first, of course.
+
+Then put your Chrome into Developement Mode on the Extensions page, and Load Unpacked the directory of your downthemall clone.
+
+### Making release zips
+
+To get a basic unofficial set of zips for Firefox and chrome, run `yarn build`.
+
+If you want to generate release builds like the ones that are eventually released in the extension stores, use `python3 util/build.py --mode=release`.
+
+The output is located in `web-ext-artifacts`.
+
+ - `-fx.zip` are Firefox builds
+ - `-crx.zip` are Chrome/Chromium builds
+ - `-opr.zip` are Opera builds (essentially like the Chrome one, but without sounds)
+
+### The AMO Editors tl;dr guide
+
+  1. Install the requirements.
+  2. `yarn && python3 build/util.py --mode=release`
+  3. Have a look in `web-ext-artifacts/dta-*-fx.zip`
+
+
+### Patches
+
+Before submitting patches, please make sure you run eslint (if this isn't done automatically in your text editor/IDE), and eslint does not report any open issues. Code contributions should favor typescript code over javascript code. External dependencies that would ship with the final product (including all npm/yarn packages) should be kept to a bare minimum and need justification.
+
+Please submit your patches as Pull Requests, and rebase your commits onto the current `master` before submitting.
+
+### Code structure
+
+The code base is comparatively large for a WebExtension, with over 11K sloc of typescript.
+It isn't as well organized as it should be in some places; hope you don't mind.
+
+* `uikit/` - The base User Interface Kit, which currently consists of
+  * the `VirtualTable` implementation, aka that interactive HTML table with columns, columns resizing and hiding, etc you see in the Manager, Select and Preferences windows/tabs
+  * the `ContextMenu` and related classes that drive the HTML-based context menus
+* `lib/` - The "backend stuff" and assorted library routines and classes.
+* `windows/` - The "frontend stuff" so all the HTML and corresponding code to make that HTML into something interactive
+* `style/` - CSS and images

extensions/downthemall/TODO.md

@@ -0,0 +1,52 @@
+TODO
+---
+
+P2
+===
+
+Planned for later.
+
+* Inter-addon API (basic)
+  * Add downloads
+* vtable perf: cache column widths
+* Download options
+  * This is a bit more limited, as we cannot modify options of downloads that have been started (and paused) or that are done.
+
+P3
+===
+
+Nice-to-haves.
+
+* Landing Page v4
+* Drag-n-drop ordering
+* Drag-n-drop files
+* Inter-addon API (hooks)
+  * Manipulate downloads (e.g. rewrite URLs)
+* Native context menus?
+  * Would require massive reworks incl the need for new icon formats, but potentially feasible.
+* Download priorities (manual scheduling overrides)
+* Remove `any` types as possible, and generally improve typescript (new language to me)
+
+P4
+===
+
+Stuff that probably cannot be implemented due to WeberEension limitations.
+
+* Avoid downloads going "missing" after a browser restart.
+  * Firefox helpfully keeps different lists of downloads. One for newly added downloads, and other ones for "previous" downloads. Turns out the WebExtension API only ever queries the "new" list.
+* Segmented downloads
+  * Cannot be done with WebExtensions - downloads API has no support and manually downloading, storing in temporary add-on storage and reassmbling the downloaded parts later is not only efficient but does not reliabliy work due to storage limitations.
+* Conflicts: ask when a file exists
+  * Not supported by Firefox
+* Speed limiter
+  * Cannot be done with the WebExtensions downloads API
+* contenthandling aka video sniffing, request manipulation?
+  * PITA and/or infeasible - Essentially cannot be done for a large part and the other prt is extemely inefficient
+* Checksums/Hashes?
+  * Cannot be done with WebExtensions - cannot actually read the downloaded data
+* Mirrors?
+  * Cannot be done with WebExtensions - no low level APIs, see segmented downloads
+* Metalink?
+  * Currently infeasible, as we cannot look into download data streams.
+    Headers-only might be an option.
+    But then again, metalink's features are mirrors and checksums, and we cannot do those.

extensions/downthemall/_locales/Readme.md

@@ -0,0 +1,33 @@
+# Translations
+
+Right now we did not standardize on a tool/website/community use for translations
+
+## Website-based Translation
+
+Please go to [https://downthemall.github.io/translate/](https://downthemall.github.io/translate/) for a "good enough" tool to translate DownThemAll! for now. It will load the English locale as a base automatically.
+
+Then you can translate (your progress will be saved in the browser). Once done, you can Download the `messages.json` and test it or submit it for inclusion.
+
+You can also import your or other people's existing translations to modify. This will overwrite any progress you made so far, tho.
+
+## Manual Translation
+
+* Get the [`en/messages.json`](https://github.com/downthemall/downthemall/raw/master/_locales/en/messages.json) as a base.
+* Translate the `"message"` items in that file only. Whip our your favorite text editor, JSON editor, special translation tool, what have you.
+  * Do not translate anything besides the "message" elements. Pay attention to the descriptions.
+  * Do not remove anything.
+  * Do not translate `$PLACEHOLDERS$`. Placeholders should appear in your translation with the same spelling and all uppercase.
+    They will be relaced at runtime with actual values.
+* Make sure you save the file in an "utf-8" encoding. If you need double quotes, you need to escape the quotes with a backslash, e.g. `"some \"quoted\" text"`
+* You should translate all strings. If you want to skip a string, set it to an empty `""` string. DTA will then use the English string.
+
+## Testing Your Translation
+
+* Go to the DownThemAll! Preferences where you will find a "Load custom translation" button.
+* Select your translated `messages.json`. (it doesn't have to be named exactly like that, but should have a `.json` extension)
+  * If everything was OK, you will be asked to reload the extension (this will only reload DTA not the entire browser).
+* See your strings in action once you reloaded DTA (either by answering OK when asked, or disable/enable the extension manually or restart your browser).
+
+## Submitting Your Translation
+
+If you're happy with the result and would like to contribute it back, you can either file a full Pull Request, or just file an issue and post a link to e.g. a [gist](https://gist.github.com/) or paste the translation in the issue text.

extensions/downthemall/_locales/all.json

@@ -0,0 +1,26 @@
+{
+  "ar": "العربية [ar]",
+  "bg": "Български [bg]",
+  "cs": "Čeština (CZ) [cs]",
+  "da": "Dansk [da]",
+  "de": "Deutsch [de]",
+  "el": "Ελληνικά [el]",
+  "en": "English (US) [en]",
+  "es": "Español (España) [es]",
+  "et": "Eesti keel [et]",
+  "fr": "Français [fr]",
+  "hu": "Magyar (HU) [hu]",
+  "id": "Bahasa Indonesia [id]",
+  "it": "Italiano [it]",
+  "ja": "日本語 (JP) [ja]",
+  "ko": "한국어 [ko]",
+  "lt": "Lietuvių [lt]",
+  "nl": "Nederlands [nl]",
+  "pl": "Polski [pl]",
+  "pt": "Português (Brasil) [pt]",
+  "ru": "Русский [ru]",
+  "sv": "Svenska (SV) [sv]",
+  "tr": "Türkçe TR) [tr]",
+  "zh_CN": "中文（简体） [zh_CN]",
+  "zh_TW": "正體中文 (TW) [zh_TW]"
+}

extensions/downthemall/data/filters.json

@@ -0,0 +1,71 @@
+{
+	"deffilter-all": {
+		"label": "All files",
+		"expr": "/.*/i",
+		"type": 3,
+		"active": false
+	},
+	"deffilter-arch": {
+		"label": "Archives",
+		"expr": "/\\.(?:z(?:ip|[0-9]{2})|r(?:ar|[0-9]{2})|jar|bz2|gz|tar|rpm|7z(?:ip)?|lzma|xz)$/i",
+		"type": 1,
+		"active": false,
+		"icon": "zip"
+	},
+	"deffilter-aud": {
+		"label": "Audio",
+		"expr": "/\\.(?:mp3|wav|og(?:g|a)|flac|midi?|rm|aac|wma|mka|ape|opus)$/i",
+		"type": 3,
+		"active": false,
+		"icon": "mp3"
+	},
+	"deffilter-bin": {
+		"label": "Software",
+		"expr": "/\\.(?:exe|msi|dmg|bin|xpi|iso)$/i",
+		"type": 1,
+		"active": false,
+		"icon": "exe"
+	},
+	"deffilter-doc": {
+		"label": "Documents",
+		"expr": "/\\.(?:pdf|xlsx?|docx?|odf|odt|rtf|txt|nfo)$/i",
+		"type": 1,
+		"active": false,
+		"icon": "pdf"
+	},
+	"deffilter-img": {
+		"label": "Images",
+		"expr": "/\\.(?:jp(?:e?g|e|2)|gif|png|tiff?|bmp|ico|heic|heif|webp|jxr|wdp|dng|cr2|arw)$/i",
+		"type": 3,
+		"active": false,
+		"icon": "jpg"
+	},
+	"deffilter-imggif": {
+		"label": "GIF",
+		"expr": "/\\.gif$/i",
+		"type": 2,
+		"active": false,
+		"icon": "gif"
+	},
+	"deffilter-imgjpg": {
+		"label": "JPEG",
+		"expr": "/\\.jp(e?g|e|2)$/i",
+		"type": 3,
+		"active": true,
+		"icon": "jpg"
+	},
+	"deffilter-imgpng": {
+		"label": "PNG",
+		"expr": "/\\.png$/i",
+		"type": 2,
+		"active": false,
+		"icon": "png"
+	},
+	"deffilter-vid": {
+		"label": "Videos",
+		"expr": "/\\.(?:mpeg|ra?m|avi|mp(?:g|e|4)|mov|divx|asf|qt|wmv|m\\dv|rv|vob|asx|ogm|ogv|webm|flv|mkv|f4v|m4v)$/i",
+		"type": 3,
+		"active": true,
+		"icon": "mkv"
+	}
+}

extensions/downthemall/data/icons.json

@@ -0,0 +1,217 @@
+{
+	"pdf": [
+		"pdf"
+	],
+	"word": [
+		"doc",
+		"docm",
+		"docx",
+		"dot",
+		"dotm",
+		"dotx",
+		"odt",
+		"ppt",
+		"pptm",
+		"pptx",
+		"rtf",
+		"xls",
+		"xlsb",
+		"xlsm",
+		"xlsx",
+		"xltm",
+		"xltx"
+	],
+	"doc": [
+		"c",
+		"chm",
+		"cpp",
+		"csv",
+		"cxx",
+		"h",
+		"hpp",
+		"htm",
+		"html",
+		"hxx",
+		"ini",
+		"java",
+		"js",
+		"lua",
+		"mht",
+		"mhtml",
+		"potx",
+		"potm",
+		"ppam",
+		"ppsm",
+		"ppsx",
+		"pps",
+		"sldm",
+		"sldx",
+		"thmx",
+		"txt",
+		"vsd",
+		"wpd",
+		"wps",
+		"wri",
+		"xlam",
+		"xml",
+		"log",
+		"rb",
+		"py",
+		"php",
+		"cc",
+		"c++",
+		"m",
+		"mm",
+		"json"
+	],
+	"image": [
+		"ani",
+		"apng",
+		"bmp",
+		"gif",
+		"ico",
+		"jpe",
+		"jpeg",
+		"jpg",
+		"pcx",
+		"png",
+		"psd",
+		"tga",
+		"tif",
+		"tiff",
+		"wmf",
+		"webp",
+		"heic",
+		"heif",
+		"jxr",
+		"wdp"
+	],
+	"video": [
+		"3g2",
+		"3gp",
+		"3gp2",
+		"3gpp",
+		"amr",
+		"amv",
+		"asf",
+		"avi",
+		"bdmv",
+		"bik",
+		"d2v",
+		"divx",
+		"drc",
+		"dsa",
+		"dsm",
+		"dss",
+		"dsv",
+		"evo",
+		"f4v",
+		"flc",
+		"fli",
+		"flic",
+		"flv",
+		"hdmov",
+		"ifo",
+		"ivf",
+		"m1v",
+		"m2p",
+		"m2t",
+		"m2ts",
+		"m2v",
+		"m4b",
+		"m4p",
+		"m4v",
+		"mkv",
+		"mp2v",
+		"mp4",
+		"mp4v",
+		"mpe",
+		"mpeg",
+		"mpg",
+		"mpls",
+		"mpv2",
+		"mpv4",
+		"mov",
+		"mts",
+		"ogm",
+		"ogv",
+		"pss",
+		"pva",
+		"qt",
+		"ram",
+		"ratdvd",
+		"rm",
+		"rmm",
+		"rmvb",
+		"roq",
+		"rpm",
+		"smil",
+		"smk",
+		"swf",
+		"tp",
+		"tpr",
+		"ts",
+		"vob",
+		"vp6",
+		"webm",
+		"wm",
+		"wmp",
+		"wmv",
+		"divx"
+	],
+	"archive": [
+		"7z",
+		"ace",
+		"arj",
+		"bz2",
+		"cab",
+		"gz",
+		"gzip",
+		"jar",
+		"lzma",
+		"r00",
+		"rar",
+		"tar",
+		"tgz",
+		"txz",
+		"xz",
+		"z",
+		"zip"
+	],
+	"audio": [
+		"aac",
+		"ac3",
+		"aif",
+		"aifc",
+		"aiff",
+		"au",
+		"cda",
+		"dts",
+		"fla",
+		"flac",
+		"it",
+		"m1a",
+		"m2a",
+		"m3u",
+		"m4a",
+		"mid",
+		"midi",
+		"mka",
+		"mod",
+		"mp2",
+		"mp3",
+		"mpa",
+		"ogg",
+		"opus",
+		"ra",
+		"rmi",
+		"rmi",
+		"snd",
+		"spc",
+		"umx",
+		"voc",
+		"wav",
+		"wma",
+		"xm"
+	]
+}

extensions/downthemall/data/mime.json

@@ -0,0 +1,396 @@
+{
+  "e": {
+    "3gpp": "3gp",
+    "asx": "asf",
+    "gz": "gzip",
+    "heic": "heif",
+    "html": [
+      "htm",
+      "shtml",
+      "php"
+    ],
+    "jar": [
+      "war",
+      "ear"
+    ],
+    "jpg": [
+      "jpeg",
+      "jpe",
+      "jfif"
+    ],
+    "js": "jsx",
+    "mid": [
+      "midi",
+      "kar"
+    ],
+    "mkv": [
+      "mk3d",
+      "mks"
+    ],
+    "mov": [
+      "qt",
+      "moov"
+    ],
+    "mpg": [
+      "mpe",
+      "mpeg"
+    ],
+    "pem": [
+      "crt",
+      "der"
+    ],
+    "pl": "pm",
+    "prc": "pdb",
+    "ps": [
+      "eps",
+      "ai"
+    ],
+    "svg": "svgz",
+    "tcl": "tk",
+    "tif": "tiff"
+  },
+  "m": {
+    "application/7z": "7z",
+    "application/7z-compressed": "7z",
+    "application/ai": "ps",
+    "application/atom": "atom",
+    "application/atom+xml": "atom",
+    "application/bz2": "bz2",
+    "application/bzip2": "bz2",
+    "application/cco": "cco",
+    "application/cocoa": "cco",
+    "application/compressed": "gz",
+    "application/crt": "pem",
+    "application/der": "pem",
+    "application/doc": "doc",
+    "application/ear": "jar",
+    "application/eot": "eot",
+    "application/eps": "ps",
+    "application/gz": "gz",
+    "application/gzip": "gz",
+    "application/hqx": "hqx",
+    "application/jar": "jar",
+    "application/jardiff": "jardiff",
+    "application/java-archive": "jar",
+    "application/java-archive-diff": "jardiff",
+    "application/java-jnlp-file": "jnlp",
+    "application/javascript": "js",
+    "application/jnlp": "jnlp",
+    "application/js": "js",
+    "application/json": "json",
+    "application/jsx": "js",
+    "application/kml": "kml",
+    "application/kmz": "kmz",
+    "application/m3u8": "m3u8",
+    "application/mac-binhex40": "hqx",
+    "application/makeself": "run",
+    "application/msword": "doc",
+    "application/odg": "odg",
+    "application/odp": "odp",
+    "application/ods": "ods",
+    "application/odt": "odt",
+    "application/pdb": "prc",
+    "application/pdf": "pdf",
+    "application/pem": "pem",
+    "application/perl": "pl",
+    "application/pilot": "prc",
+    "application/pl": "pl",
+    "application/pm": "pl",
+    "application/postscript": "ps",
+    "application/ppt": "ppt",
+    "application/prc": "prc",
+    "application/ps": "ps",
+    "application/rar": "rar",
+    "application/rar-compressed": "rar",
+    "application/redhat-package-manager": "rpm",
+    "application/rpm": "rpm",
+    "application/rss": "rss",
+    "application/rss+xml": "rss",
+    "application/rtf": "rtf",
+    "application/run": "run",
+    "application/sea": "sea",
+    "application/shockwave-flash": "swf",
+    "application/sit": "sit",
+    "application/stuffit": "sit",
+    "application/swf": "swf",
+    "application/tar": "tar",
+    "application/tcl": "tcl",
+    "application/tk": "tcl",
+    "application/vnd.apple.mpegurl": "m3u8",
+    "application/vnd.google-earth.kml+xml": "kml",
+    "application/vnd.google-earth.kmz": "kmz",
+    "application/vnd.ms-excel": "xls",
+    "application/vnd.ms-fontobject": "eot",
+    "application/vnd.ms-powerpoint": "ppt",
+    "application/vnd.oasis.opendocument.graphics": "odg",
+    "application/vnd.oasis.opendocument.presentation": "odp",
+    "application/vnd.oasis.opendocument.spreadsheet": "ods",
+    "application/vnd.oasis.opendocument.text": "odt",
+    "application/vnd.wap.wmlc": "wmlc",
+    "application/war": "jar",
+    "application/wmlc": "wmlc",
+    "application/x-7z": "7z",
+    "application/x-7z-compressed": "7z",
+    "application/x-ai": "ps",
+    "application/x-atom": "atom",
+    "application/x-atom+xml": "atom",
+    "application/x-bz2": "bz2",
+    "application/x-bzip2": "bz2",
+    "application/x-cco": "cco",
+    "application/x-cocoa": "cco",
+    "application/x-compressed": "gz",
+    "application/x-crt": "pem",
+    "application/x-der": "pem",
+    "application/x-doc": "doc",
+    "application/x-ear": "jar",
+    "application/x-eot": "eot",
+    "application/x-eps": "ps",
+    "application/x-gz": "gz",
+    "application/x-gzip": "gz",
+    "application/x-hqx": "hqx",
+    "application/x-jar": "jar",
+    "application/x-jardiff": "jardiff",
+    "application/x-java-archive": "jar",
+    "application/x-java-archive-diff": "jardiff",
+    "application/x-java-jnlp-file": "jnlp",
+    "application/x-javascript": "js",
+    "application/x-jnlp": "jnlp",
+    "application/x-js": "js",
+    "application/x-json": "json",
+    "application/x-jsx": "js",
+    "application/x-kml": "kml",
+    "application/x-kmz": "kmz",
+    "application/x-m3u8": "m3u8",
+    "application/x-mac-binhex40": "hqx",
+    "application/x-makeself": "run",
+    "application/x-msword": "doc",
+    "application/x-odg": "odg",
+    "application/x-odp": "odp",
+    "application/x-ods": "ods",
+    "application/x-odt": "odt",
+    "application/x-pdb": "prc",
+    "application/x-pdf": "pdf",
+    "application/x-pem": "pem",
+    "application/x-perl": "pl",
+    "application/x-pilot": "prc",
+    "application/x-pl": "pl",
+    "application/x-pm": "pl",
+    "application/x-postscript": "ps",
+    "application/x-ppt": "ppt",
+    "application/x-prc": "prc",
+    "application/x-ps": "ps",
+    "application/x-rar": "rar",
+    "application/x-rar-compressed": "rar",
+    "application/x-redhat-package-manager": "rpm",
+    "application/x-rpm": "rpm",
+    "application/x-rss": "rss",
+    "application/x-rss+xml": "rss",
+    "application/x-rtf": "rtf",
+    "application/x-run": "run",
+    "application/x-sea": "sea",
+    "application/x-shockwave-flash": "swf",
+    "application/x-sit": "sit",
+    "application/x-stuffit": "sit",
+    "application/x-swf": "swf",
+    "application/x-tar": "tar",
+    "application/x-tcl": "tcl",
+    "application/x-tk": "tcl",
+    "application/x-vnd.apple.mpegurl": "m3u8",
+    "application/x-vnd.google-earth.kml+xml": "kml",
+    "application/x-vnd.google-earth.kmz": "kmz",
+    "application/x-vnd.ms-excel": "xls",
+    "application/x-vnd.ms-fontobject": "eot",
+    "application/x-vnd.ms-powerpoint": "ppt",
+    "application/x-vnd.oasis.opendocument.graphics": "odg",
+    "application/x-vnd.oasis.opendocument.presentation": "odp",
+    "application/x-vnd.oasis.opendocument.spreadsheet": "ods",
+    "application/x-vnd.oasis.opendocument.text": "odt",
+    "application/x-vnd.wap.wmlc": "wmlc",
+    "application/x-war": "jar",
+    "application/x-wmlc": "wmlc",
+    "application/x-x509-ca-cert": "pem",
+    "application/x-xhtml": "xhtml",
+    "application/x-xhtml+xml": "xhtml",
+    "application/x-xls": "xls",
+    "application/x-xpi": "xpi",
+    "application/x-xpinstall": "xpi",
+    "application/x-xspf": "xspf",
+    "application/x-xspf+xml": "xspf",
+    "application/x-xz": "xz",
+    "application/x-zip": "zip",
+    "application/x509-ca-cert": "pem",
+    "application/xhtml": "xhtml",
+    "application/xhtml+xml": "xhtml",
+    "application/xls": "xls",
+    "application/xpi": "xpi",
+    "application/xpinstall": "xpi",
+    "application/xspf": "xspf",
+    "application/xspf+xml": "xspf",
+    "application/xz": "xz",
+    "application/zip": "zip",
+    "audio/kar": "mid",
+    "audio/m4a": "m4a",
+    "audio/matroska": "mka",
+    "audio/mid": "mid",
+    "audio/midi": "mid",
+    "audio/mka": "mka",
+    "audio/mp3": "mp3",
+    "audio/mpeg": "mp3",
+    "audio/ogg": "ogg",
+    "audio/ra": "ra",
+    "audio/realaudio": "ra",
+    "audio/x-kar": "mid",
+    "audio/x-m4a": "m4a",
+    "audio/x-matroska": "mka",
+    "audio/x-mid": "mid",
+    "audio/x-midi": "mid",
+    "audio/x-mka": "mka",
+    "audio/x-mp3": "mp3",
+    "audio/x-mpeg": "mp3",
+    "audio/x-ogg": "ogg",
+    "audio/x-ra": "ra",
+    "audio/x-realaudio": "ra",
+    "font/woff": "woff",
+    "font/woff2": "woff2",
+    "font/x-woff": "woff",
+    "font/x-woff2": "woff2",
+    "image/bmp": "bmp",
+    "image/gif": "gif",
+    "image/heic": "heic",
+    "image/heif": "heic",
+    "image/heif-sequence": "heic",
+    "image/ico": "ico",
+    "image/icon": "ico",
+    "image/jfif": "jpg",
+    "image/jng": "jng",
+    "image/jpe": "jpg",
+    "image/jpeg": "jpg",
+    "image/jpg": "jpg",
+    "image/ms-bmp": "bmp",
+    "image/png": "png",
+    "image/svg": "svg",
+    "image/svg+xml": "svg",
+    "image/svgz": "svg",
+    "image/tif": "tif",
+    "image/tiff": "tif",
+    "image/vnd.wap.wbmp": "wbmp",
+    "image/wbmp": "wbmp",
+    "image/webp": "webp",
+    "image/x-bmp": "bmp",
+    "image/x-gif": "gif",
+    "image/x-heic": "heic",
+    "image/x-heif": "heic",
+    "image/x-heif-sequence": "heic",
+    "image/x-ico": "ico",
+    "image/x-icon": "ico",
+    "image/x-jfif": "jpg",
+    "image/x-jng": "jng",
+    "image/x-jpe": "jpg",
+    "image/x-jpeg": "jpg",
+    "image/x-jpg": "jpg",
+    "image/x-ms-bmp": "bmp",
+    "image/x-png": "png",
+    "image/x-svg": "svg",
+    "image/x-svg+xml": "svg",
+    "image/x-svgz": "svg",
+    "image/x-tif": "tif",
+    "image/x-tiff": "tif",
+    "image/x-vnd.wap.wbmp": "wbmp",
+    "image/x-wbmp": "wbmp",
+    "image/x-webp": "webp",
+    "text/component": "htc",
+    "text/css": "css",
+    "text/htc": "htc",
+    "text/htm": "html",
+    "text/html": "html",
+    "text/jad": "jad",
+    "text/javascript": "js",
+    "text/js": "js",
+    "text/jsx": "js",
+    "text/mathml": "mml",
+    "text/mml": "mml",
+    "text/php": "html",
+    "text/plain": "txt",
+    "text/shtml": "html",
+    "text/txt": "txt",
+    "text/vnd.sun.j2me.app-descriptor": "jad",
+    "text/vnd.wap.wml": "wml",
+    "text/wml": "wml",
+    "text/x-component": "htc",
+    "text/x-css": "css",
+    "text/x-htc": "htc",
+    "text/x-htm": "html",
+    "text/x-html": "html",
+    "text/x-jad": "jad",
+    "text/x-javascript": "js",
+    "text/x-js": "js",
+    "text/x-jsx": "js",
+    "text/x-mathml": "mml",
+    "text/x-mml": "mml",
+    "text/x-php": "html",
+    "text/x-plain": "txt",
+    "text/x-shtml": "html",
+    "text/x-txt": "txt",
+    "text/x-vnd.sun.j2me.app-descriptor": "jad",
+    "text/x-vnd.wap.wml": "wml",
+    "text/x-wml": "wml",
+    "text/x-xml": "xml",
+    "text/xml": "xml",
+    "video/3gp": "3gpp",
+    "video/3gpp": "3gpp",
+    "video/asf": "asx",
+    "video/asx": "asx",
+    "video/avi": "avi",
+    "video/flv": "flv",
+    "video/m4v": "m4v",
+    "video/matroska": "mkv",
+    "video/mk3d": "mkv",
+    "video/mks": "mkv",
+    "video/mkv": "mkv",
+    "video/mng": "mng",
+    "video/moov": "mov",
+    "video/mov": "mov",
+    "video/mp2t": "ts",
+    "video/mp4": "mp4",
+    "video/mpe": "mpg",
+    "video/mpeg": "mpg",
+    "video/mpg": "mpg",
+    "video/ms-asf": "asx",
+    "video/ms-wmv": "wmv",
+    "video/msvideo": "avi",
+    "video/opus": "opus",
+    "video/qt": "mov",
+    "video/quicktime": "mov",
+    "video/ts": "ts",
+    "video/webm": "webm",
+    "video/wmv": "wmv",
+    "video/x-3gp": "3gpp",
+    "video/x-3gpp": "3gpp",
+    "video/x-asf": "asx",
+    "video/x-asx": "asx",
+    "video/x-avi": "avi",
+    "video/x-flv": "flv",
+    "video/x-m4v": "m4v",
+    "video/x-matroska": "mkv",
+    "video/x-mk3d": "mkv",
+    "video/x-mks": "mkv",
+    "video/x-mkv": "mkv",
+    "video/x-mng": "mng",
+    "video/x-moov": "mov",
+    "video/x-mov": "mov",
+    "video/x-mp2t": "ts",
+    "video/x-mp4": "mp4",
+    "video/x-mpe": "mpg",
+    "video/x-mpeg": "mpg",
+    "video/x-mpg": "mpg",
+    "video/x-ms-asf": "asx",
+    "video/x-ms-wmv": "wmv",
+    "video/x-msvideo": "avi",
+    "video/x-opus": "opus",
+    "video/x-qt": "mov",
+    "video/x-quicktime": "mov",
+    "video/x-ts": "ts",
+    "video/x-webm": "webm",
+    "video/x-wmv": "wmv"
+  }
+}

extensions/downthemall/data/prefs.json

@@ -0,0 +1,27 @@
+{
+	"button-type": "popup",
+	"manager-in-popup": false,
+	"concurrent": 4,
+	"queue-notification": true,
+	"finish-notification": true,
+	"sounds": true,
+	"open-manager-on-queue": true,
+	"text-links": true,
+	"add-paused": false,
+	"hide-context": false,
+	"conflict-action": "uniquify",
+	"nagging": 0,
+	"nagging-next": 7,
+	"tooltip": true,
+	"show-urls": false,
+	"remove-missing-on-init": false,
+	"retries": 5,
+	"retry-time": 10,
+	"theme": "default",
+	"limits": [
+		{
+			"domain": "*",
+			"concurrent": -1
+		}
+	]
+}

extensions/downthemall/lib/api.ts

@@ -0,0 +1,149 @@
+"use strict";
+// License: MIT
+
+import { TYPE_LINK, TYPE_MEDIA } from "./constants";
+import { filters } from "./filters";
+import { Prefs } from "./prefs";
+import { lazy } from "./util";
+// eslint-disable-next-line no-unused-vars
+import { Item, makeUniqueItems, BaseItem } from "./item";
+import { getManager } from "./manager/man";
+import { select } from "./select";
+import { single } from "./single";
+import { Notification } from "./notifications";
+import { MASK, FASTFILTER, SUBFOLDER } from "./recentlist";
+import { openManager } from "./windowutils";
+import { _ } from "./i18n";
+
+const MAX_BATCH = 10000;
+
+export interface QueueOptions {
+  mask?: string;
+  subfolder?: string;
+  paused?: boolean;
+}
+
+export const API = new class APIImpl {
+  async filter(arr: BaseItem[], type: number) {
+    return await (await filters()).filterItemsByType(arr, type);
+  }
+
+  async queue(items: BaseItem[], options: QueueOptions) {
+    await Promise.all([MASK.init(), SUBFOLDER.init()]);
+    const {mask = MASK.current} = options;
+    const {subfolder = SUBFOLDER.current} = options;
+
+    const {paused = false} = options;
+    const defaults: any = {
+      _idx: 0,
+      get idx() {
+        return ++this._idx;
+      },
+      referrer: null,
+      usableReferrer: null,
+      fileName: null,
+      title: "",
+      description: "",
+      startDate: new Date(),
+      private: false,
+      postData: null,
+      mask,
+      subfolder,
+      date: Date.now(),
+      paused
+    };
+    let currentBatch = await Prefs.get("currentBatch", 0);
+    const initialBatch = currentBatch;
+    lazy(defaults, "batch", () => {
+      if (++currentBatch >= MAX_BATCH) {
+        currentBatch = 0;
+      }
+      return currentBatch;
+    });
+    items = items.map(i => {
+      delete i.idx;
+      return new Item(i, defaults);
+    });
+    if (!items) {
+      return;
+    }
+    if (initialBatch !== currentBatch) {
+      await Prefs.set("currentBatch", currentBatch);
+    }
+    const manager = await getManager();
+    await manager.addNewDownloads(items);
+    if (await Prefs.get("queue-notification")) {
+      if (items.length === 1) {
+        new Notification(null, _("queued-download"));
+      }
+      else {
+        new Notification(null, _("queued-downloads", items.length));
+      }
+    }
+    if (await Prefs.get("open-manager-on-queue")) {
+      await openManager(false);
+    }
+  }
+
+  sanity(links: BaseItem[], media: BaseItem[]) {
+    if (!links.length && !media.length) {
+      new Notification(null, _("no-links"));
+      return false;
+    }
+    return true;
+  }
+
+  async turbo(links: BaseItem[], media: BaseItem[]) {
+    if (!this.sanity(links, media)) {
+      return false;
+    }
+    const type = await Prefs.get("last-type", "links");
+    const items = await (async () => {
+      if (type === "links") {
+        return await API.filter(links, TYPE_LINK);
+      }
+      return await API.filter(media, TYPE_MEDIA);
+    })();
+    const selected = makeUniqueItems([items]);
+    if (!selected.length) {
+      return await this.regular(links, media);
+    }
+    return await this.queue(selected, {paused: await Prefs.get("add-paused")});
+  }
+
+  async regularInternal(selected: BaseItem[], options: any) {
+    if (options.mask && !options.maskOnce) {
+      await MASK.init();
+      await MASK.push(options.mask);
+    }
+    if (typeof options.fast === "string" && !options.fastOnce) {
+      await FASTFILTER.init();
+      await FASTFILTER.push(options.fast);
+    }
+    if (typeof options.subfolder === "string" && !options.subfolderOnce) {
+      await SUBFOLDER.init();
+      await SUBFOLDER.push(options.subfolder);
+    }
+    if (typeof options.type === "string") {
+      await Prefs.set("last-type", options.type);
+    }
+    return await this.queue(selected, options);
+  }
+
+  async regular(links: BaseItem[], media: BaseItem[]) {
+    if (!this.sanity(links, media)) {
+      return false;
+    }
+    const {items, options} = await select(links, media);
+    return this.regularInternal(items, options);
+  }
+
+  async singleTurbo(item: BaseItem) {
+    return await this.queue([item], {paused: await Prefs.get("add-paused")});
+  }
+
+  async singleRegular(item: BaseItem | null) {
+    const {items, options} = await single(item);
+    return this.regularInternal(items, options);
+  }
+}();

extensions/downthemall/lib/background.ts

@@ -0,0 +1,679 @@
+"use strict";
+// License: MIT
+
+import { ALLOWED_SCHEMES, TRANSFERABLE_PROPERTIES } from "./constants";
+import { API } from "./api";
+import { Finisher, makeUniqueItems } from "./item";
+import { Prefs } from "./prefs";
+import { _, locale } from "./i18n";
+import { openPrefs, openManager } from "./windowutils";
+import { filters } from "./filters";
+import { getManager } from "./manager/man";
+import {
+  browserAction as action,
+  menus as _menus, contextMenus as _cmenus,
+  tabs,
+  webNavigation as nav,
+  // eslint-disable-next-line no-unused-vars
+  Tab,
+  // eslint-disable-next-line no-unused-vars
+  MenuClickInfo,
+  CHROME,
+  runtime,
+  history,
+  sessions,
+  // eslint-disable-next-line no-unused-vars
+  OnInstalled,
+} from "./browser";
+import { Bus } from "./bus";
+import { filterInSitu } from "./util";
+import { DB } from "./db";
+
+
+const menus = typeof (_menus) !== "undefined" && _menus || _cmenus;
+
+const GATHER = "/bundles/content-gather.js";
+
+const CHROME_CONTEXTS = Object.freeze(new Set([
+  "all",
+  "audio",
+  "browser_action",
+  "editable",
+  "frame",
+  "image",
+  "launcher",
+  "link",
+  "page",
+  "page_action",
+  "selection",
+  "video",
+]));
+
+async function runContentJob(tab: Tab, file: string, msg: any) {
+  try {
+    if (tab && tab.incognito && msg) {
+      msg.private = tab.incognito;
+    }
+    const res = await tabs.executeScript(tab.id, {
+      file,
+      allFrames: true,
+      runAt: "document_start"
+    });
+    if (!msg) {
+      return res;
+    }
+    const promises = [];
+    const results: any[] = [];
+    for (const frame of await nav.getAllFrames({ tabId: tab.id })) {
+      promises.push(tabs.sendMessage(tab.id, msg, {
+        frameId: frame.frameId}
+      ).then(function(res: any) {
+        results.push(res);
+      }).catch(console.error));
+    }
+    await Promise.all(promises);
+    return results;
+  }
+  catch (ex) {
+    console.error("Failed to execute content script", file,
+      ex.message || ex.toString(), ex);
+    return [];
+  }
+}
+
+type SelectionOptions = {
+  selectionOnly: boolean;
+  allTabs: boolean;
+  turbo: boolean;
+  tab: Tab;
+};
+
+
+class Handler {
+  async processResults(turbo = false, results: any[]) {
+    const links = this.makeUnique(results, "links");
+    const media = this.makeUnique(results, "media");
+    await API[turbo ? "turbo" : "regular"](links, media);
+  }
+
+  makeUnique(results: any[], what: string) {
+    return makeUniqueItems(
+      results.filter(e => e[what]).map(e => {
+        const finisher = new Finisher(e);
+        return filterInSitu(e[what].
+          map((item: any) => finisher.finish(item)), e => !!e);
+      }));
+  }
+
+  async performSelection(options: SelectionOptions) {
+    try {
+      const tabOptions: any = {
+        currentWindow: true,
+        discarded: false,
+      };
+      if (!CHROME) {
+        tabOptions.hidden = false;
+      }
+      const selectedTabs = options.allTabs ?
+        await tabs.query(tabOptions) as any[] :
+        [options.tab];
+
+      const textLinks = await Prefs.get("text-links", true);
+      const gatherOptions = {
+        type: "DTA:gather",
+        selectionOnly: options.selectionOnly,
+        textLinks,
+        schemes: Array.from(ALLOWED_SCHEMES.values()),
+        transferable: TRANSFERABLE_PROPERTIES,
+      };
+
+      const results = await Promise.all(selectedTabs.
+        map((tab: any) => runContentJob(tab, GATHER, gatherOptions)));
+
+      await this.processResults(options.turbo, results.flat());
+    }
+    catch (ex) {
+      console.error(ex.toString(), ex.stack, ex);
+    }
+  }
+}
+
+function getMajor(version?: string) {
+  if (!version) {
+    return "";
+  }
+  const match = version.match(/^\d+\.\d+/);
+  if (!match) {
+    return "";
+  }
+  return match[0];
+}
+
+runtime.onInstalled.addListener(({reason, previousVersion}: OnInstalled) => {
+  const {version} = runtime.getManifest();
+  const major = getMajor(version);
+  const prevMajor = getMajor(previousVersion);
+  if (reason === "update" && major !== prevMajor) {
+    tabs.create({
+      url: `https://about.downthemall.org/changelog/?cur=${major}&prev=${prevMajor}`,
+    });
+  }
+  else if (reason === "install") {
+    tabs.create({
+      url: `https://about.downthemall.org/4.0/?cur=${major}`,
+    });
+  }
+});
+
+locale.then(() => {
+  const menuHandler = new class Menus extends Handler {
+    constructor() {
+      super();
+      this.onClicked = this.onClicked.bind(this);
+      const alls = new Map<string, string[]>();
+      const menuCreate = (options: any) => {
+        if (CHROME) {
+          delete options.icons;
+          options.contexts = options.contexts.
+            filter((e: string) => CHROME_CONTEXTS.has(e));
+          if (!options.contexts.length) {
+            return;
+          }
+        }
+        if (options.contexts.includes("all")) {
+          alls.set(options.id, options.contexts);
+        }
+        menus.create(options);
+      };
+      menuCreate({
+        id: "DTARegularLink",
+        contexts: ["link"],
+        icons: {
+          16: "/style/button-regular.png",
+          32: "/style/button-regular@2x.png",
+        },
+        title: _("dta.regular.link"),
+      });
+      menuCreate({
+        id: "DTATurboLink",
+        contexts: ["link"],
+        icons: {
+          16: "/style/button-turbo.png",
+          32: "/style/button-turbo@2x.png",
+        },
+        title: _("dta.turbo.link"),
+      });
+      menuCreate({
+        id: "DTARegularImage",
+        contexts: ["image"],
+        icons: {
+          16: "/style/button-regular.png",
+          32: "/style/button-regular@2x.png",
+        },
+        title: _("dta.regular.image"),
+      });
+      menuCreate({
+        id: "DTATurboImage",
+        contexts: ["image"],
+        icons: {
+          16: "/style/button-turbo.png",
+          32: "/style/button-turbo@2x.png",
+        },
+        title: _("dta.turbo.image"),
+      });
+      menuCreate({
+        id: "DTARegularMedia",
+        contexts: ["video", "audio"],
+        icons: {
+          16: "/style/button-regular.png",
+          32: "/style/button-regular@2x.png",
+        },
+        title: _("dta.regular.media"),
+      });
+      menuCreate({
+        id: "DTATurboMedia",
+        contexts: ["video", "audio"],
+        icons: {
+          16: "/style/button-turbo.png",
+          32: "/style/button-turbo@2x.png",
+        },
+        title: _("dta.turbo.media"),
+      });
+      menuCreate({
+        id: "DTARegularSelection",
+        contexts: ["selection"],
+        icons: {
+          16: "/style/button-regular.png",
+          32: "/style/button-regular@2x.png",
+        },
+        title: _("dta.regular.selection"),
+      });
+      menuCreate({
+        id: "DTATurboSelection",
+        contexts: ["selection"],
+        icons: {
+          16: "/style/button-turbo.png",
+          32: "/style/button-turbo@2x.png",
+        },
+        title: _("dta.turbo.selection"),
+      });
+      menuCreate({
+        id: "DTARegular",
+        contexts: ["all", "browser_action", "tools_menu"],
+        icons: {
+          16: "/style/button-regular.png",
+          32: "/style/button-regular@2x.png",
+        },
+        title: _("dta.regular"),
+      });
+      menuCreate({
+        id: "DTATurbo",
+        contexts: ["all", "browser_action", "tools_menu"],
+        icons: {
+          16: "/style/button-turbo.png",
+          32: "/style/button-turbo@2x.png",
+        },
+        title: _("dta.turbo"),
+      });
+      menuCreate({
+        id: "sep-1",
+        contexts: ["all", "browser_action", "tools_menu"],
+        type: "separator"
+      });
+      menuCreate({
+        id: "DTARegularAll",
+        contexts: ["all", "browser_action", "tools_menu"],
+        icons: {
+          16: "/style/button-regular.png",
+          32: "/style/button-regular@2x.png",
+        },
+        title: _("dta-regular-all"),
+      });
+      menuCreate({
+        id: "DTATurboAll",
+        contexts: ["all", "browser_action", "tools_menu"],
+        icons: {
+          16: "/style/button-turbo.png",
+          32: "/style/button-turbo@2x.png",
+        },
+        title: _("dta-turbo-all"),
+      });
+      const sep2ctx = menus.ACTION_MENU_TOP_LEVEL_LIMIT === 6 ?
+        ["all", "tools_menu"] :
+        ["all", "browser_action", "tools_menu"];
+      menuCreate({
+        id: "sep-2",
+        contexts: sep2ctx,
+        type: "separator"
+      });
+      menuCreate({
+        id: "DTAAdd",
+        contexts: ["all", "browser_action", "tools_menu"],
+        icons: {
+          16: "/style/add.svg",
+          32: "/style/add.svg",
+          64: "/style/add.svg",
+          128: "/style/add.svg",
+        },
+        title: _("add-download"),
+      });
+      menuCreate({
+        id: "sep-3",
+        contexts: ["all", "browser_action", "tools_menu"],
+        type: "separator"
+      });
+      menuCreate({
+        id: "DTAManager",
+        contexts: ["all", "browser_action", "tools_menu"],
+        icons: {
+          16: "/style/button-manager.png",
+          32: "/style/button-manager@2x.png",
+        },
+        title: _("manager.short"),
+      });
+      menuCreate({
+        id: "DTAPrefs",
+        contexts: ["all", "browser_action", "tools_menu"],
+        icons: {
+          16: "/style/settings.svg",
+          32: "/style/settings.svg",
+          64: "/style/settings.svg",
+          128: "/style/settings.svg",
+        },
+        title: _("prefs.short"),
+      });
+      Object.freeze(alls);
+
+      const adjustMenus = (v: boolean) => {
+        for (const [id, contexts] of alls.entries()) {
+          const adjusted = v ?
+            contexts.filter(e => e !== "all") :
+            contexts;
+          menus.update(id, {
+            contexts: adjusted
+          });
+        }
+      };
+      Prefs.get("hide-context", false).then((v: boolean) => {
+      // This is the initial load, so no need to adjust when visible already
+        if (!v) {
+          return;
+        }
+        adjustMenus(v);
+      });
+      Prefs.on("hide-context", (prefs, key, value: boolean) => {
+        adjustMenus(value);
+      });
+
+      menus.onClicked.addListener(this.onClicked);
+    }
+
+    *makeSingleItemList(url: string, results: any[]) {
+      for (const result of results) {
+        const finisher = new Finisher(result);
+        for (const list of [result.links, result.media]) {
+          for (const e of list) {
+            if (e.url !== url) {
+              continue;
+            }
+            const finished = finisher.finish(e);
+            if (!finished) {
+              continue;
+            }
+            yield finished;
+          }
+        }
+      }
+    }
+
+    async findSingleItem(tab: Tab, url: string, turbo = false) {
+      if (!url) {
+        return;
+      }
+      const results = await runContentJob(
+        tab, "/bundles/content-gather.js", {
+          type: "DTA:gather",
+          selectionOnly: false,
+          schemes: Array.from(ALLOWED_SCHEMES.values()),
+          transferable: TRANSFERABLE_PROPERTIES,
+        });
+      const found = Array.from(this.makeSingleItemList(url, results));
+      const unique = makeUniqueItems([found]);
+      if (!unique.length) {
+        return;
+      }
+      const [item] = unique;
+      API[turbo ? "singleTurbo" : "singleRegular"](item);
+    }
+
+    onClicked(info: MenuClickInfo, tab: Tab) {
+      if (!tab.id) {
+        return;
+      }
+      const {menuItemId} = info;
+      const {[`onClicked${menuItemId}`]: handler}: any = this;
+      if (!handler) {
+        console.error("Invalid Handler for", menuItemId);
+        return;
+      }
+      const rv: Promise<void> | void = handler.call(this, info, tab);
+      if (rv && rv.catch) {
+        rv.catch(console.error);
+      }
+    }
+
+    async emulate(action: string) {
+      const tab = await tabs.query({
+        active: true,
+        currentWindow: true,
+      });
+      if (!tab || !tab.length) {
+        return;
+      }
+      this.onClicked({
+        menuItemId: action
+      }, tab[0]);
+    }
+
+    async onClickedDTARegular(info: MenuClickInfo, tab: Tab) {
+      return await this.performSelection({
+        selectionOnly: false,
+        allTabs: false,
+        turbo: false,
+        tab,
+      });
+    }
+
+    async onClickedDTARegularAll(info: MenuClickInfo, tab: Tab) {
+      return await this.performSelection({
+        selectionOnly: false,
+        allTabs: true,
+        turbo: false,
+        tab,
+      });
+    }
+
+    async onClickedDTARegularSelection(info: MenuClickInfo, tab: Tab) {
+      return await this.performSelection({
+        selectionOnly: true,
+        allTabs: false,
+        turbo: false,
+        tab,
+      });
+    }
+
+    async onClickedDTATurbo(info: MenuClickInfo, tab: Tab) {
+      return await this.performSelection({
+        selectionOnly: false,
+        allTabs: false,
+        turbo: true,
+        tab,
+      });
+    }
+
+    async onClickedDTATurboAll(info: MenuClickInfo, tab: Tab) {
+      return await this.performSelection({
+        selectionOnly: false,
+        allTabs: true,
+        turbo: true,
+        tab,
+      });
+    }
+
+    async onClickedDTATurboSelection(info: MenuClickInfo, tab: Tab) {
+      return await this.performSelection({
+        selectionOnly: true,
+        allTabs: false,
+        turbo: true,
+        tab,
+      });
+    }
+
+    async onClickedDTARegularLink(info: MenuClickInfo, tab: Tab) {
+      if (!info.linkUrl) {
+        return;
+      }
+      await this.findSingleItem(tab, info.linkUrl, false);
+    }
+
+    async onClickedDTATurboLink(info: MenuClickInfo, tab: Tab) {
+      if (!info.linkUrl) {
+        return;
+      }
+      await this.findSingleItem(tab, info.linkUrl, true);
+    }
+
+    async onClickedDTARegularImage(info: MenuClickInfo, tab: Tab) {
+      if (!info.srcUrl) {
+        return;
+      }
+      await this.findSingleItem(tab, info.srcUrl, false);
+    }
+
+    async onClickedDTATurboImage(info: MenuClickInfo, tab: Tab) {
+      if (!info.srcUrl) {
+        return;
+      }
+      await this.findSingleItem(tab, info.srcUrl, true);
+    }
+
+    async onClickedDTARegularMedia(info: MenuClickInfo, tab: Tab) {
+      if (!info.srcUrl) {
+        return;
+      }
+      await this.findSingleItem(tab, info.srcUrl, false);
+    }
+
+    async onClickedDTATurboMedia(info: MenuClickInfo, tab: Tab) {
+      if (!info.srcUrl) {
+        return;
+      }
+      await this.findSingleItem(tab, info.srcUrl, true);
+    }
+
+    onClickedDTAAdd() {
+      API.singleRegular(null);
+    }
+
+    async onClickedDTAManager() {
+      await openManager();
+    }
+
+    async onClickedDTAPrefs() {
+      await openPrefs();
+    }
+  }();
+
+  new class Action extends Handler {
+    constructor() {
+      super();
+      this.onClicked = this.onClicked.bind(this);
+      action.onClicked.addListener(this.onClicked);
+      Prefs.get("button-type", false).then(v => this.adjust(v));
+      Prefs.on("button-type", (prefs, key, value) => {
+        this.adjust(value);
+      });
+    }
+
+    adjust(type: string) {
+      action.setPopup({
+        popup: type !== "popup" ? "" : "/windows/popup.html"
+      });
+      let icons;
+      switch (type) {
+      case "popup":
+        icons = {
+          16: "/style/icon16.png",
+          32: "/style/icon32.png",
+          48: "/style/icon48.png",
+          64: "/style/icon64.png",
+          128: "/style/icon128.png",
+          256: "/style/icon256.png"
+        };
+        break;
+
+      case "dta":
+        icons = {
+          16: "/style/button-regular.png",
+          32: "/style/button-regular@2x.png",
+        };
+        break;
+
+      case "turbo":
+        icons = {
+          16: "/style/button-turbo.png",
+          32: "/style/button-turbo@2x.png",
+        };
+        break;
+
+      case "manager":
+        icons = {
+          16: "/style/button-manager.png",
+          32: "/style/button-manager@2x.png",
+        };
+        break;
+      }
+      action.setIcon({path: icons});
+    }
+
+    async onClicked() {
+      switch (await Prefs.get("button-type")) {
+      case "popup":
+        break;
+
+      case "dta":
+        menuHandler.emulate("DTARegular");
+        break;
+
+      case "turbo":
+        menuHandler.emulate("DTATurbo");
+        break;
+
+      case "manager":
+        menuHandler.emulate("DTAManager");
+        break;
+      }
+    }
+  }();
+
+
+  Bus.on("do-regular", () => menuHandler.emulate("DTARegular"));
+  Bus.on("do-regular-all", () => menuHandler.emulate("DTARegularAll"));
+  Bus.on("do-turbo", () => menuHandler.emulate("DTATurbo"));
+  Bus.on("do-turbo-all", () => menuHandler.emulate("DTATurboAll"));
+  Bus.on("do-single", () => API.singleRegular(null));
+  Bus.on("open-manager", () => openManager(true));
+  Bus.on("open-prefs", () => openPrefs());
+
+  (async function init() {
+    const urlBase = runtime.getURL("");
+    history.onVisited.addListener(({url}: {url: string}) => {
+      if (!url || !url.startsWith(urlBase)) {
+        return;
+      }
+      history.deleteUrl({url});
+    });
+    const results: {url?: string}[] = await history.search({text: urlBase});
+    for (const {url} of results) {
+      if (!url) {
+        continue;
+      }
+      history.deleteUrl({url});
+    }
+
+    if (!CHROME) {
+      const sessionRemover = async () => {
+        for (const s of await sessions.getRecentlyClosed()) {
+          if (s.tab) {
+            if (s.tab.url.startsWith(urlBase)) {
+              await sessions.forgetClosedTab(s.tab.windowId, s.tab.sessionId);
+            }
+            continue;
+          }
+          if (!s.window || !s.window.tabs || s.window.tabs.length > 1) {
+            continue;
+          }
+          const [tab] = s.window.tabs;
+          if (tab.url.startsWith(urlBase)) {
+            await sessions.forgetClosedWindow(s.window.sessionId);
+          }
+        }
+      };
+      sessions.onChanged.addListener(sessionRemover);
+      await sessionRemover();
+    }
+
+    try {
+      await DB.init();
+    }
+    catch (ex) {
+      console.error("db init", ex.toString(), ex.message, ex.stack, ex);
+    }
+
+    await Prefs.set("last-run", new Date());
+    await filters();
+    await getManager();
+  })().catch(ex => {
+    console.error("Failed to init components", ex.toString(), ex.stack, ex);
+  });
+});

extensions/downthemall/lib/batches.ts

@@ -0,0 +1,224 @@
+"use strict";
+// License: MIT
+
+const PROCESS = Symbol();
+
+interface Generator extends Iterable<string> {
+  readonly preview: string;
+  readonly length: number;
+}
+
+class Literal implements Generator {
+  public readonly preview: string;
+
+  public readonly str: string;
+
+  public readonly length: number;
+
+  constructor(str: string) {
+    this.preview = this.str = str;
+    this.length = 1;
+    Object.freeze(this);
+  }
+
+  *[Symbol.iterator]() {
+    yield this.str;
+  }
+}
+
+function reallyParseInt(str: string) {
+  if (!/^[+-]?[0-9]+$/.test(str)) {
+    throw new Error("Not a number");
+  }
+  const rv = parseInt(str, 10);
+  if (isNaN(rv) || rv !== (rv | 0)) {
+    throw new Error("Not a number");
+  }
+  return rv;
+}
+
+class Numeral implements Generator {
+  public readonly start: number;
+
+  public readonly stop: number;
+
+  public readonly step: number;
+
+  public readonly digits: number;
+
+  public readonly length: number;
+
+  public readonly preview: string;
+
+  constructor(str: string) {
+    const rawpieces = str.split(":").map(e => e.trim());
+    const pieces = rawpieces.map(e => reallyParseInt(e));
+    if (pieces.length < 2) {
+      throw new Error("Invalid input");
+    }
+    const [start, stop, step] = pieces;
+    if (step === 0) {
+      throw new Error("Invalid step");
+    }
+    this.step = !step ? 1 : step;
+    const dir = this.step > 0;
+    if (dir && start > stop) {
+      throw new Error("Invalid sequence");
+    }
+    else if (!dir && start < stop) {
+      throw new Error("Invalid sequence");
+    }
+    this.start = start;
+    this.stop = stop;
+    this.digits = dir ? rawpieces[0].length : rawpieces[1].length;
+    this.length = Math.floor(
+      (this.stop - this.start + (dir ? 1 : -1)) / this.step);
+    this.preview = this[Symbol.iterator]().next().value as string;
+    Object.freeze(this);
+  }
+
+  *[Symbol.iterator]() {
+    const {digits, start, stop, step} = this;
+    const dir = step > 0;
+    for (let i = start; (dir ? i <= stop : i >= stop); i += step) {
+      const rv = i.toString();
+      const len = digits - rv.length;
+      if (len > 0) {
+        yield "0".repeat(len) + rv;
+      }
+      else {
+        yield rv;
+      }
+    }
+  }
+}
+
+class Character implements Generator {
+  public readonly start: number;
+
+  public readonly stop: number;
+
+  public readonly step: number;
+
+  public readonly length: number;
+
+  public readonly preview: string;
+
+  constructor(str: string) {
+    const rawpieces = str.split(":").map(e => e.trim());
+    const pieces = rawpieces.map((e, i) => {
+      if (i === 2) {
+        return reallyParseInt(e);
+      }
+      if (e.length > 1) {
+        throw new Error("Malformed Character sequence");
+      }
+      return e.charCodeAt(0);
+    });
+    if (pieces.length < 2) {
+      throw new Error("Invalid input");
+    }
+    const [start, stop, step] = pieces;
+    if (step === 0) {
+      throw new Error("Invalid step");
+    }
+    this.step = !step ? 1 : step;
+    const dir = this.step > 0;
+    if (dir && start > stop) {
+      throw new Error("Invalid sequence");
+    }
+    else if (!dir && start < stop) {
+      throw new Error("Invalid sequence");
+    }
+    this.start = start;
+    this.stop = stop;
+    this.length = Math.floor(
+      (this.stop - this.start + (dir ? 1 : -1)) / this.step);
+    this.preview = this[Symbol.iterator]().next().value as string;
+    Object.freeze(this);
+  }
+
+  *[Symbol.iterator]() {
+    const {start, stop, step} = this;
+    const dir = step > 0;
+    for (let i = start; (dir ? i <= stop : i >= stop); i += step) {
+      yield String.fromCharCode(i);
+    }
+  }
+}
+
+export class BatchGenerator implements Generator {
+  private readonly gens: Generator[];
+
+  public readonly hasInvalid: boolean;
+
+  public readonly length: number;
+
+  public readonly preview: string;
+
+  constructor(str: string) {
+    this.gens = [];
+    let i;
+    this.hasInvalid = false;
+    while ((i = str.search(/\[.+?:.+?\]/)) !== -1) {
+      if (i !== 0) {
+        this.gens.push(new Literal(str.slice(0, i)));
+        str = str.slice(i);
+      }
+      const end = str.indexOf("]");
+      if (end <= 0) {
+        throw new Error("Something went terribly wrong");
+      }
+      const tok = str.slice(1, end);
+      str = str.slice(end + 1);
+      try {
+        this.gens.push(new Numeral(tok));
+      }
+      catch {
+        try {
+          this.gens.push(new Character(tok));
+        }
+        catch {
+          this.gens.push(new Literal(`[${tok}]`));
+          this.hasInvalid = true;
+        }
+      }
+    }
+    if (str) {
+      this.gens.push(new Literal(str));
+    }
+
+    // Merge literls
+    for (let i = this.gens.length; i > 1; --i) {
+      const sgen0 = this.gens[i - 1];
+      const sgen1 = this.gens[i];
+      if (sgen0 instanceof Literal && sgen1 instanceof Literal) {
+        this.gens[i - 1] = new Literal(sgen0.str + sgen1.str);
+        this.gens.splice(i, 1);
+      }
+    }
+    this.length = this.gens.reduce((p, c) => p * c.length, 1);
+    this.preview = this.gens.reduce((p, c) => p + c.preview, "");
+  }
+
+  static *[PROCESS](gens: Generator[]): Iterable<string> {
+    const cur = gens.pop();
+    if (!cur) {
+      yield "";
+      return;
+    }
+    for (const g of BatchGenerator[PROCESS](gens)) {
+      for (const tail of cur) {
+        yield g + tail;
+      }
+    }
+  }
+
+  *[Symbol.iterator]() {
+    if (this.length === 1) {
+      yield this.preview;
+      return;
+    }
+    yield *BatchGenerator[PROCESS](this.gens.slice());
+  }
+}

extensions/downthemall/lib/browser.ts

@@ -0,0 +1,120 @@
+"use strict";
+
+// eslint-disable-next-line @typescript-eslint/no-var-requires
+const polyfill = require("webextension-polyfill");
+
+interface ExtensionListener {
+  addListener: (listener: Function) => void;
+  removeListener: (listener: Function) => void;
+}
+
+export interface MessageSender {
+  readonly tab?: Tab;
+  readonly frameId?: number;
+  readonly id?: number;
+  readonly url?: string;
+  readonly tlsChannelId?: string;
+}
+
+
+export interface Tab {
+  readonly id?: number;
+  readonly incognito?: boolean;
+}
+
+export interface MenuClickInfo {
+  readonly menuItemId: string | number;
+  readonly button?: number;
+  readonly linkUrl?: string;
+  readonly srcUrl?: string;
+}
+
+
+export interface RawPort {
+  readonly error: any;
+  readonly name: string;
+  readonly sender?: MessageSender;
+  readonly onDisconnect: ExtensionListener;
+  readonly onMessage: ExtensionListener;
+  disconnect: () => void;
+  postMessage: (message: any) => void;
+}
+
+interface WebRequestFilter {
+  urls?: string[];
+}
+
+interface WebRequestListener {
+  addListener(
+    callback: Function,
+    filter: WebRequestFilter,
+    extraInfoSpec: string[]
+    ): void;
+  removeListener(callback: Function): void;
+}
+
+type Header = {name: string; value: string};
+
+export interface DownloadOptions {
+  conflictAction: string;
+  filename?: string;
+  saveAs: boolean;
+  url: string;
+  method?: string;
+  body?: string;
+  incognito?: boolean;
+  headers: Header[];
+}
+
+export interface DownloadsQuery {
+  id?: number;
+}
+
+interface Downloads {
+  download(download: DownloadOptions): Promise<number>;
+  open(manId: number): Promise<void>;
+  show(manId: number): Promise<void>;
+  pause(manId: number): Promise<void>;
+  resume(manId: number): Promise<void>;
+  cancel(manId: number): Promise<void>;
+  erase(query: DownloadsQuery): Promise<void>;
+  search(query: DownloadsQuery): Promise<any[]>;
+  getFileIcon(id: number, options?: any): Promise<string>;
+  setShelfEnabled(state: boolean): void;
+  removeFile(manId: number): Promise<void>;
+  readonly onCreated: ExtensionListener;
+  readonly onChanged: ExtensionListener;
+  readonly onErased: ExtensionListener;
+  readonly onDeterminingFilename?: ExtensionListener;
+}
+
+interface WebRequest {
+  readonly onBeforeSendHeaders: WebRequestListener;
+  readonly onSendHeaders: WebRequestListener;
+  readonly onHeadersReceived: WebRequestListener;
+}
+
+export interface OnInstalled {
+  readonly reason: string;
+  readonly previousVersion?: string;
+  readonly temporary: boolean;
+}
+
+export const {browserAction} = polyfill;
+export const {contextMenus} = polyfill;
+export const {downloads}: {downloads: Downloads} = polyfill;
+export const {extension} = polyfill;
+export const {history} = polyfill;
+export const {menus} = polyfill;
+export const {notifications} = polyfill;
+export const {runtime} = polyfill;
+export const {sessions} = polyfill;
+export const {storage} = polyfill;
+export const {tabs} = polyfill;
+export const {webNavigation} = polyfill;
+export const {webRequest}: {webRequest: WebRequest} = polyfill;
+export const {windows} = polyfill;
+export const {theme} = polyfill;
+
+export const CHROME = navigator.appVersion.includes("Chrome/");
+export const OPERA = navigator.appVersion.includes("OPR/");

extensions/downthemall/lib/bus.ts

@@ -0,0 +1,131 @@
+"use strict";
+// License: MIT
+
+import { EventEmitter } from "./events";
+// eslint-disable-next-line no-unused-vars
+import {runtime, tabs, RawPort, MessageSender} from "./browser";
+
+export class Port extends EventEmitter {
+  private port: RawPort | null;
+
+  private disconnected = false;
+
+  constructor(port: RawPort) {
+    super();
+    this.port = port;
+
+    // Nasty firefox bug, thus listen for tab removal explicitly
+    if (port.sender && port.sender.tab && port.sender.tab.id) {
+      const otherTabId = port.sender.tab.id;
+      const tabListener = (tabId: number) => {
+        if (tabId !== otherTabId) {
+          return;
+        }
+        this.disconnect();
+      };
+      tabs.onRemoved.addListener(tabListener);
+    }
+    port.onMessage.addListener(this.onMessage.bind(this));
+    port.onDisconnect.addListener(this.disconnect.bind(this));
+  }
+
+  disconnect() {
+    if (this.disconnected) {
+      return;
+    }
+    this.disconnected = true;
+    const {port} = this;
+    this.port = null; // Break the cycle
+    this.emit("disconnect", this, port);
+  }
+
+  get name() {
+    if (!this.port) {
+      return null;
+    }
+    return this.port.name;
+  }
+
+  get id() {
+    if (!this.port || !this.port.sender) {
+      return null;
+    }
+    return this.port.sender.id;
+  }
+
+  get isSelf() {
+    return this.id === runtime.id;
+  }
+
+  post(msg: string, ...data: any[]) {
+    if (!this.port) {
+      return;
+    }
+    if (!data) {
+      this.port.postMessage({msg});
+      return;
+    }
+    if (data.length === 1) {
+      [data] = data;
+    }
+    this.port.postMessage({msg, data});
+  }
+
+  onMessage(message: any) {
+    if (!this.port) {
+      return;
+    }
+    if (Array.isArray(message)) {
+      message.forEach(this.onMessage, this);
+      return;
+    }
+    if (Object.keys(message).includes("msg")) {
+      this.emit(message.msg, message);
+      return;
+    }
+    if (typeof message === "string") {
+      this.emit(message);
+      return;
+    }
+    console.error(`Unhandled message in ${this.port.name}:`, message);
+  }
+}
+
+export const Bus = new class extends EventEmitter {
+  private readonly ports: EventEmitter;
+
+  public readonly onPort: (event: string, port: (port: Port) => void) => void;
+
+  public readonly offPort: Function;
+
+  public readonly oncePort: (event: string, port: (port: Port) => void) => void;
+
+  constructor() {
+    super();
+    this.ports = new EventEmitter();
+    this.onPort = this.ports.on.bind(this.ports);
+    this.offPort = this.ports.off.bind(this.ports);
+    this.oncePort = this.ports.once.bind(this.ports);
+    runtime.onMessage.addListener(this.onMessage.bind(this));
+    runtime.onConnect.addListener(this.onConnect.bind(this));
+  }
+
+  onMessage(msg: any, sender: MessageSender, callback: any) {
+    let {type = null} = msg;
+    if (!type) {
+      type = msg;
+    }
+    this.emit(type, msg, callback);
+  }
+
+  onConnect(port: RawPort) {
+    if (!port.name) {
+      port.disconnect();
+      return;
+    }
+    const wrapped = new Port(port);
+    if (!this.ports.emit(port.name, wrapped)) {
+      wrapped.disconnect();
+    }
+  }
+}();

extensions/downthemall/lib/cdheaderparser.ts

@@ -0,0 +1,230 @@
+/**
+ * (c) 2017 Rob Wu <rob@robwu.nl> (https://robwu.nl)
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+/* eslint-disable max-len,no-magic-numbers */
+// License: MPL-2
+
+/**
+  * This typescript port was done by Nils Maier based on
+  * https://github.com/Rob--W/open-in-browser/blob/83248155b633ed41bc9cdb1205042653e644abd2/extension/content-disposition.js
+  * Special thanks goes to Rob doing all the heavy lifting and putting
+  * it together in a reuseable, open source'd library.
+  */
+
+const R_RFC6266 = /(?:^|;)\s*filename\*\s*=\s*([^";\s][^;\s]*|"(?:[^"\\]|\\"?)+"?)/i;
+const R_RFC5987 = /(?:^|;)\s*filename\s*=\s*([^";\s][^;\s]*|"(?:[^"\\]|\\"?)+"?)/i;
+
+function unquoteRFC2616(value: string) {
+  if (!value.startsWith("\"")) {
+    return value;
+  }
+
+  const parts = value.slice(1).split("\\\"");
+  // Find the first unescaped " and terminate there.
+  for (let i = 0; i < parts.length; ++i) {
+    const quotindex = parts[i].indexOf("\"");
+    if (quotindex !== -1) {
+      parts[i] = parts[i].slice(0, quotindex);
+      // Truncate and stop the iteration.
+      parts.length = i + 1;
+    }
+    parts[i] = parts[i].replace(/\\(.)/g, "$1");
+  }
+  value = parts.join("\"");
+  return value;
+}
+
+export class CDHeaderParser {
+  private needsFixup: boolean;
+
+  // We need to keep this per instance, because of the global flag.
+  // Hence we need to reset it after a use.
+  private R_MULTI = /(?:^|;)\s*filename\*((?!0\d)\d+)(\*?)\s*=\s*([^";\s][^;\s]*|"(?:[^"\\]|\\"?)+"?)/gi;
+
+  /**
+   * Parse a content-disposition header, with relaxed spec tolerance
+   *
+   * @param {string} header Header to parse
+   * @returns {string} Parsed header
+   */
+  parse(header: string) {
+    this.needsFixup = true;
+
+    // filename*=ext-value ("ext-value" from RFC 5987, referenced by RFC 6266).
+    {
+      const match = R_RFC6266.exec(header);
+      if (match) {
+        const [, tmp] = match;
+        let filename = unquoteRFC2616(tmp);
+        filename = unescape(filename);
+        filename = this.decodeRFC5897(filename);
+        filename = this.decodeRFC2047(filename);
+        return this.maybeFixupEncoding(filename);
+      }
+    }
+
+    // Continuations (RFC 2231 section 3, referenced by RFC 5987 section 3.1).
+    // filename*n*=part
+    // filename*n=part
+    {
+      const tmp = this.getParamRFC2231(header);
+      if (tmp) {
+        // RFC 2047, section
+        const filename = this.decodeRFC2047(tmp);
+        return this.maybeFixupEncoding(filename);
+      }
+    }
+
+    // filename=value (RFC 5987, section 4.1).
+    {
+      const match = R_RFC5987.exec(header);
+      if (match) {
+        const [, tmp] = match;
+        let filename = unquoteRFC2616(tmp);
+        filename = this.decodeRFC2047(filename);
+        return this.maybeFixupEncoding(filename);
+      }
+    }
+    return "";
+  }
+
+  private maybeDecode(encoding: string, value: string) {
+    if (!encoding) {
+      return value;
+    }
+    const bytes = Array.from(value, c => c.charCodeAt(0));
+    if (!bytes.every(code => code <= 0xff)) {
+      return value;
+    }
+    try {
+      value = new TextDecoder(encoding, {fatal: true}).
+        decode(new Uint8Array(bytes));
+      this.needsFixup = false;
+    }
+    catch {
+      // TextDecoder constructor threw - unrecognized encoding.
+    }
+    return value;
+  }
+
+  private maybeFixupEncoding(value: string) {
+    if (!this.needsFixup && /[\x80-\xff]/.test(value)) {
+      return value;
+    }
+
+    // Maybe multi-byte UTF-8.
+    value = this.maybeDecode("utf-8", value);
+    if (!this.needsFixup) {
+      return value;
+    }
+
+    // Try iso-8859-1 encoding.
+    return this.maybeDecode("iso-8859-1", value);
+  }
+
+  private getParamRFC2231(value: string) {
+    const matches: string[][] = [];
+
+    // Iterate over all filename*n= and filename*n*= with n being an integer
+    // of at least zero. Any non-zero number must not start with '0'.
+    let match;
+    this.R_MULTI.lastIndex = 0;
+    while ((match = this.R_MULTI.exec(value)) !== null) {
+      const [, num, quot, part] = match;
+      const n = parseInt(num, 10);
+      if (n in matches) {
+        // Ignore anything after the invalid second filename*0.
+        if (n === 0) {
+          break;
+        }
+        continue;
+      }
+      matches[n] = [quot, part];
+    }
+
+    const parts: string[] = [];
+    for (let n = 0; n < matches.length; ++n) {
+      if (!(n in matches)) {
+        // Numbers must be consecutive. Truncate when there is a hole.
+        break;
+      }
+      const [quot, rawPart] = matches[n];
+      let part = unquoteRFC2616(rawPart);
+      if (quot) {
+        part = unescape(part);
+        if (n === 0) {
+          part = this.decodeRFC5897(part);
+        }
+      }
+      parts.push(part);
+    }
+    return parts.join("");
+  }
+
+  private decodeRFC2047(value: string) {
+    // RFC 2047-decode the result. Firefox tried to drop support for it, but
+    // backed out because some servers use it - https://bugzil.la/875615
+    // Firefox's condition for decoding is here:
+
+    // eslint-disable-next-line max-len
+    // https://searchfox.org/mozilla-central/rev/4a590a5a15e35d88a3b23dd6ac3c471cf85b04a8/netwerk/mime/nsMIMEHeaderParamImpl.cpp#742-748
+
+    // We are more strict and only recognize RFC 2047-encoding if the value
+    // starts with "=?", since then it is likely that the full value is
+    // RFC 2047-encoded.
+
+    // Firefox also decodes words even where RFC 2047 section 5 states:
+    // "An 'encoded-word' MUST NOT appear within a 'quoted-string'."
+
+    // eslint-disable-next-line no-control-regex
+    if (!value.startsWith("=?") || /[\x00-\x19\x80-\xff]/.test(value)) {
+      return value;
+    }
+    // RFC 2047, section 2.4
+    // encoded-word = "=?" charset "?" encoding "?" encoded-text "?="
+    // charset = token (but let's restrict to characters that denote a
+    //           possibly valid encoding).
+    // encoding = q or b
+    // encoded-text = any printable ASCII character other than ? or space.
+    //                ... but Firefox permits ? and space.
+    return value.replace(
+      /=\?([\w-]*)\?([QqBb])\?((?:[^?]|\?(?!=))*)\?=/g,
+      (_, charset, encoding, text) => {
+        if (encoding === "q" || encoding === "Q") {
+          // RFC 2047 section 4.2.
+          text = text.replace(/_/g, " ");
+          text = text.replace(/=([0-9a-fA-F]{2})/g,
+            (_: string, hex: string) => String.fromCharCode(parseInt(hex, 16)));
+          return this.maybeDecode(charset, text);
+        }
+
+        // else encoding is b or B - base64 (RFC 2047 section 4.1)
+        try {
+          text = atob(text);
+        }
+        catch {
+          // ignored
+        }
+        return this.maybeDecode(charset, text);
+      });
+  }
+
+  private decodeRFC5897(extValue: string) {
+    // Decodes "ext-value" from RFC 5987.
+    const extEnd = extValue.indexOf("'");
+    if (extEnd < 0) {
+      // Some servers send "filename*=" without encoding'language' prefix,
+      // e.g. in https://github.com/Rob--W/open-in-browser/issues/26
+      // Let's accept the value like Firefox (57) (Chrome 62 rejects it).
+      return extValue;
+    }
+    const encoding = extValue.slice(0, extEnd);
+    const langvalue = extValue.slice(extEnd + 1);
+    // Ignore language (RFC 5987 section 3.2.1, and RFC 6266 section 4.1 ).
+    return this.maybeDecode(encoding, langvalue.replace(/^[^']*'/, ""));
+  }
+}

extensions/downthemall/lib/constants.ts

@@ -0,0 +1,18 @@
+"use strict";
+// License: MIT
+
+export const ALLOWED_SCHEMES = Object.freeze(new Set<string>([
+  "http:",
+  "https:",
+  "ftp:",
+]));
+
+export const TRANSFERABLE_PROPERTIES = Object.freeze([
+  "fileName",
+  "title",
+  "description"
+]);
+
+export const TYPE_LINK = 1;
+export const TYPE_MEDIA = 2;
+export const TYPE_ALL = 3;

extensions/downthemall/lib/db.ts

@@ -0,0 +1,266 @@
+"use strict";
+// License: MIT
+
+// eslint-disable-next-line no-unused-vars
+import { BaseItem } from "./item";
+// eslint-disable-next-line no-unused-vars
+import { Download } from "./manager/download";
+import { RUNNING, QUEUED, RETRYING } from "./manager/state";
+import { storage } from "./browser";
+import { sort } from "./sorting";
+
+const VERSION = 1;
+const STORE = "queue";
+
+interface Database {
+  init(): Promise<void>;
+  saveItems(items: Download[]): Promise<unknown>;
+  deleteItems(items: any[]): Promise<void>;
+  getAll(): Promise<BaseItem[]>;
+}
+
+export class IDB implements Database {
+  private db?: IDBDatabase;
+
+  constructor() {
+    this.db = undefined;
+    this.getAllInternal = this.getAllInternal.bind(this);
+  }
+
+  async init() {
+    if (this.db) {
+      return;
+    }
+    await new Promise((resolve, reject) => {
+      const req = indexedDB.open("downloads", VERSION);
+      req.onupgradeneeded = evt => {
+        const db = req.result;
+        switch (evt.oldVersion) {
+        case 0: {
+          const queueStore = db.createObjectStore(STORE, {
+            keyPath: "dbId",
+            autoIncrement: true
+          });
+          queueStore.createIndex("by_position", "position", {unique: false});
+          break;
+        }
+        }
+      };
+      req.onerror = ex => reject(ex);
+      req.onsuccess = () => {
+        this.db = req.result;
+        resolve();
+      };
+    });
+  }
+
+  getAllInternal(resolve: (items: BaseItem[]) => void, reject: Function) {
+    if (!this.db) {
+      reject(new Error("db closed"));
+      return;
+    }
+    const items: BaseItem[] = [];
+    const transaction = this.db.transaction(STORE, "readonly");
+    transaction.onerror = ex => reject(ex);
+    const store = transaction.objectStore(STORE);
+    const index = store.index("by_position");
+    index.openCursor().onsuccess = event => {
+      const cursor = (event.target as IDBRequest<IDBCursorWithValue>).result;
+      if (!cursor) {
+        resolve(items);
+        return;
+      }
+      items.push(cursor.value);
+      cursor.continue();
+    };
+  }
+
+  async getAll() {
+    await this.init();
+    return await new Promise(this.getAllInternal);
+  }
+
+  saveItemsInternal(items: Download[], resolve: Function, reject: Function) {
+    if (!items || !items.length || !this.db) {
+      resolve();
+      return;
+    }
+    try {
+      const transaction = this.db.transaction(STORE, "readwrite");
+      transaction.onerror = ex => reject(ex);
+      transaction.oncomplete = () => resolve();
+      const store = transaction.objectStore(STORE);
+      for (const item of items) {
+        if (item.private) {
+          continue;
+        }
+        const json = item.toJSON();
+        if (item.state === RUNNING || item.state === RETRYING) {
+          json.state = QUEUED;
+        }
+        const req = store.put(json);
+        if (!("dbId" in item) || item.dbId < 0) {
+          req.onsuccess = () => item.dbId = req.result as number;
+        }
+      }
+    }
+    catch (ex) {
+      reject(ex);
+    }
+  }
+
+  async saveItems(items: Download[]) {
+    await this.init();
+    return await new Promise(this.saveItemsInternal.bind(this, items));
+  }
+
+  deleteItemsInternal(items: any[], resolve: () => void, reject: Function) {
+    if (!items || !items.length || !this.db) {
+      resolve();
+      return;
+    }
+    try {
+      const transaction = this.db.transaction(STORE, "readwrite");
+      transaction.onerror = ex => reject(ex);
+      transaction.oncomplete = () => resolve();
+      const store = transaction.objectStore(STORE);
+      for (const item of items) {
+        if (item.private) {
+          continue;
+        }
+        if (!("dbId" in item)) {
+          continue;
+        }
+        store.delete(item.dbId);
+      }
+    }
+    catch (ex) {
+      console.error(ex.message, ex);
+      reject(ex);
+    }
+  }
+
+  async deleteItems(items: any[]) {
+    if (!items.length) {
+      return;
+    }
+    await this.init();
+    await new Promise(this.deleteItemsInternal.bind(this, items));
+  }
+}
+
+class StorageDB implements Database {
+  private counter = 1;
+
+  async init(): Promise<void> {
+    const {db = null} = await storage.local.get("db");
+    if (!db || !db.counter) {
+      return;
+    }
+    this.counter = db.counter;
+  }
+
+  async saveItems(items: Download[]) {
+    const db: any = {items: []};
+    for (const item of items) {
+      if (!item.dbId) {
+        item.dbId = ++this.counter;
+      }
+      db.items.push(item.toJSON());
+    }
+    db.counter = this.counter;
+    await storage.local.set({db});
+  }
+
+  async deleteItems(items: any[]): Promise<void> {
+    const gone = new Set(items.map(i => i.dbId));
+    const {db = null} = await storage.local.get("db");
+    if (!db) {
+      return;
+    }
+    db.items = db.items.filter((i: any) => !gone.has(i.dbId));
+    await storage.local.set({db});
+  }
+
+  async getAll() {
+    const {db = null} = await storage.local.get("db");
+    if (!db || !Array.isArray(db.items)) {
+      return [];
+    }
+    return sort(db.items, (i: any) => i.position) as BaseItem[];
+  }
+}
+
+class MemoryDB implements Database {
+  private counter = 1;
+
+  private items = new Map();
+
+  init(): Promise<void> {
+    return Promise.resolve();
+  }
+
+  saveItems(items: Download[]) {
+    for (const item of items) {
+      if (item.private) {
+        continue;
+      }
+      if (!item.dbId) {
+        item.dbId = ++this.counter;
+      }
+      this.items.set(item.dbId, item.toJSON());
+    }
+    return Promise.resolve();
+  }
+
+  deleteItems(items: any[]) {
+    for (const item of items) {
+      if (!("dbId" in item)) {
+        continue;
+      }
+      this.items.delete(item.dbId);
+    }
+    return Promise.resolve();
+  }
+
+  getAll(): Promise<BaseItem[]> {
+    return Promise.resolve(Array.from(this.items.values()));
+  }
+}
+
+export const DB = new class DBWrapper implements Database {
+  saveItems(items: Download[]): Promise<unknown> {
+    return this.db.saveItems(items);
+  }
+
+  deleteItems(items: any[]): Promise<void> {
+    return this.db.deleteItems(items);
+  }
+
+  getAll(): Promise<BaseItem[]> {
+    return this.db.getAll();
+  }
+
+  private db: Database;
+
+  async init() {
+    try {
+      this.db = new IDB();
+      await this.db.init();
+    }
+    catch (ex) {
+      console.warn(
+        "Failed to initialize idb backend, using storage db fallback", ex);
+      try {
+        this.db = new StorageDB();
+        await this.db.init();
+      }
+      catch (ex) {
+        console.warn(
+          "Failed to initialize storage backend, using memory db fallback", ex);
+        this.db = new MemoryDB();
+        await this.db.init();
+      }
+    }
+  }
+}();

extensions/downthemall/lib/events.ts

@@ -0,0 +1,4 @@
+"use strict";
+// License: MIT
+
+export {EventEmitter} from "../uikit/lib/events";

extensions/downthemall/lib/filters.ts

@@ -0,0 +1,575 @@
+"use strict";
+// License: MIT
+
+import uuid from "./uuid";
+
+import "./objectoverlay";
+import { storage } from "./browser";
+import { EventEmitter } from "./events";
+import { TYPE_LINK, TYPE_MEDIA, TYPE_ALL } from "./constants";
+// eslint-disable-next-line no-unused-vars
+import { Overlayable } from "./objectoverlay";
+import DEFAULT_FILTERS from "../data/filters.json";
+import { FASTFILTER } from "./recentlist";
+import { _, locale } from "./i18n";
+// eslint-disable-next-line no-unused-vars
+import { BaseItem } from "./item";
+
+const REG_ESCAPE = /[{}()[\]\\^$.]/g;
+const REG_FNMATCH = /[*?]/;
+const REG_REG = /^\/(.+?)\/(i)?$/;
+const REG_WILD = /\*/g;
+const REG_WILD2 = /\?/g;
+
+export const FAST = Symbol();
+
+function mergeUnique(e: RegExp) {
+  if (this.has(e.source)) {
+    return false;
+  }
+  this.add(e.source);
+  return e;
+}
+
+function mergeMap(e: RegExp) {
+  if (e.unicode) {
+    this.add("u");
+  }
+  if (e.ignoreCase) {
+    this.add("i");
+  }
+  return `(?:${e.source})`;
+}
+
+
+function mergeRegexps(expressions: RegExp[]) {
+  if (!expressions.length) {
+    return null;
+  }
+  if (expressions.length < 2) {
+    return expressions[0];
+  }
+  const filtered = expressions.filter(mergeUnique, new Set());
+  const flags = new Set();
+  const mapped = filtered.map(mergeMap, flags);
+  return new RegExp(mapped.join("|"), Array.from(flags).join(""));
+}
+
+function consolidateRegexps(expressions: Iterable<RegExp>) {
+  const nc = [];
+  const ic = [];
+  for (const expr of expressions) {
+    if (expr.ignoreCase) {
+      ic.push(expr);
+    }
+    else {
+      nc.push(expr);
+    }
+  }
+  return {
+    sensitive: mergeRegexps(nc),
+    insensitive: mergeRegexps(ic)
+  };
+}
+
+function *parseIntoRegexpInternal(str: string): Iterable<RegExp> {
+  str = str.trim();
+  // Try complete regexp
+  if (str.length > 2 && str[0] === "/") {
+    try {
+      const m = str.match(REG_REG);
+      if (!m) {
+        throw new Error("Invalid RegExp supplied");
+      }
+      if (!m[1].length) {
+        return;
+      }
+      yield new RegExp(m[1], m[2]);
+      return;
+    }
+    catch (ex) {
+      // fall-through
+    }
+  }
+
+  // multi-expression
+  if (str.includes(",")) {
+    for (const part of str.split(",")) {
+      yield *parseIntoRegexpInternal(part);
+    }
+    return;
+  }
+
+  // might be an fnmatch
+  const fnmatch = REG_FNMATCH.test(str);
+  str = str.replace(REG_ESCAPE, "\\$&");
+  if (fnmatch) {
+    str = `^${str.replace(REG_WILD, ".*").replace(REG_WILD2, ".")}$`;
+  }
+  if (str.length) {
+    yield new RegExp(str, "i");
+  }
+}
+
+function parseIntoRegexp(expr: string) {
+  const expressions = Array.from(parseIntoRegexpInternal(expr));
+  if (!expressions.length) {
+    throw new Error(
+      "Invalid filtea rexpression did not yield a regular expression");
+  }
+  return expressions;
+}
+
+export class Matcher {
+  match: (str: string) => boolean;
+
+  private sensitive: RegExp;
+
+  private insensitive: RegExp;
+
+  constructor(expressions: Iterable<RegExp>) {
+    Object.assign(this, consolidateRegexps(expressions));
+    if (this.sensitive && this.insensitive) {
+      this.match = this.matchBoth;
+    }
+    else if (this.sensitive) {
+      this.match = this.matchSensitive;
+    }
+    else if (this.insensitive) {
+      this.match = this.matchInsensitive;
+    }
+    else {
+      this.match = this.matchNone;
+    }
+    Object.freeze(this);
+  }
+
+  static fromExpression(expr: string) {
+    return new Matcher(parseIntoRegexp(expr));
+  }
+
+  *[Symbol.iterator]() {
+    if (this.sensitive) {
+      yield this.sensitive;
+    }
+    if (this.insensitive) {
+      yield this.insensitive;
+    }
+  }
+
+  matchBoth(str: string) {
+    return this.sensitive.test(str) || this.insensitive.test(str);
+  }
+
+  matchSensitive(str: string) {
+    return this.sensitive.test(str);
+  }
+
+  matchInsensitive(str: string) {
+    return this.insensitive.test(str);
+  }
+
+  /* eslint-disable no-unused-vars */
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  matchNone(_: string) {
+    return false;
+  }
+  /* eslint-enable no-unused-vars */
+
+  matchItem(item: BaseItem) {
+    const {usable = "", title = "", description = "", fileName = ""} = item;
+    return this.match(usable) || this.match(title) ||
+             this.match(description) || this.match(fileName);
+  }
+}
+
+interface RawFilter extends Object {
+  active: boolean;
+  type: number;
+  label: string;
+  expr: string;
+  icon?: string;
+  custom?: boolean;
+  isOverridden?: (prop: string) => boolean;
+  reset?: () => void;
+  toJSON?: () => any;
+}
+
+export class Filter {
+  private readonly owner: Filters;
+
+  public readonly id: string | symbol;
+
+  private readonly raw: RawFilter;
+
+  private _label: string;
+
+  private _reg: Matcher;
+
+  constructor(owner: Filters, id: string | symbol, raw: RawFilter) {
+    if (!owner || !id || !raw) {
+      throw new Error("null argument");
+    }
+    this.owner = owner;
+    this.id = id;
+    this.raw = raw;
+    this.init();
+  }
+
+  init() {
+    this._label = this.raw.label;
+    if (typeof this.raw.isOverridden !== "undefined" &&
+        typeof this.id === "string") {
+      if (this.id.startsWith("deffilter-") && !this.raw.isOverridden("label")) {
+        this._label = _(this.id) || this._label;
+      }
+    }
+    this._reg = Matcher.fromExpression(this.expr);
+    Object.seal(this);
+  }
+
+  get descriptor() {
+    return {
+      active: this.active,
+      id: this.id,
+      label: this.label,
+      type: this.type,
+    };
+  }
+
+  [Symbol.iterator]() {
+    return this._reg[Symbol.iterator]();
+  }
+
+  get label() {
+    return this._label;
+  }
+
+  set label(nv) {
+    this.raw.label = this._label = nv;
+  }
+
+  get expr() {
+    return this.raw.expr;
+  }
+
+  set expr(nv) {
+    if (nv === this.raw.expr) {
+      return;
+    }
+    const reg = Matcher.fromExpression(nv);
+    this._reg = reg;
+    this.raw.expr = nv;
+  }
+
+  get active() {
+    return this.raw.active;
+  }
+
+  set active(nv) {
+    this.raw.active = !!nv;
+  }
+
+  get type() {
+    return this.raw.type;
+  }
+
+  set type(nv) {
+    if (nv !== TYPE_ALL && nv !== TYPE_LINK && nv !== TYPE_MEDIA) {
+      throw new Error("Invalid filter type");
+    }
+    this.raw.type = nv;
+  }
+
+  get icon() {
+    return this.raw.icon;
+  }
+
+  set icon(nv) {
+    this.raw.icon = nv;
+  }
+
+  async save() {
+    return await this.owner.save();
+  }
+
+  get custom() {
+    return !!this.raw.custom;
+  }
+
+  async reset() {
+    if (!this.raw.reset) {
+      throw Error("Cannot reset non-default filter");
+    }
+    this.raw.reset();
+    await this.owner.save();
+    this.init();
+  }
+
+  async "delete"() {
+    if (!this.raw.custom) {
+      throw new Error("Cannot delete default filter");
+    }
+    if (typeof this.id !== "string") {
+      throw new Error("Cannot delete symbolized");
+    }
+    await this.owner.delete(this.id);
+  }
+
+  match(str: string) {
+    return this._reg.match(str);
+  }
+
+  matchItem(item: BaseItem) {
+    return this._reg.matchItem(item);
+  }
+
+  toJSON() {
+    return this.raw.toJSON && this.raw.toJSON() || this.raw;
+  }
+}
+
+class FastFilter extends Filter {
+  constructor(owner: Filters, value: string) {
+    if (!value) {
+      throw new Error("Invalid fast filter value");
+    }
+    super(owner, FAST, {
+      label: "fast",
+      type: TYPE_ALL,
+      active: true,
+      expr: value,
+    });
+  }
+}
+
+class Collection {
+  exprs: Filter[];
+
+  constructor() {
+    this.exprs = [];
+  }
+
+  push(filter: Filter) {
+    this.exprs.push(filter);
+  }
+
+  *[Symbol.iterator]() {
+    for (const e of this.exprs) {
+      if (!e.active) {
+        continue;
+      }
+      yield *e;
+    }
+  }
+}
+
+
+class Filters extends EventEmitter {
+  private loaded: boolean;
+
+  private filters: Filter[];
+
+  ignoreNext: boolean;
+
+  private readonly typeMatchers: Map<number, Matcher>;
+
+  constructor() {
+    super();
+    this.typeMatchers = new Map();
+    this.loaded = false;
+    this.filters = [];
+    this.ignoreNext = false;
+    this.regenerate();
+    storage.onChanged.addListener(async (changes: any) => {
+      if (this.ignoreNext) {
+        this.ignoreNext = false;
+        return;
+      }
+      if (!("userFilters" in changes)) {
+        return;
+      }
+      await this.load();
+    });
+    Object.seal(this);
+  }
+
+  get all() {
+    return Array.from(this.filters);
+  }
+
+  get linkFilters() {
+    return this.filters.filter(f => f.type & TYPE_LINK);
+  }
+
+  get mediaFilters() {
+    return this.filters.filter(f => f.type & TYPE_MEDIA);
+  }
+
+  get active() {
+    return this.filters.filter(e => e.active);
+  }
+
+  [Symbol.iterator]() {
+    return this.filters[Symbol.iterator]();
+  }
+
+  async create(label: string, expr: string, type: number) {
+    const id = `custom-${uuid()}`;
+    const filter = new Filter(this, id, {
+      active: true,
+      custom: true,
+      label,
+      expr,
+      type,
+    });
+    this.filters.push(filter);
+    await this.save();
+  }
+
+  "get"(id: string | symbol) {
+    return this.filters.find(e => e.id === id);
+  }
+
+  async "delete"(id: string) {
+    const idx = this.filters.findIndex(e => e.id === id);
+    if (idx < 0) {
+      return;
+    }
+    this.filters.splice(idx, 1);
+    await this.save();
+  }
+
+  async save() {
+    if (!this.loaded) {
+      throw new Error("Filters not initialized yet");
+    }
+    const json = this.toJSON();
+    this.ignoreNext = true;
+    await storage.local.set({userFilters: json});
+    this.regenerate();
+  }
+
+  getFastFilterFor(value: string) {
+    return new FastFilter(this, value);
+  }
+
+  async getFastFilter() {
+    await FASTFILTER.init();
+    if (!FASTFILTER.current) {
+      return null;
+    }
+    return new FastFilter(this, FASTFILTER.current);
+  }
+
+  regenerate() {
+    const all = new Collection();
+    const links = new Collection();
+    const media = new Collection();
+    for (const current of this.filters) {
+      try {
+        if (current.type & TYPE_ALL) {
+          all.push(current);
+          links.push(current);
+          media.push(current);
+        }
+        else if (current.type & TYPE_LINK) {
+          links.push(current);
+        }
+        else if (current.type & TYPE_MEDIA) {
+          media.push(current);
+        }
+        else {
+          throw Error("Invalid type mask");
+        }
+      }
+      catch (ex) {
+        console.error("Filter", current.label || "unknown", ex);
+      }
+    }
+    this.typeMatchers.set(TYPE_ALL, new Matcher(all));
+    this.typeMatchers.set(TYPE_LINK, new Matcher(links));
+    this.typeMatchers.set(TYPE_MEDIA, new Matcher(media));
+    this.emit("changed");
+  }
+
+  async load() {
+    await locale;
+    const defaultFilters = DEFAULT_FILTERS as any;
+    let savedFilters = (await storage.local.get("userFilters"));
+    if (savedFilters && "userFilters" in savedFilters) {
+      savedFilters = savedFilters.userFilters;
+    }
+    else {
+      savedFilters = {};
+    }
+    const stub = Object.freeze({custom: true});
+    this.filters.length = 0;
+    const known = new Set();
+    for (const filter of Object.keys(savedFilters)) {
+      let current;
+      if (filter in defaultFilters) {
+        current = defaultFilters[filter].overlay(savedFilters[filter]);
+        known.add(filter);
+      }
+      else {
+        current = (stub as unknown as Overlayable).overlay(
+          savedFilters[filter]);
+      }
+      try {
+        this.filters.push(new Filter(this, filter, current));
+      }
+      catch (ex) {
+        console.error("Failed to load filter", filter, ex);
+      }
+    }
+    for (const filter of Object.keys(defaultFilters)) {
+      if (known.has(filter)) {
+        continue;
+      }
+      const current = ({custom: false} as unknown as Overlayable).overlay(
+        defaultFilters[filter]);
+      this.filters.push(new Filter(this, filter, current));
+    }
+    this.loaded = true;
+    this.regenerate();
+  }
+
+  async filterItemsByType(items: BaseItem[], type: number) {
+    const matcher = this.typeMatchers.get(type);
+    const fast = await this.getFastFilter();
+    return items.filter(function(item) {
+      if (fast && fast.matchItem(item)) {
+        return true;
+      }
+      return matcher && matcher.matchItem(item);
+    });
+  }
+
+  toJSON() {
+    const rv: any = {};
+    for (const filter of this.filters) {
+      if (filter.id === FAST) {
+        continue;
+      }
+      const tosave = filter.toJSON();
+      if (!tosave) {
+        continue;
+      }
+      rv[filter.id] = tosave;
+    }
+    return rv;
+  }
+}
+
+let _filters: Filters;
+let _loader: Promise<void>;
+
+export async function filters(): Promise<Filters> {
+  if (!_loader) {
+    _filters = new Filters();
+    _loader = _filters.load();
+  }
+  await _loader;
+  return _filters;
+}

extensions/downthemall/lib/formatters.ts

@@ -0,0 +1,114 @@
+"use strict";
+// License: MIT
+
+import {_} from "./i18n";
+import {memoize} from "./memoize";
+
+export function formatInteger(num: number, digits?: number) {
+  const neg = num < 0;
+  const snum = Math.abs(num).toFixed(0);
+  if (typeof digits === "undefined" || !isFinite(digits)) {
+    digits = 3;
+  }
+  if (digits <= 0) {
+    throw new Error("Invalid digit count");
+  }
+  if (snum.length >= digits) {
+    return num.toFixed(0);
+  }
+  if (neg) {
+    return `-${snum.padStart(digits, "0")}`;
+  }
+  return snum.padStart(digits, "0");
+}
+
+const HOURS_PER_DAY = 24;
+const SEC_PER_MIN = 60;
+const MIN_PER_HOUR = 60;
+const SECS_PER_HOUR = SEC_PER_MIN * MIN_PER_HOUR;
+
+export function formatTimeDelta(delta: number) {
+  let rv = delta < 0 ? "-" : "";
+  delta = Math.abs(delta);
+
+  let h = Math.floor(delta / SECS_PER_HOUR);
+  const m = Math.floor((delta % SECS_PER_HOUR) / SEC_PER_MIN);
+  const s = Math.floor(delta % SEC_PER_MIN);
+
+  if (h) {
+    if (h >= HOURS_PER_DAY) {
+      const days = Math.floor(h / HOURS_PER_DAY);
+      if (days > 9) {
+        return "∞";
+      }
+      rv += `${days}d::`;
+      h %= HOURS_PER_DAY;
+    }
+    rv += `${formatInteger(h, 2)}:`;
+  }
+  return `${rv + formatInteger(m, 2)}:${formatInteger(s, 2)}`;
+}
+
+export function makeNumberFormatter(fracDigits: number) {
+  const rv = new Intl.NumberFormat(undefined, {
+    style: "decimal",
+    useGrouping: false,
+    minimumFractionDigits: fracDigits,
+    maximumFractionDigits: fracDigits
+  });
+  return rv.format.bind(rv);
+}
+
+
+const fmt0 = makeNumberFormatter(0);
+const fmt1 = makeNumberFormatter(1);
+const fmt2 = makeNumberFormatter(2);
+const fmt3 = makeNumberFormatter(3);
+
+const SIZE_UNITS = [
+  ["sizeB", fmt0],
+  ["sizeKB", fmt1],
+  ["sizeMB", fmt2],
+  ["sizeGB", fmt2],
+  ["sizeTB", fmt3],
+  ["sizePB", fmt3],
+];
+const SIZE_NUINITS = SIZE_UNITS.length;
+const SIZE_SCALE = 875;
+const SIZE_KILO = 1024;
+
+export const formatSize = memoize(function formatSize(
+    size: number, fractions = true) {
+  const neg = size < 0;
+  size = Math.abs(size);
+  let i = 0;
+  while (size > SIZE_SCALE && ++i < SIZE_NUINITS) {
+    size /= SIZE_KILO;
+  }
+  if (neg) {
+    size = -size;
+  }
+  const [unit, fmt] = SIZE_UNITS[i];
+  return _(unit, fractions ? fmt(size) : fmt0(size));
+}, 1000, 2);
+
+const SPEED_UNITS = [
+  ["speedB", fmt0],
+  ["speedKB", fmt2],
+  ["speedMB", fmt2],
+];
+const SPEED_NUNITS = SIZE_UNITS.length;
+
+export const formatSpeed = memoize(function formatSpeed(size: number) {
+  const neg = size < 0;
+  size = Math.abs(size);
+  let i = 0;
+  while (size > SIZE_KILO && ++i < SPEED_NUNITS) {
+    size /= SIZE_KILO;
+  }
+  if (neg) {
+    size = -size;
+  }
+  const [unit, fmt] = SPEED_UNITS[i];
+  return _(unit, fmt(size));
+});

extensions/downthemall/lib/i18n.ts

@@ -0,0 +1,315 @@
+"use strict";
+// License: MIT
+
+import {memoize} from "./memoize";
+import langs from "../_locales/all.json";
+import { sorted, naturalCaseCompare } from "./sorting";
+import lf from "localforage";
+
+export const ALL_LANGS = Object.freeze(new Map<string, string>(
+  sorted(Object.entries(langs), e => {
+    return [e[1], e[0]];
+  }, naturalCaseCompare)));
+
+let CURRENT = "en";
+export function getCurrentLanguage() {
+  return CURRENT;
+}
+
+declare let browser: any;
+declare let chrome: any;
+
+const CACHE_KEY = "_cached_locales";
+const CUSTOM_KEY = "_custom_locale";
+
+const normalizer = /[^A-Za-z0-9_]/g;
+
+interface JSONEntry {
+  message: string;
+  placeholders: any;
+}
+
+class Entry {
+  private message: string;
+
+  constructor(entry: JSONEntry) {
+    if (!entry.message.includes("$")) {
+      throw new Error("Not entry-able");
+    }
+    let hit = false;
+    this.message = entry.message.replace(/\$[A-Z0-9]+\$/g, (r: string) => {
+      hit = true;
+      const id = r.substr(1, r.length - 2).toLocaleLowerCase();
+      const placeholder = entry.placeholders[id];
+      if (!placeholder || !placeholder.content) {
+        throw new Error(`Invalid placeholder: ${id}`);
+      }
+      return `${placeholder.content}$`;
+    });
+    if (!hit) {
+      throw new Error("Not entry-able");
+    }
+  }
+
+  localize(args: any[]) {
+    return this.message.replace(/\$\d+\$/g, (r: string) => {
+      const idx = parseInt(r.substr(1, r.length - 2), 10) - 1;
+      return args[idx] || "";
+    });
+  }
+}
+
+class Localization {
+  private strings: Map<string, Entry | string>;
+
+  constructor(baseLanguage: any, ...overlayLanguages: any) {
+    this.strings = new Map();
+    const mapLanguage = (lang: any) => {
+      for (const [id, entry] of Object.entries<JSONEntry>(lang)) {
+        if (!entry.message) {
+          continue;
+        }
+        try {
+          if (entry.message.includes("$")) {
+            this.strings.set(id, new Entry(entry));
+          }
+          else {
+            this.strings.set(id, entry.message);
+          }
+        }
+        catch (ex) {
+          this.strings.set(id, entry.message);
+        }
+      }
+    };
+    mapLanguage(baseLanguage);
+    overlayLanguages.forEach(mapLanguage);
+  }
+
+  localize(id: string, ...args: any[]) {
+    const entry = this.strings.get(id.replace(normalizer, "_"));
+    if (!entry) {
+      return "";
+    }
+    if (typeof entry === "string") {
+      return entry;
+    }
+    if (args.length === 1 && Array.isArray(args)) {
+      [args] = args;
+    }
+    return entry.localize(args);
+  }
+}
+
+function checkBrowser() {
+  // eslint-disable-next-line @typescript-eslint/no-var-requires
+  if (typeof browser !== "undefined" && browser.i18n) {
+    return;
+  }
+  if (typeof chrome !== "undefined" && chrome.i18n) {
+    return;
+  }
+  throw new Error("not in a webext");
+}
+
+async function fetchLanguage(code: string) {
+  try {
+    const resp = await fetch(`/_locales/${code}/messages.json`);
+    return await resp.json();
+  }
+  catch {
+    return null;
+  }
+}
+
+
+async function loadCached(): Promise<any> {
+  const cached = await lf.getItem<string>(CACHE_KEY);
+  if (!cached) {
+    return null;
+  }
+  const parsed = JSON.parse(cached);
+  if (!Array.isArray(parsed) || !parsed[0].CRASH || !parsed[0].CRASH.message) {
+    console.warn("rejecting cached locales", parsed);
+    return null;
+  }
+  return parsed;
+}
+
+async function loadRawLocales() {
+  // en is the base locale, always to be loaded
+  // The loader will override string from it with more specific string
+  // from other locales
+  const langs = new Set<string>(["en"]);
+
+  const uiLang: string = (typeof browser !== "undefined" ? browser : chrome).
+    i18n.getUILanguage();
+
+  // Chrome will only look for underscore versions of locale codes,
+  // while Firefox will look for both.
+  // So we better normalize the code to the underscore version.
+  // However, the API seems to always return the dash-version.
+
+  // Add all base locales into ascending order of priority,
+  // starting with the most unspecific base locale, ending
+  // with the most specific locale.
+  // e.g. this will transform ["zh", "CN"] -> ["zh", "zh_CN"]
+  uiLang.split(/[_-]/g).reduce<string[]>((prev, curr) => {
+    prev.push(curr);
+    langs.add(prev.join("_"));
+    return prev;
+  }, []);
+
+  if (CURRENT && CURRENT !== "default") {
+    langs.delete(CURRENT);
+    langs.add(CURRENT);
+  }
+
+  const valid = Array.from(langs).filter(e => ALL_LANGS.has(e));
+  const fetched = await Promise.all(Array.from(valid, fetchLanguage));
+  return fetched.filter(e => !!e);
+}
+
+async function load(): Promise<Localization> {
+  try {
+    checkBrowser();
+    try {
+      let currentLang: any = "";
+      if (typeof browser !== "undefined") {
+        currentLang = await browser.storage.sync.get("language");
+      }
+      else {
+        currentLang = await new Promise(
+          resolve => chrome.storage.sync.get("language", resolve));
+      }
+      if ("language" in currentLang) {
+        currentLang = currentLang.language;
+      }
+      if (!currentLang || !currentLang.length) {
+        currentLang = "default";
+      }
+      CURRENT = currentLang;
+      // en is the base locale
+      let valid = await loadCached();
+      if (!valid) {
+        valid = await loadRawLocales();
+        await lf.setItem(CACHE_KEY, JSON.stringify(valid));
+      }
+      if (!valid.length) {
+        throw new Error("Could not load ANY of these locales");
+      }
+
+      const custom = await lf.getItem<string>(CUSTOM_KEY);
+      if (custom) {
+        try {
+          valid.push(JSON.parse(custom));
+        }
+        catch (ex) {
+          console.error(ex);
+          // ignored
+        }
+      }
+
+      const base = valid.shift();
+      const rv = new Localization(base, ...valid);
+      return rv;
+    }
+    catch (ex) {
+      console.error("Failed to load locale", ex.toString(), ex.stack, ex);
+      return new Localization({});
+    }
+  }
+  catch {
+    // We might be running under node for tests
+
+    // eslint-disable-next-line @typescript-eslint/no-var-requires
+    const messages = require("../_locales/en/messages.json");
+
+    return new Localization(messages);
+  }
+}
+
+type MemoLocalize = (id: string, ...args: any[]) => string;
+
+export const locale = load();
+let loc: Localization | null;
+let memoLocalize: MemoLocalize | null = null;
+locale.then(l => {
+  loc = l;
+  memoLocalize = memoize(loc.localize.bind(loc), 10 * 1000, 10);
+});
+
+/**
+ * Localize a message
+ * @param {string} id Identifier of the string to localize
+ * @param {string[]} [subst] Message substitutions
+ * @returns {string} Localized message
+ */
+export function _(id: string, ...subst: any[]) {
+  if (!loc || !memoLocalize) {
+    console.trace("TOO SOON");
+    throw new Error("Called too soon");
+  }
+  if (!subst.length) {
+    return memoLocalize(id);
+  }
+  return loc.localize(id, subst);
+}
+
+function localize_<T extends HTMLElement | DocumentFragment>(elem: T): T {
+  for (const tmpl of elem.querySelectorAll<HTMLTemplateElement>("template")) {
+    localize_(tmpl.content);
+  }
+
+  for (const el of elem.querySelectorAll<HTMLElement>("*[data-i18n]")) {
+    const {i18n: i} = el.dataset;
+    if (!i) {
+      continue;
+    }
+    for (let piece of i.split(",")) {
+      piece = piece.trim();
+      if (!piece) {
+        continue;
+      }
+      const idx = piece.indexOf("=");
+      if (idx < 0) {
+        let childElements;
+        if (el.childElementCount) {
+          childElements = Array.from(el.children);
+        }
+        el.textContent = _(piece);
+        if (childElements) {
+          childElements.forEach(e => el.appendChild(e));
+        }
+        continue;
+      }
+      const attr = piece.substr(0, idx).trim();
+      piece = piece.substr(idx + 1).trim();
+      el.setAttribute(attr, _(piece));
+    }
+  }
+  for (const el of document.querySelectorAll("*[data-l18n]")) {
+    console.error("wrong!", el);
+  }
+  return elem as T;
+}
+
+/**
+ * Localize a DOM
+ * @param {Element} elem DOM to localize
+ * @returns {Element} Passed in element (fluent)
+ */
+export async function localize<T extends HTMLElement | DocumentFragment>(
+    elem: T): Promise<T> {
+  await locale;
+  return localize_(elem);
+}
+
+export async function saveCustomLocale(data?: string) {
+  if (!data) {
+    await lf.removeItem(CUSTOM_KEY);
+    return;
+  }
+  new Localization(JSON.parse(data));
+  await localStorage.setItem(CUSTOM_KEY, data);
+}

extensions/downthemall/lib/iconcache.ts

@@ -0,0 +1,124 @@
+"use strict";
+// License: MIT
+
+import { downloads, CHROME } from "./browser";
+import { EventEmitter } from "../uikit/lib/events";
+import { PromiseSerializer } from "./pserializer";
+import lf from "localforage";
+
+
+const STORE = "iconcache";
+
+// eslint-disable-next-line no-magic-numbers
+const CACHE_SIZES = CHROME ? [16, 32] : [16, 32, 64, 127];
+
+const BLACKLISTED = Object.freeze(new Set([
+  "",
+  "ext",
+  "ico",
+  "pif",
+  "scr",
+  "ani",
+  "cur",
+  "ttf",
+  "otf",
+  "woff",
+  "woff2",
+  "cpl",
+  "desktop",
+  "app",
+]));
+
+async function getIcon(size: number, manId: number) {
+  const raw = await downloads.getFileIcon(manId, {size});
+  const icon = new URL(raw);
+  if (icon.protocol === "data:") {
+    const res = await fetch(icon.toString());
+    const blob = await res.blob();
+    return {size, icon: blob};
+  }
+  return {size, icon};
+}
+
+const SYNONYMS = Object.freeze(new Map<string, string>([
+  ["jpe", "jpg"],
+  ["jpeg", "jpg"],
+  ["jfif", "jpg"],
+  ["mpe", "mpg"],
+  ["mpeg", "mpg"],
+  ["m4v", "mp4"],
+]));
+
+export const IconCache = new class IconCache extends EventEmitter {
+  private db = lf.createInstance({name: STORE});
+
+  private cache: Map<string, string>;
+
+  constructor() {
+    super();
+    this.cache = new Map();
+    this.get = PromiseSerializer.wrapNew(8, this, this.get);
+    this.set = PromiseSerializer.wrapNew(1, this, this.set);
+  }
+
+  private normalize(ext: string) {
+    ext = ext.toLocaleLowerCase();
+    return SYNONYMS.get(ext) || ext;
+  }
+
+  // eslint-disable-next-line no-magic-numbers
+  async get(ext: string, size = 16) {
+    ext = this.normalize(ext);
+    if (BLACKLISTED.has(ext)) {
+      return undefined;
+    }
+    const sext = `${ext}-${size}`;
+    let rv = this.cache.get(sext);
+    if (rv) {
+      return rv;
+    }
+    rv = this.cache.get(sext);
+    if (rv) {
+      return rv;
+    }
+    let result = await this.db.getItem<any>(sext);
+    if (!result) {
+      return this.cache.get(sext);
+    }
+    rv = this.cache.get(sext);
+    if (rv) {
+      return rv;
+    }
+    if (typeof result !== "string") {
+      result = URL.createObjectURL(result).toString();
+    }
+
+    this.cache.set(sext, result);
+    this.cache.set(ext, "");
+    return result;
+  }
+
+  async set(ext: string, manId: number) {
+    ext = this.normalize(ext);
+    if (BLACKLISTED.has(ext)) {
+      return;
+    }
+    if (this.cache.has(ext)) {
+      // already processed in this session
+      return;
+    }
+    // eslint-disable-next-line no-magic-numbers
+    const urls = await Promise.all(CACHE_SIZES.map(
+      size => getIcon(size, manId)));
+    if (this.cache.has(ext)) {
+      // already processed in this session
+      return;
+    }
+    for (const {size, icon} of urls) {
+      this.cache.set(`${ext}-${size}`, URL.createObjectURL(icon));
+      await this.db.setItem(`${ext}-${size}`, icon);
+    }
+    this.cache.set(ext, "");
+    this.emit("cached", ext);
+  }
+}();

extensions/downthemall/lib/imex.ts

@@ -0,0 +1,261 @@
+"use strict";
+// License: MIT
+
+import { getTextLinks } from "./textlinks";
+// eslint-disable-next-line no-unused-vars
+import { BaseItem } from "./item";
+import { ALLOWED_SCHEMES } from "./constants";
+
+export const NS_METALINK_RFC5854 = "urn:ietf:params:xml:ns:metalink";
+export const NS_DTA = "http://www.downthemall.net/properties#";
+
+function parseNum(
+    file: Element,
+    attr: string,
+    defaultValue: number,
+    ns = NS_METALINK_RFC5854) {
+  const val = file.getAttributeNS(ns, attr);
+  if (!val) {
+    return defaultValue + 1;
+  }
+  const num = parseInt(val, 10);
+  if (isFinite(num)) {
+    return num;
+  }
+  return defaultValue + 1;
+}
+
+function importMeta4(data: string) {
+  const parser = new DOMParser();
+  const document = parser.parseFromString(data, "text/xml");
+  const {documentElement} = document;
+  const items: BaseItem[] = [];
+  let batch = 0;
+  for (const file of documentElement.querySelectorAll("file")) {
+    try {
+      const url = Array.from(file.querySelectorAll("url")).map(u => {
+        try {
+          const {textContent} = u;
+          if (!textContent) {
+            return null;
+          }
+          const url = new URL(textContent);
+          if (!ALLOWED_SCHEMES.has(url.protocol)) {
+            return null;
+          }
+          const prio = parseNum(u, "priority", 0);
+          return {
+            url,
+            prio
+          };
+        }
+        catch {
+          return null;
+        }
+      }).filter(u => !!u).reduce((p, c) => {
+        if (!c) {
+          return null;
+        }
+        if (!p || p.prio < c.prio) {
+          return c;
+        }
+        return p;
+      });
+      if (!url) {
+        continue;
+      }
+      batch = parseNum(file, "num", batch, NS_DTA);
+      const idx = parseNum(file, "idx", 0, NS_DTA);
+      const item: BaseItem = {
+        url: url.url.toString(),
+        usable: decodeURIComponent(url.url.toString()),
+        batch,
+        idx
+      };
+      const ref = file.getAttributeNS(NS_DTA, "referrer");
+      if (ref) {
+        item.referrer = ref;
+        item.usableReferrer = decodeURIComponent(ref);
+      }
+      const mask = file.getAttributeNS(NS_DTA, "mask");
+      if (mask) {
+        item.mask = mask;
+      }
+      const description = file.querySelector("description");
+      if (description && description.textContent) {
+        item.description = description.textContent.trim();
+      }
+      const title = file.getElementsByTagNameNS(NS_DTA, "title");
+      if (title && title[0] && title[0].textContent) {
+        item.title = title[0].textContent;
+      }
+      items.push(item);
+    }
+    catch (ex) {
+      console.error("Failed to import file", ex);
+    }
+  }
+  return items;
+}
+
+function parseKV(current: BaseItem, line: string) {
+  const [k, v] = line.split("=", 2);
+  switch (k.toLocaleLowerCase().trim()) {
+  case "referer": {
+    const refererUrls = getTextLinks(v);
+    if (refererUrls && refererUrls.length) {
+      current.referrer = refererUrls.pop();
+      current.usableReferrer = decodeURIComponent(current.referrer || "");
+    }
+    break;
+  }
+  }
+}
+
+export function importText(data: string) {
+  if (data.includes(NS_METALINK_RFC5854)) {
+    return importMeta4(data);
+  }
+  const splitter = /((?:.|\r)+)\n|(.+)$/g;
+  const spacer = /^\s+/;
+  let match;
+  let current: BaseItem | undefined = undefined;
+  let idx = 0;
+  const items = [];
+  while ((match = splitter.exec(data)) !== null) {
+    try {
+      const line = match[0].trimRight();
+      if (!line) {
+        continue;
+      }
+      if (spacer.test(line)) {
+        if (!current) {
+          continue;
+        }
+        parseKV(current, line);
+        continue;
+      }
+      const urls = getTextLinks(line);
+      if (!urls || !urls.length) {
+        continue;
+      }
+      current = {
+        url: urls[0],
+        usable: decodeURIComponent(urls[0]),
+        idx: ++idx
+      };
+      items.push(current);
+    }
+    catch (ex) {
+      current = undefined;
+      console.error("Failed to import", ex);
+    }
+  }
+  return items;
+}
+
+export interface Exporter {
+  fileName: string;
+  getText(items: BaseItem[]): string;
+}
+
+class TextExporter {
+  readonly fileName: string;
+
+  constructor() {
+    this.fileName = "links.txt";
+  }
+
+  getText(items: BaseItem[]) {
+    const lines = [];
+    for (const item of items) {
+      lines.push(item.url);
+    }
+    return lines.join("\n");
+  }
+}
+
+class Aria2Exporter {
+  readonly fileName: string;
+
+  constructor() {
+    this.fileName = "links.aria2.txt";
+  }
+
+  getText(items: BaseItem[]) {
+    const lines = [];
+    for (const item of items) {
+      lines.push(item.url);
+      if (item.referrer) {
+        lines.push(`  referer=${item.referrer}`);
+      }
+    }
+    return lines.join("\n");
+  }
+}
+
+class MetalinkExporter {
+  readonly fileName: string;
+
+  constructor() {
+    this.fileName = "links.meta4";
+  }
+
+  getText(items: BaseItem[]) {
+    const document = window.document.implementation.
+      createDocument(NS_METALINK_RFC5854, "metalink", null);
+    const root = document.documentElement;
+    root.setAttributeNS(NS_DTA, "generator", "DownThemAll!");
+    root.appendChild(document.createComment(
+      "metalink as exported by DownThemAll!",
+    ));
+
+    for (const item of items) {
+      const anyItem = item as any;
+      const f = document.createElementNS(NS_METALINK_RFC5854, "file");
+      f.setAttribute("name", anyItem.currentName);
+      if (item.batch) {
+        f.setAttributeNS(NS_DTA, "num", item.batch.toString());
+      }
+      if (item.idx) {
+        f.setAttributeNS(NS_DTA, "idx", item.idx.toString());
+      }
+      if (item.referrer) {
+        f.setAttributeNS(NS_DTA, "referrer", item.referrer);
+      }
+      if (item.mask) {
+        f.setAttributeNS(NS_DTA, "mask", item.mask);
+      }
+
+      if (item.description) {
+        const n = document.createElementNS(NS_METALINK_RFC5854, "description");
+        n.textContent = item.description;
+        f.appendChild(n);
+      }
+
+      if (item.title) {
+        const n = document.createElementNS(NS_DTA, "title");
+        n.textContent = item.title;
+        f.appendChild(n);
+      }
+
+      const u = document.createElementNS(NS_METALINK_RFC5854, "url");
+      u.textContent = item.url;
+      f.appendChild(u);
+
+      if (anyItem.totalSize > 0) {
+        const s = document.createElementNS(NS_METALINK_RFC5854, "size");
+        s.textContent = anyItem.totalSize.toString();
+        f.appendChild(s);
+      }
+      root.appendChild(f);
+    }
+    let xml = "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n";
+    xml += root.outerHTML;
+    return xml;
+  }
+}
+
+export const textExporter = new TextExporter();
+export const aria2Exporter = new Aria2Exporter();
+export const metalinkExporter = new MetalinkExporter();

extensions/downthemall/lib/ipreg.ts

@@ -0,0 +1,4 @@
+"use strict";
+// License: MIT
+
+export const IPReg = /^(?:(?:[0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])[.]){3}(?:[0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])$|^(?:(?:(?:[0-9a-fA-F]{1,4}):){7}(?:(?:[0-9a-fA-F]{1,4})|:)|(?:(?:[0-9a-fA-F]{1,4}):){6}(?:((?:[0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])[.]){3}(?:[0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])|:(?:[0-9a-fA-F]{1,4})|:)|(?:(?:[0-9a-fA-F]{1,4}):){5}(?::((?:[0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])[.]){3}(?:[0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])|(:(?:[0-9a-fA-F]{1,4})){1,2}|:)|(?:(?:[0-9a-fA-F]{1,4}):){4}(?:(:(?:[0-9a-fA-F]{1,4})){0,1}:((?:[0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])[.]){3}(?:[0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])|(:(?:[0-9a-fA-F]{1,4})){1,3}|:)|(?:(?:[0-9a-fA-F]{1,4}):){3}(?:(:(?:[0-9a-fA-F]{1,4})){0,2}:((?:[0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])[.]){3}(?:[0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])|(:(?:[0-9a-fA-F]{1,4})){1,4}|:)|(?:(?:[0-9a-fA-F]{1,4}):){2}(?:(:(?:[0-9a-fA-F]{1,4})){0,3}:((?:[0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])[.]){3}(?:[0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])|(:(?:[0-9a-fA-F]{1,4})){1,5}|:)|(?:(?:[0-9a-fA-F]{1,4}):){1}(?:(:(?:[0-9a-fA-F]{1,4})){0,4}:((?:[0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])[.]){3}(?:[0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])|(:(?:[0-9a-fA-F]{1,4})){1,6}|:)|(?::((?::(?:[0-9a-fA-F]{1,4})){0,5}:((?:[0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])[.]){3}(?:[0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])|(?::(?:[0-9a-fA-F]{1,4})){1,7}|:)))(%[0-9a-zA-Z]{1,})?$/;

extensions/downthemall/lib/item.ts

@@ -0,0 +1,139 @@
+"use strict";
+// License: MIT
+
+import { ALLOWED_SCHEMES } from "./constants";
+import { TRANSFERABLE_PROPERTIES } from "./constants";
+
+export interface BaseItem {
+  url: string;
+  usable: string;
+  referrer?: string;
+  usableReferrer?: string;
+  description?: string;
+  title?: string;
+  fileName?: string;
+  batch?: number;
+  idx: number;
+  mask?: string;
+  subfolder?: string;
+  startDate?: number;
+  private?: boolean;
+  postData?: string;
+  paused?: boolean;
+}
+
+const OPTIONPROPS = Object.freeze([
+  "referrer", "usableReferrer",
+  "description", "title",
+  "fileName",
+  "batch", "idx",
+  "mask",
+  "subfolder",
+  "startDate",
+  "private",
+  "postData",
+  "paused"
+]);
+
+function maybeAssign(options: any, what: any) {
+  const type = typeof this[what];
+  if (type === "number" || type === "string" || type === "boolean") {
+    return;
+  }
+  if (type === "object" && this[what]) {
+    return;
+  }
+  let val;
+  if (what in options) {
+    val = options[what];
+  }
+  this[what] = val;
+}
+
+export class Item implements BaseItem {
+  public url: string;
+
+  public usable: string;
+
+  public referrer: string;
+
+  public usableReferrer: string;
+
+  public idx: number;
+
+  constructor(raw: any, options?: any) {
+    Object.assign(this, raw);
+    OPTIONPROPS.forEach(maybeAssign.bind(this, options || {}));
+
+    this.usable = Item.makeUsable(this.url, this.usable);
+    this.usableReferrer = Item.makeUsable(this.referrer, this.usableReferrer);
+  }
+
+  static makeUsable(unusable: string, usable: string | boolean) {
+    if (usable === true) {
+      return unusable;
+    }
+    if (usable) {
+      return usable;
+    }
+    try {
+      return decodeURIComponent(unusable);
+    }
+    catch (ex) {
+      return unusable;
+    }
+  }
+
+  toString() {
+    return `<Item(${this.url})>`;
+  }
+}
+
+export class Finisher {
+  public referrer: string;
+
+  public usableReferrer: string;
+
+  constructor(options: any) {
+    this.referrer = options.baseURL;
+    this.usableReferrer = Item.makeUsable(
+      options.baseURL, options.usable || null);
+  }
+
+  finish(item: any) {
+    if (!ALLOWED_SCHEMES.has(new URL(item.url).protocol)) {
+      return null;
+    }
+    return new Item(item, this);
+  }
+}
+
+function transfer(e: any, other: any) {
+  for (const p of TRANSFERABLE_PROPERTIES) {
+    if (!other[p] && e[p]) {
+      other[p] = e[p];
+    }
+  }
+}
+
+
+export function makeUniqueItems(items: any[][], mapping?: Function) {
+  const known = new Map();
+  const unique = [];
+  for (const itemlist of items) {
+    for (const e of itemlist) {
+      const other = known.get(e.url);
+      if (other) {
+        transfer(e, other);
+        continue;
+      }
+      const finished = mapping ? mapping(e) : e;
+      if (!finished) {
+        continue;
+      }
+      known.set(finished.url, finished);
+      unique.push(finished);
+    }
+  }
+  return unique;
+}

extensions/downthemall/lib/manager/basedownload.ts

@@ -0,0 +1,200 @@
+"use strict";
+// License: MIT
+
+// eslint-disable-next-line no-unused-vars
+import { parsePath, URLd } from "../util";
+import { QUEUED, RUNNING, PAUSED } from "./state";
+import Renamer from "./renamer";
+// eslint-disable-next-line no-unused-vars
+import { BaseItem } from "../item";
+
+const SAVEDPROPS = [
+  "state",
+  "url",
+  "usable",
+  "referrer",
+  "usableReferrer",
+  "fileName",
+  "mask",
+  "subfolder",
+  "date",
+  // batches
+  "batch",
+  "idx",
+  // meta data
+  "description",
+  "title",
+  "postData",
+  // progress
+  "totalSize",
+  "written",
+  // server stuff
+  "serverName",
+  "browserName",
+  "mime",
+  "prerolled",
+  // other options
+  "private",
+  // db
+  "manId",
+  "dbId",
+  "position",
+];
+
+const DEFAULTS = {
+  state: QUEUED,
+  error: "",
+  serverName: "",
+  browserName: "",
+  fileName: "",
+  totalSize: 0,
+  written: 0,
+  manId: 0,
+  mime: "",
+  prerolled: false,
+  retries: 0,
+  deadline: 0
+};
+
+let sessionId = 0;
+
+
+export class BaseDownload {
+  public state: number;
+
+  public sessionId: number;
+
+  public renamer: Renamer;
+
+  public uURL: URLd;
+
+  public url: string;
+
+  public usable: string;
+
+  public uReferrer: URLd;
+
+  public referrer: string;
+
+  public usableReferrer: string;
+
+  public startDate: Date;
+
+  public fileName: string;
+
+  public description?: string;
+
+  public title?: string;
+
+  public batch: number;
+
+  public idx: number;
+
+  public error: string;
+
+  public postData: any;
+
+  public private: boolean;
+
+  public written: number;
+
+  public totalSize: number;
+
+  public serverName: string;
+
+  public browserName: string;
+
+  public mime: string;
+
+  public mask: string;
+
+  public subfolder: string;
+
+  public prerolled: boolean;
+
+  public retries: number;
+
+  constructor(options: BaseItem) {
+    Object.assign(this, DEFAULTS);
+    this.assign(options);
+    if (this.state === RUNNING) {
+      this.state = QUEUED;
+    }
+    this.sessionId = ++sessionId;
+    this.renamer = new Renamer(this);
+    this.retries = 0;
+  }
+
+  assign(options: BaseItem) {
+    // eslint-disable-next-line @typescript-eslint/no-this-alias
+    const self: any = this;
+    const other: any = options;
+    for (const prop of SAVEDPROPS) {
+      if (prop in options) {
+        self[prop] = other[prop];
+      }
+    }
+    this.uURL = new URL(this.url) as URLd;
+    this.uReferrer = (this.referrer && new URL(this.referrer)) as URLd;
+    this.startDate = new Date(options.startDate || Date.now());
+    if (options.paused) {
+      this.state = PAUSED;
+    }
+    if (!this.startDate) {
+      this.startDate = new Date(Date.now());
+    }
+  }
+
+  get finalName() {
+    return this.serverName || this.fileName || this.urlName || "index.html";
+  }
+
+  get currentName() {
+    return this.browserName || this.dest.name || this.finalName;
+  }
+
+  get urlName() {
+    const path = parsePath(this.uURL);
+    if (path.name) {
+      return path.name;
+    }
+    return parsePath(path.path).name;
+  }
+
+  get dest() {
+    return parsePath(this.renamer.toString());
+  }
+
+  toString() {
+    return `Download(${this.url})`;
+  }
+
+  toJSON() {
+    // eslint-disable-next-line @typescript-eslint/no-this-alias
+    const self: any = this;
+    const rv: any = {};
+    for (const prop of SAVEDPROPS) {
+      if (prop in self) {
+        rv[prop] = self[prop];
+      }
+    }
+    rv.startDate = +self.startDate;
+    return rv;
+  }
+
+  toMsg() {
+    const rv = this.toJSON();
+    rv.sessionId = this.sessionId;
+    rv.finalName = this.finalName;
+    const {dest} = this;
+    rv.destName = dest.name;
+    rv.destPath = dest.path;
+    rv.destFull = dest.full;
+    rv.currentName = this.browserName || rv.destName || rv.finalName;
+    rv.currentFull = `${dest.path}/${rv.currentName}`;
+    rv.error = this.error;
+    rv.ext = this.renamer.p_ext;
+    rv.retries = this.retries;
+    return rv;
+  }
+}

extensions/downthemall/lib/manager/download.ts

@@ -0,0 +1,424 @@
+"use strict";
+// License: MIT
+
+// eslint-disable-next-line no-unused-vars
+import { CHROME, downloads, DownloadOptions } from "../browser";
+import { Prefs, PrefWatcher } from "../prefs";
+import { PromiseSerializer } from "../pserializer";
+import { filterInSitu, parsePath } from "../util";
+import { BaseDownload } from "./basedownload";
+// eslint-disable-next-line no-unused-vars
+import { Manager } from "./man";
+import Renamer from "./renamer";
+import {
+  CANCELABLE,
+  CANCELED,
+  DONE,
+  FORCABLE,
+  MISSING,
+  PAUSEABLE,
+  PAUSED,
+  QUEUED,
+  RUNNING,
+  RETRYING
+} from "./state";
+// eslint-disable-next-line no-unused-vars
+import { Preroller, PrerollResults } from "./preroller";
+
+function isRecoverable(error: string) {
+  switch (error) {
+  case "SERVER_FAILED":
+    return true;
+
+  default:
+    return error.startsWith("NETWORK_");
+  }
+}
+
+const RETRIES = new PrefWatcher("retries", 5);
+const RETRY_TIME = new PrefWatcher("retry-time", 5);
+
+export class Download extends BaseDownload {
+  public manager: Manager;
+
+  public manId: number;
+
+  public removed: boolean;
+
+  public position: number;
+
+  public error: string;
+
+  public dbId: number;
+
+  public deadline: number;
+
+  constructor(manager: Manager, options: any) {
+    super(options);
+    this.manager = manager;
+    this.start = PromiseSerializer.wrapNew(1, this, this.start);
+    this.removed = false;
+    this.position = -1;
+  }
+
+  markDirty() {
+    this.renamer = new Renamer(this);
+    this.manager.setDirty(this);
+  }
+
+  changeState(newState: number) {
+    const oldState = this.state;
+    if (oldState === newState) {
+      return;
+    }
+    this.state = newState;
+    this.error = "";
+    this.manager.changedState(this, oldState, this.state);
+    this.markDirty();
+  }
+
+  async start() {
+    if (this.state !== QUEUED) {
+      throw new Error("invalid state");
+    }
+    if (this.manId) {
+      const {manId: id} = this;
+      try {
+        const state = (await downloads.search({id})).pop() || {};
+        if (state.state === "in_progress" && !state.error && !state.paused) {
+          this.changeState(RUNNING);
+          this.updateStateFromBrowser();
+          return;
+        }
+        if (state.state === "complete") {
+          this.changeState(DONE);
+          this.updateStateFromBrowser();
+          return;
+        }
+        if (!state.canResume) {
+          throw new Error("Cannot resume");
+        }
+        // Cannot await here
+        // Firefox bug: will not return until download is finished
+        downloads.resume(id).catch(console.error);
+        this.changeState(RUNNING);
+        return;
+      }
+      catch (ex) {
+        console.error("cannot resume", ex);
+        this.manager.removeManId(this.manId);
+        this.removeFromBrowser();
+      }
+    }
+    if (this.state !== QUEUED) {
+      throw new Error("invalid state");
+    }
+    console.log("starting", this.toString(), this.toMsg());
+    this.changeState(RUNNING);
+
+    // Do NOT await
+    this.reallyStart();
+  }
+
+  private async reallyStart() {
+    try {
+      if (!this.prerolled) {
+        await this.maybePreroll();
+        if (this.state !== RUNNING) {
+          // Aborted by preroll
+          return;
+        }
+      }
+      const options: DownloadOptions = {
+        conflictAction: await Prefs.get("conflict-action"),
+        saveAs: false,
+        url: this.url,
+        headers: [],
+      };
+      if (!CHROME) {
+        options.filename = this.dest.full;
+      }
+      if (!CHROME && this.private) {
+        options.incognito = true;
+      }
+      if (this.postData) {
+        options.body = this.postData;
+        options.method = "POST";
+      }
+      if (!CHROME && this.referrer) {
+        options.headers.push({
+          name: "Referer",
+          value: this.referrer
+        });
+      }
+      else if (CHROME) {
+        options.headers.push({
+          name: "X-DTA-ID",
+          value: this.sessionId.toString(),
+        });
+      }
+      if (this.manId) {
+        this.manager.removeManId(this.manId);
+      }
+
+      try {
+        this.manager.addManId(
+          this.manId = await downloads.download(options), this);
+      }
+      catch (ex) {
+        if (!this.referrer) {
+          throw ex;
+        }
+        // Re-attempt without referrer
+        filterInSitu(options.headers, h => h.name !== "Referer");
+        this.manager.addManId(
+          this.manId = await downloads.download(options), this);
+      }
+      this.markDirty();
+    }
+    catch (ex) {
+      console.error("failed to start download", ex.toString(), ex);
+      this.changeState(CANCELED);
+      this.error = ex.toString();
+    }
+  }
+
+  private async maybePreroll() {
+    try {
+      if (this.prerolled) {
+        // Check again, just in case, async and all
+        return;
+      }
+      const roller = new Preroller(this);
+      if (!roller.shouldPreroll) {
+        return;
+      }
+      const res = await roller.roll();
+      if (!res) {
+        return;
+      }
+      this.adoptPrerollResults(res);
+    }
+    catch (ex) {
+      console.error("Failed to preroll", this, ex.toString(), ex.stack, ex);
+    }
+    finally {
+      if (this.state === RUNNING) {
+        this.prerolled = true;
+        this.markDirty();
+      }
+    }
+  }
+
+  adoptPrerollResults(res: PrerollResults) {
+    if (res.mime) {
+      this.mime = res.mime;
+    }
+    if (res.name) {
+      this.serverName = res.name;
+    }
+    if (res.error) {
+      this.cancelAccordingToError(res.error);
+    }
+  }
+
+  resume(forced = false) {
+    if (!(FORCABLE & this.state)) {
+      return;
+    }
+    if (this.state !== QUEUED) {
+      this.changeState(QUEUED);
+    }
+    if (forced) {
+      this.manager.startDownload(this);
+    }
+  }
+
+  async pause(retry?: boolean) {
+    if (!(PAUSEABLE & this.state)) {
+      return;
+    }
+
+    if (!retry) {
+      this.retries = 0;
+      this.deadline = 0;
+    }
+    else {
+      // eslint-disable-next-line no-magic-numbers
+      this.deadline = Date.now() + RETRY_TIME.value * 60 * 1000;
+    }
+
+    if (this.state === RUNNING && this.manId) {
+      try {
+        await downloads.pause(this.manId);
+      }
+      catch (ex) {
+        console.error("pause", ex.toString(), ex);
+        this.cancel();
+        return;
+      }
+    }
+
+    this.changeState(retry ? RETRYING : PAUSED);
+  }
+
+  reset() {
+    this.prerolled = false;
+    this.manId = 0;
+    this.written = this.totalSize = 0;
+    this.mime = this.serverName = this.browserName = "";
+    this.retries = 0;
+    this.deadline = 0;
+  }
+
+  async removeFromBrowser() {
+    const {manId: id} = this;
+    try {
+      await downloads.cancel(id);
+    }
+    catch (ex) {
+      // ignored
+    }
+    await new Promise(r => setTimeout(r, 1000));
+    try {
+      await downloads.erase({id});
+    }
+    catch (ex) {
+      console.error(id, ex.toString(), ex);
+      // ignored
+    }
+  }
+
+  cancel() {
+    if (!(CANCELABLE & this.state)) {
+      return;
+    }
+    if (this.manId) {
+      this.manager.removeManId(this.manId);
+      this.removeFromBrowser();
+    }
+    this.reset();
+    this.changeState(CANCELED);
+  }
+
+  async cancelAccordingToError(error: string) {
+    if (!isRecoverable(error) || ++this.retries > RETRIES.value) {
+      this.cancel();
+      this.error = error;
+      return;
+    }
+
+    await this.pause(true);
+    this.error = error;
+  }
+
+  setMissing() {
+    if (this.manId) {
+      this.manager.removeManId(this.manId);
+      this.removeFromBrowser();
+    }
+    this.reset();
+    this.changeState(MISSING);
+  }
+
+  async maybeMissing() {
+    if (!this.manId) {
+      return null;
+    }
+    const {manId: id} = this;
+    try {
+      const dls = await downloads.search({id});
+      if (!dls.length) {
+        this.setMissing();
+        return this;
+      }
+    }
+    catch (ex) {
+      console.error("oops", id, ex.toString(), ex);
+      this.setMissing();
+      return this;
+    }
+    return null;
+  }
+
+  adoptSize(state: any) {
+    const {
+      bytesReceived,
+      totalBytes,
+      fileSize
+    } = state;
+    this.written = Math.max(0, bytesReceived);
+    this.totalSize = Math.max(0, fileSize >= 0 ? fileSize : totalBytes);
+  }
+
+  async updateStateFromBrowser() {
+    try {
+      const state = (await downloads.search({id: this.manId})).pop();
+      const {filename, error} = state;
+      const path = parsePath(filename);
+      this.browserName = path.name;
+      this.adoptSize(state);
+      if (!this.mime && state.mime) {
+        this.mime = state.mime;
+      }
+      this.markDirty();
+      switch (state.state) {
+      case "in_progress":
+        if (state.paused) {
+          this.changeState(PAUSED);
+        }
+        else if (error) {
+          this.cancelAccordingToError(error);
+        }
+        else {
+          this.changeState(RUNNING);
+        }
+        break;
+
+      case "interrupted":
+        if (state.paused) {
+          this.changeState(PAUSED);
+        }
+        else if (error) {
+          this.cancelAccordingToError(error);
+        }
+        else {
+          this.cancel();
+          this.error = error || "";
+        }
+        break;
+
+      case "complete":
+        this.changeState(DONE);
+        break;
+      }
+    }
+    catch (ex) {
+      console.error("failed to handle state", ex.toString(), ex.stack, ex);
+      this.setMissing();
+    }
+  }
+
+  updateFromSuggestion(state: any) {
+    const res: PrerollResults = {};
+    if (state.mime) {
+      res.mime = state.mime;
+    }
+    if (state.filename) {
+      res.name = state.filename;
+    }
+    if (state.finalUrl) {
+      res.finalURL = state.finalUrl;
+      const detected = Preroller.maybeFindNameFromSearchParams(this, res);
+      if (detected) {
+        res.name = detected;
+      }
+    }
+    try {
+      this.adoptPrerollResults(res);
+    }
+    finally {
+      this.markDirty();
+    }
+  }
+}

extensions/downthemall/lib/manager/limits.ts

@@ -0,0 +1,115 @@
+"use strict";
+// License: MIT
+
+import { Prefs } from "../prefs";
+import { EventEmitter } from "../events";
+
+const DEFAULT = {
+  concurrent: -1,
+};
+
+class Limit {
+  public readonly domain: string;
+
+  public concurrent: number;
+
+  constructor(raw: any) {
+    Object.assign(this, DEFAULT, raw);
+    if (!this.domain) {
+      throw new Error("No domain");
+    }
+    if (!isFinite(this.concurrent) ||
+      (this.concurrent | 0) !== this.concurrent ||
+      this.concurrent < -1) {
+      throw new Error("Invalid concurrent");
+    }
+  }
+
+  toJSON() {
+    return {
+      domain: this.domain,
+      concurrent: this.concurrent
+    };
+  }
+}
+
+export const Limits = new class Limits extends EventEmitter {
+  public concurrent: number;
+
+  private limits: Map<string, Limit>;
+
+  constructor() {
+    super();
+    this.concurrent = 4;
+    this.limits = new Map();
+    const onpref = this.onpref.bind(this);
+    Prefs.on("concurrent", onpref);
+    Prefs.on("limits", onpref);
+  }
+
+  *[Symbol.iterator]() {
+    for (const [domain, v] of this.limits.entries()) {
+      const {concurrent} = v;
+      yield {
+        domain,
+        concurrent,
+      };
+    }
+  }
+
+  onpref(prefs: any, key: string, value: any) {
+    switch (key) {
+    case "limits":
+      this.limits = new Map(value.map((e: any) => [e.domain, new Limit(e)]));
+      break;
+
+    case "concurrent":
+      this.concurrent = value;
+      break;
+    }
+    this.emit("changed");
+  }
+
+  async load() {
+    this.concurrent = await Prefs.get("concurrent", this.concurrent);
+    const rawlimits = await Prefs.get("limits");
+    this.limits = new Map(rawlimits.map((e: any) => [e.domain, new Limit(e)]));
+    this.load = (() => {}) as unknown as () => Promise<void>;
+    this.emit("changed");
+  }
+
+  getConcurrentFor(domain: string) {
+    let rv: number;
+    const dlimit = this.limits.get(domain);
+    if (dlimit) {
+      rv = dlimit.concurrent;
+    }
+    else {
+      const limit = this.limits.get("*");
+      rv = limit && limit.concurrent || -1;
+    }
+    return rv > 0 ? rv : this.concurrent;
+  }
+
+  async saveEntry(domain: string, descriptor: any) {
+    const limit = new Limit(Object.assign({}, descriptor, {domain}));
+    this.limits.set(limit.domain, limit);
+    await this.save();
+  }
+
+  async save() {
+    const limits = JSON.parse(JSON.stringify(this));
+    await Prefs.set("limits", limits);
+  }
+
+  async "delete"(domain: string) {
+    if (!this.limits.delete(domain)) {
+      return;
+    }
+    await this.save();
+  }
+
+  toJSON() {
+    return Array.from(this.limits.values());
+  }
+}();

extensions/downthemall/lib/manager/man.ts

@@ -0,0 +1,516 @@
+"use strict";
+// License: MIT
+
+import { EventEmitter } from "../events";
+import { Notification } from "../notifications";
+import { DB } from "../db";
+import { QUEUED, CANCELED, RUNNING, RETRYING } from "./state";
+// eslint-disable-next-line no-unused-vars
+import { Bus, Port } from "../bus";
+import { sort } from "../sorting";
+import { Prefs, PrefWatcher } from "../prefs";
+import { _ } from "../i18n";
+import { CoalescedUpdate, mapFilterInSitu, filterInSitu } from "../util";
+import { PromiseSerializer } from "../pserializer";
+import { Download } from "./download";
+import { ManagerPort } from "./port";
+import { Scheduler } from "./scheduler";
+import { Limits } from "./limits";
+import { downloads, runtime, webRequest, CHROME, OPERA } from "../browser";
+
+const US = runtime.getURL("");
+
+const AUTOSAVE_TIMEOUT = 2000;
+const DIRTY_TIMEOUT = 100;
+// eslint-disable-next-line no-magic-numbers
+const MISSING_TIMEOUT = 12 * 1000;
+const RELOAD_TIMEOUT = 10 * 1000;
+
+const setShelfEnabled = downloads.setShelfEnabled || function() {
+  // ignored
+};
+
+const FINISH_NOTIFICATION = new PrefWatcher("finish-notification", true);
+const SOUNDS = new PrefWatcher("sounds", false);
+
+export class Manager extends EventEmitter {
+  private items: Download[];
+
+  public active: boolean;
+
+  private notifiedFinished: boolean;
+
+  private readonly saveQueue: CoalescedUpdate<Download>;
+
+  private readonly dirty: CoalescedUpdate<Download>;
+
+  private readonly sids: Map<number, Download>;
+
+  private readonly manIds: Map<number, Download>;
+
+  private readonly ports: Set<ManagerPort>;
+
+  private readonly running: Set<Download>;
+
+  private readonly retrying: Set<Download>;
+
+  private scheduler: Scheduler | null;
+
+  private shouldReload: boolean;
+
+  private deadlineTimer: number;
+
+  constructor() {
+    if (!document.location.href.includes("background")) {
+      throw new Error("Not on background");
+    }
+    super();
+    this.active = true;
+    this.shouldReload = false;
+    this.notifiedFinished = true;
+    this.items = [];
+    this.saveQueue = new CoalescedUpdate(
+      AUTOSAVE_TIMEOUT, this.save.bind(this));
+    this.dirty = new CoalescedUpdate(
+      DIRTY_TIMEOUT, this.processDirty.bind(this));
+    this.processDeadlines = this.processDeadlines.bind(this);
+    this.sids = new Map();
+    this.manIds = new Map();
+    this.ports = new Set();
+    this.scheduler = null;
+    this.running = new Set();
+    this.retrying = new Set();
+
+    this.startNext = PromiseSerializer.wrapNew(1, this, this.startNext);
+
+    downloads.onChanged.addListener(this.onChanged.bind(this));
+    downloads.onErased.addListener(this.onErased.bind(this));
+    if (CHROME && downloads.onDeterminingFilename) {
+      downloads.onDeterminingFilename.addListener(
+        this.onDeterminingFilename.bind(this));
+    }
+
+    Bus.onPort("manager", (port: Port) => {
+      const managerPort = new ManagerPort(this, port);
+      port.on("disconnect", () => {
+        this.ports.delete(managerPort);
+      });
+      this.ports.add(managerPort);
+      return true;
+    });
+    Limits.on("changed", () => {
+      this.resetScheduler();
+    });
+
+    if (CHROME) {
+      webRequest.onBeforeSendHeaders.addListener(
+        this.stuffReferrer.bind(this),
+        {urls: ["<all_urls>"]},
+        ["blocking", "requestHeaders", "extraHeaders"]
+      );
+    }
+  }
+
+  async init() {
+    const items = await DB.getAll();
+    items.forEach((i: any, idx: number) => {
+      const rv = new Download(this, i);
+      rv.position = idx;
+      this.sids.set(rv.sessionId, rv);
+      if (rv.manId) {
+        this.manIds.set(rv.manId, rv);
+      }
+      this.items.push(rv);
+    });
+
+    // Do not wait for the scheduler
+    this.resetScheduler();
+
+    this.emit("initialized");
+    setTimeout(() => this.checkMissing(), MISSING_TIMEOUT);
+    runtime.onUpdateAvailable.addListener(() => {
+      if (this.running.size) {
+        this.shouldReload = true;
+        return;
+      }
+      runtime.reload();
+    });
+    return this;
+  }
+
+  async checkMissing() {
+    const serializer = new PromiseSerializer(2);
+    const missing = await Promise.all(this.items.map(
+      item => serializer.scheduleWithContext(item, item.maybeMissing)));
+    if (!(await Prefs.get("remove-missing-on-init"))) {
+      return;
+    }
+    this.remove(filterInSitu(missing, e => !!e));
+  }
+
+  onChanged(changes: {id: number}) {
+    const item = this.manIds.get(changes.id);
+    if (!item) {
+      return;
+    }
+    item.updateStateFromBrowser();
+  }
+
+  onErased(downloadId: number) {
+    const item = this.manIds.get(downloadId);
+    if (!item) {
+      return;
+    }
+    item.setMissing();
+    this.manIds.delete(downloadId);
+  }
+
+  onDeterminingFilename(state: any, suggest: Function) {
+    const download = this.manIds.get(state.id);
+    if (!download) {
+      return;
+    }
+    try {
+      download.updateFromSuggestion(state);
+    }
+    finally {
+      const suggestion = {filename: download.dest.full};
+      suggest(suggestion);
+    }
+  }
+
+  async resetScheduler() {
+    this.scheduler = null;
+    await this.startNext();
+  }
+
+  async startNext() {
+    if (!this.active) {
+      return;
+    }
+    while (this.running.size < Limits.concurrent) {
+      if (!this.scheduler) {
+        this.scheduler = new Scheduler(this.items);
+      }
+      const next = await this.scheduler.next(this.running);
+      if (!next) {
+        this.maybeRunFinishActions();
+        break;
+      }
+      if (this.running.has(next) || next.state !== QUEUED) {
+        continue;
+      }
+      try {
+        await this.startDownload(next);
+      }
+      catch (ex) {
+        next.changeState(CANCELED);
+        next.error = ex.toString();
+        console.error(ex.toString(), ex);
+      }
+    }
+  }
+
+  async startDownload(download: Download) {
+    // Add to running first, so we don't confuse the scheduler and other parts
+    this.running.add(download);
+    setShelfEnabled(false);
+    await download.start();
+    this.notifiedFinished = false;
+  }
+
+  maybeRunFinishActions() {
+    if (this.running.size) {
+      return;
+    }
+    this.maybeNotifyFinished();
+    if (this.shouldReload) {
+      this.saveQueue.trigger();
+      setTimeout(() => {
+        if (this.running.size) {
+          return;
+        }
+        runtime.reload();
+      }, RELOAD_TIMEOUT);
+    }
+    setShelfEnabled(true);
+  }
+
+  maybeNotifyFinished() {
+    if (this.notifiedFinished || this.running.size || this.retrying.size) {
+      return;
+    }
+    if (SOUNDS.value && !OPERA) {
+      const audio = new Audio(runtime.getURL("/style/done.opus"));
+      audio.addEventListener("canplaythrough", () => audio.play());
+      audio.addEventListener("ended", () => document.body.removeChild(audio));
+      audio.addEventListener("error", () => document.body.removeChild(audio));
+      document.body.appendChild(audio);
+    }
+    if (FINISH_NOTIFICATION.value) {
+      new Notification(null, _("queue-finished"));
+    }
+    this.notifiedFinished = true;
+  }
+
+  addManId(id: number, download: Download) {
+    this.manIds.set(id, download);
+  }
+
+  removeManId(id: number) {
+    this.manIds.delete(id);
+  }
+
+  addNewDownloads(items: any[]) {
+    if (!items || !items.length) {
+      return;
+    }
+    items = items.map(i => {
+      const dl = new Download(this, i);
+      dl.position = this.items.push(dl) - 1;
+      this.sids.set(dl.sessionId, dl);
+      dl.markDirty();
+      return dl;
+    });
+
+    Prefs.get("nagging", 0).
+      then(v => {
+        return Prefs.set("nagging", (v || 0) + items.length);
+      }).
+      catch(console.error);
+
+    this.scheduler = null;
+    this.save(items);
+    this.startNext();
+  }
+
+  setDirty(item: Download) {
+    this.dirty.add(item);
+  }
+
+  removeDirty(item: Download) {
+    this.dirty.delete(item);
+  }
+
+  processDirty(items: Download[]) {
+    items = items.filter(i => !i.removed);
+    items.forEach(item => this.saveQueue.add(item));
+    this.emit("dirty", items);
+  }
+
+  private save(items: Download[]) {
+    DB.saveItems(items.filter(i => !i.removed)).
+      catch(console.error);
+  }
+
+  setPositions() {
+    const items = this.items.filter((e, idx) => {
+      if (e.position === idx) {
+        return false;
+      }
+      e.position = idx;
+      e.markDirty();
+      return true;
+    });
+    if (!items.length) {
+      return;
+    }
+    this.save(items);
+    this.resetScheduler();
+  }
+
+  forEach(sids: number[], cb: (item: Download) => void) {
+    sids.forEach(sid => {
+      const download = this.sids.get(sid);
+      if (!download) {
+        return;
+      }
+      cb.call(this, download);
+    });
+  }
+
+  resumeDownloads(sids: number[], forced = false) {
+    this.forEach(sids, download => download.resume(forced));
+  }
+
+  pauseDownloads(sids: number[]) {
+    this.forEach(sids, download => download.pause());
+  }
+
+  cancelDownloads(sids: number[]) {
+    this.forEach(sids, download => download.cancel());
+  }
+
+  setMissing(sid: number) {
+    this.forEach([sid], download => download.setMissing());
+  }
+
+  changedState(download: Download, oldState: number, newState: number) {
+    if (oldState === RUNNING) {
+      this.running.delete(download);
+    }
+    else if (oldState === RETRYING) {
+      this.retrying.delete(download);
+      this.findDeadline();
+    }
+    if (newState === QUEUED) {
+      this.resetScheduler();
+      this.startNext().catch(console.error);
+    }
+    else if (newState === RUNNING) {
+      // Usually we already added it. But if a user uses the built-in
+      // download manager to restart
+      // a download, we have not, so make sure it is added either way
+      this.running.add(download);
+    }
+    else {
+      if (newState === RETRYING) {
+        this.addRetry(download);
+      }
+      this.startNext().catch(console.error);
+    }
+  }
+
+  addRetry(download: Download) {
+    this.retrying.add(download);
+    this.findDeadline();
+  }
+
+  private findDeadline() {
+    let deadline = Array.from(this.retrying).
+      reduce<number>((deadline, item) => {
+        if (deadline) {
+          return item.deadline ? Math.min(deadline, item.deadline) : deadline;
+        }
+        return item.deadline;
+      }, 0);
+    if (deadline <= 0) {
+      return;
+    }
+    deadline -= Date.now();
+    if (deadline <= 0) {
+      return;
+    }
+
+    if (this.deadlineTimer) {
+      window.clearTimeout(this.deadlineTimer);
+    }
+    this.deadlineTimer = window.setTimeout(this.processDeadlines, deadline);
+  }
+
+  private processDeadlines() {
+    this.deadlineTimer = 0;
+    try {
+      const now = Date.now();
+      this.items.forEach(item => {
+        if (item.deadline && Math.abs(item.deadline - now) < 1000) {
+          this.retrying.delete(item);
+          item.resume(false);
+        }
+      });
+    }
+    finally {
+      this.findDeadline();
+    }
+  }
+
+  sorted(sids: number[]) {
+    try {
+      // Construct new items
+      const currentSids = new Map(this.sids);
+      let items = mapFilterInSitu(sids, sid => {
+        const item = currentSids.get(sid);
+        if (!item) {
+          return null;
+        }
+        currentSids.delete(sid);
+        return item;
+      }, e => !!e);
+      if (currentSids.size) {
+        items = items.concat(
+          sort(Array.from(currentSids.values()), i => i.position));
+      }
+      this.items = items;
+      this.setPositions();
+    }
+    catch (ex) {
+      console.error("sorted", "sids", sids, "ex", ex.message, ex);
+    }
+  }
+
+  remove(items: Download[]) {
+    if (!items.length) {
+      return;
+    }
+    items.forEach(item => {
+      item.removed = true;
+      if (!item.manId) {
+        return;
+      }
+      this.removeManId(item.manId);
+      item.cancel();
+    });
+    DB.deleteItems(items).then(() => {
+      const sids = items.map(item => item.sessionId);
+      sids.forEach(sid => this.sids.delete(sid));
+      sort(items.map(item => item.position)).
+        reverse().
+        forEach(idx => this.items.splice(idx, 1));
+      this.emit("removed", sids);
+      this.setPositions();
+      this.resetScheduler();
+    }).catch(console.error);
+  }
+
+  removeBySids(sids: number[]) {
+    const items = mapFilterInSitu(sids, sid => this.sids.get(sid), e => !!e);
+    return this.remove(items);
+  }
+
+  toggleActive() {
+    this.active = !this.active;
+    if (this.active) {
+      this.startNext();
+    }
+    this.emit("active", this.active);
+  }
+
+  getMsgItems() {
+    return this.items.map(e => e.toMsg());
+  }
+
+  stuffReferrer(details: any): any {
+    if (details.tabId > 0 && !US.startsWith(details.initiator)) {
+      return undefined;
+    }
+    const sidx = details.requestHeaders.findIndex(
+      (e: any) => e.name.toLowerCase() === "x-dta-id");
+    if (sidx < 0) {
+      return undefined;
+    }
+    const sid = parseInt(details.requestHeaders[sidx].value, 10);
+    details.requestHeaders.splice(sidx, 1);
+    const item = this.sids.get(sid);
+    if (!item) {
+      return undefined;
+    }
+    details.requestHeaders.push({
+      name: "Referer",
+      value: (item.uReferrer || item.uURL).toString()
+    });
+    const rv: any = {
+      requestHeaders: details.requestHeaders
+    };
+    return rv;
+  }
+}
+
+let inited: Promise<Manager>;
+
+export function getManager() {
+  if (!inited) {
+    const man = new Manager();
+    inited = man.init();
+  }
+  return inited;
+}

extensions/downthemall/lib/manager/port.ts

@@ -0,0 +1,93 @@
+"use strict";
+// License: MIT
+
+import { donate, openPrefs } from "../windowutils";
+import { API } from "../api";
+// eslint-disable-next-line no-unused-vars
+import { BaseDownload } from "./basedownload";
+// eslint-disable-next-line no-unused-vars
+import { Manager } from "./man";
+// eslint-disable-next-line no-unused-vars
+import { Port } from "../bus";
+// eslint-disable-next-line no-unused-vars
+import { BaseItem } from "../item";
+
+type SID = {sid: number};
+type SIDS = {
+  sids: number[];
+  forced?: boolean;
+};
+
+export class ManagerPort {
+  private manager: Manager;
+
+  private port: Port;
+
+  constructor(manager: any, port: any) {
+    this.manager = manager;
+    this.port = port;
+
+    this.onDirty = this.onDirty.bind(this);
+    this.onRemoved = this.onRemoved.bind(this);
+    this.onMsgRemoveSids = this.onMsgRemoveSids.bind(this);
+
+    this.manager.on("inited", () => this.sendAll());
+    this.manager.on("dirty", this.onDirty);
+    this.manager.on("removed", this.onRemoved);
+    this.manager.on("active", (active: any) => {
+      this.port.post("active", active);
+    });
+
+    port.on("donate", () => {
+      donate();
+    });
+    port.on("prefs", () => {
+      openPrefs();
+    });
+    port.on("import", ({items}: {items: BaseItem[]}) => {
+      API.regular(items, []);
+    });
+    port.on("all", () => this.sendAll());
+    port.on("removeSids", this.onMsgRemoveSids);
+    port.on("showSingle", async () => {
+      await API.singleRegular(null);
+    });
+    port.on("toggle-active", () => {
+      this.manager.toggleActive();
+    });
+    port.on("sorted", ({sids}: SIDS) => this.manager.sorted(sids));
+    port.on("resume",
+      ({sids, forced}: SIDS) => this.manager.resumeDownloads(sids, forced));
+    port.on("pause", ({sids}: SIDS) => this.manager.pauseDownloads(sids));
+    port.on("cancel", ({sids}: SIDS) => this.manager.cancelDownloads(sids));
+    port.on("missing", ({sid}: SID) => this.manager.setMissing(sid));
+
+    this.port.on("disconnect", () => {
+      this.manager.off("dirty", this.onDirty);
+      this.manager.off("removed", this.onRemoved);
+
+      port.off("removeSids", this.onMsgRemoveSids);
+      delete this.manager;
+      delete this.port;
+    });
+
+    this.port.post("active", this.manager.active);
+    this.sendAll();
+  }
+
+  onDirty(items: BaseDownload[]) {
+    this.port.post("dirty", items.map(item => item.toMsg()));
+  }
+
+  onRemoved(sids: number[]) {
+    this.port.post("removed", sids);
+  }
+
+  onMsgRemoveSids({sids}: SIDS) {
+    this.manager.removeBySids(sids);
+  }
+
+  sendAll() {
+    this.port.post("all", this.manager.getMsgItems());
+  }
+}

extensions/downthemall/lib/manager/preroller.ts

@@ -0,0 +1,252 @@
+"use strict";
+// License: MIT
+
+import MimeType from "whatwg-mimetype";
+// eslint-disable-next-line no-unused-vars
+import { Download } from "./download";
+import { CHROME, webRequest } from "../browser";
+import { CDHeaderParser } from "../cdheaderparser";
+import { sanitizePath, parsePath } from "../util";
+import { MimeDB } from "../mime";
+
+const PREROLL_HEURISTICS = /dl|attach|download|name|file|get|retr|^n$|\.(php|asp|py|pl|action|htm|shtm)/i;
+const PREROLL_HOSTS = /4cdn|chan/;
+const PREROLL_TIMEOUT = 10000;
+const PREROLL_NOPE = new Set<string>();
+
+/* eslint-disable no-magic-numbers */
+const NOPE_STATUSES = Object.freeze(new Set([
+  400,
+  401,
+  402,
+  405,
+  416,
+]));
+/* eslint-enable no-magic-numbers */
+
+const PREROLL_SEARCHEXTS = Object.freeze(new Set<string>([
+  "php",
+  "asp",
+  "aspx",
+  "inc",
+  "py",
+  "pl",
+  "action",
+  "htm",
+  "html",
+  "shtml"
+]));
+const NAME_TESTER = /\.[a-z0-9]{1,5}$/i;
+const CDPARSER = new CDHeaderParser();
+
+export interface PrerollResults {
+  error?: string;
+  name?: string;
+  mime?: string;
+  finalURL?: string;
+}
+
+export class Preroller {
+  private readonly download: Download
+
+  constructor(download: Download) {
+    this.download = download;
+  }
+
+  get shouldPreroll() {
+    if (CHROME) {
+      return false;
+    }
+    const {uURL, renamer} = this.download;
+    const {pathname, search, host} = uURL;
+    if (PREROLL_NOPE.has(host)) {
+      return false;
+    }
+    if (!renamer.p_ext) {
+      return true;
+    }
+    if (search.length) {
+      return true;
+    }
+    if (uURL.pathname.endsWith("/")) {
+      return true;
+    }
+    if (PREROLL_HEURISTICS.test(pathname)) {
+      return true;
+    }
+    if (PREROLL_HOSTS.test(host)) {
+      return true;
+    }
+    return false;
+  }
+
+  async roll() {
+    try {
+      return await (CHROME ? this.prerollChrome() : this.prerollFirefox());
+    }
+    catch (ex) {
+      console.error("Failed to preroll", this, ex.toString(), ex.stack, ex);
+    }
+    return null;
+  }
+
+  private async prerollFirefox() {
+    const controller = new AbortController();
+    const {signal} = controller;
+    const {uURL, uReferrer} = this.download;
+    const res = await fetch(uURL.toString(), {
+      method: "GET",
+      headers: new Headers({
+        Range: "bytes=0-1",
+      }),
+      mode: "same-origin",
+      signal,
+      referrer: (uReferrer || uURL).toString(),
+    });
+    if (res.body) {
+      res.body.cancel();
+    }
+    controller.abort();
+    const {headers} = res;
+    return this.finalize(headers, res);
+  }
+
+  private async prerollChrome() {
+    let rid = "";
+    const {uURL, uReferrer} = this.download;
+    const rurl = uURL.toString();
+    let listener: any;
+    const wr = new Promise<any[]>(resolve => {
+      listener = (details: any) => {
+        const {url, requestId, statusCode} = details;
+        if (rid !== requestId && url !== rurl) {
+          return;
+        }
+        // eslint-disable-next-line no-magic-numbers
+        if (statusCode >= 300 && statusCode < 400) {
+          // Redirect, continue tracking;
+          rid = requestId;
+          return;
+        }
+        resolve(details.responseHeaders);
+      };
+      webRequest.onHeadersReceived.addListener(
+        listener, {urls: ["<all_urls>"]}, ["responseHeaders"]);
+    });
+    const p = Promise.race([
+      wr,
+      new Promise<any[]>((_, reject) =>
+        setTimeout(() => reject(new Error("timeout")), PREROLL_TIMEOUT))
+    ]);
+
+    p.finally(() => {
+      webRequest.onHeadersReceived.removeListener(listener);
+    });
+    const controller = new AbortController();
+    const {signal} = controller;
+    const res = await fetch(rurl, {
+      method: "GET",
+      headers: new Headers({
+        "Range": "bytes=0-1",
+        "X-DTA-ID": this.download.sessionId.toString(),
+      }),
+      signal,
+      referrer: (uReferrer || uURL).toString(),
+    });
+    if (res.body) {
+      res.body.cancel();
+    }
+    controller.abort();
+    const headers = await p;
+    return this.finalize(
+      new Headers(headers.map(i => [i.name, i.value])), res);
+  }
+
+  private finalize(headers: Headers, res: Response): PrerollResults {
+    const rv: PrerollResults = {};
+
+    const type = MimeType.parse(headers.get("content-type") || "");
+    if (type) {
+      rv.mime = type.essence;
+    }
+
+    const dispHeader = headers.get("content-disposition");
+    if (dispHeader) {
+      const file = CDPARSER.parse(dispHeader);
+      // Sanitize
+      rv.name = sanitizePath(file.replace(/[/\\]+/g, "-"));
+    }
+    else {
+      const detected = Preroller.maybeFindNameFromSearchParams(
+        this.download, rv);
+      if (detected) {
+        rv.name = detected;
+      }
+    }
+
+    rv.finalURL = res.url;
+
+    /* eslint-disable no-magic-numbers */
+    const {status} = res;
+    if (status === 404) {
+      rv.error = "SERVER_BAD_CONTENT";
+    }
+    else if (status === 403) {
+      rv.error = "SERVER_FORBIDDEN";
+    }
+    else if (status === 402 || status === 407) {
+      rv.error = "SERVER_UNAUTHORIZED";
+    }
+    else if (NOPE_STATUSES.has(status)) {
+      PREROLL_NOPE.add(this.download.uURL.host);
+      if (PREROLL_NOPE.size > 1000) {
+        PREROLL_NOPE.delete(PREROLL_NOPE.keys().next().value);
+      }
+    }
+    else if (status > 400 && status < 500) {
+      rv.error = "SERVER_FAILED";
+    }
+    /* eslint-enable no-magic-numbers */
+
+    return rv;
+  }
+
+
+  static maybeFindNameFromSearchParams(
+      download: Download, res: PrerollResults) {
+    const {p_ext: ext} = download.renamer;
+    if (ext && !PREROLL_SEARCHEXTS.has(ext.toLocaleLowerCase())) {
+      return undefined;
+    }
+    return Preroller.findNameFromSearchParams(download.uURL, res.mime);
+  }
+
+  static findNameFromSearchParams(url: URL, mimetype?: string) {
+    const {searchParams} = url;
+    let detected = "";
+    for (const [, value] of searchParams) {
+      if (!NAME_TESTER.test(value)) {
+        continue;
+      }
+      const p = parsePath(value);
+      if (!p.base || !p.ext) {
+        continue;
+      }
+      if (!MimeDB.hasExtension(p.ext)) {
+        continue;
+      }
+      if (mimetype) {
+        const mime = MimeDB.getMime(mimetype);
+        if (mime && !mime.extensions.has(p.ext.toLowerCase())) {
+          continue;
+        }
+      }
+      const sanitized = sanitizePath(p.name);
+      if (sanitized.length <= detected.length) {
+        continue;
+      }
+      detected = sanitized;
+    }
+    return detected;
+  }
+}

extensions/downthemall/lib/manager/renamer.ts

@@ -0,0 +1,276 @@
+/* eslint-disable @typescript-eslint/camelcase */
+"use strict";
+// License: MIT
+
+import { _ } from "../i18n";
+import { MimeDB } from "../mime";
+// eslint-disable-next-line no-unused-vars
+import { parsePath, PathInfo, sanitizePath } from "../util";
+// eslint-disable-next-line no-unused-vars
+import { BaseDownload } from "./basedownload";
+
+const REPLACE_EXPR = /\*\w+\*/gi;
+
+const BATCH_FORMATTER = new Intl.NumberFormat(undefined, {
+  style: "decimal",
+  useGrouping: false,
+  minimumIntegerDigits: 3,
+  maximumFractionDigits: 0
+});
+
+const DATE_FORMATTER = new Intl.NumberFormat(undefined, {
+  style: "decimal",
+  useGrouping: false,
+  minimumIntegerDigits: 2,
+  maximumFractionDigits: 0
+});
+
+export default class Renamer {
+  private readonly d: BaseDownload;
+
+  private readonly nameinfo: PathInfo;
+
+  constructor(download: BaseDownload) {
+    this.d = download;
+    const info = parsePath(this.d.finalName);
+    this.nameinfo = this.fixupExtension(info);
+  }
+
+  private fixupExtension(info: PathInfo): PathInfo {
+    if (!this.d.mime) {
+      return info;
+    }
+    const mime = MimeDB.getMime(this.d.mime);
+    if (!mime) {
+      return info;
+    }
+    const {ext} = info;
+    if (mime.major === "image" || mime.major === "video") {
+      if (ext && mime.extensions.has(ext.toLowerCase())) {
+        return info;
+      }
+      return new PathInfo(info.base, mime.primary, info.path);
+    }
+    if (ext) {
+      return info;
+    }
+    return new PathInfo(info.base, mime.primary, info.path);
+  }
+
+  get ref() {
+    return this.d.uReferrer;
+  }
+
+  get p_name() {
+    return this.nameinfo.base;
+  }
+
+  get p_ext() {
+    return this.nameinfo.ext;
+  }
+
+  get p_text() {
+    return this.d.description;
+  }
+
+  get p_title() {
+    return this.d.title;
+  }
+
+  get p_host() {
+    return this.d.uURL.host;
+  }
+
+  get p_domain() {
+    return this.d.uURL.domain;
+  }
+
+  get p_subdirs() {
+    return parsePath(this.d.uURL).path;
+  }
+
+  get p_qstring() {
+    const {search} = this.d.uURL;
+    return search && search.substr(1).replace(/\/+/g, "-");
+  }
+
+  get p_url() {
+    return this.d.usable.substr(this.d.uURL.protocol.length + 2);
+  }
+
+  get p_batch() {
+    return BATCH_FORMATTER.format(this.d.batch);
+  }
+
+  get p_num() {
+    return BATCH_FORMATTER.format(this.d.batch);
+  }
+
+  get p_idx() {
+    return BATCH_FORMATTER.format(this.d.idx);
+  }
+
+  get p_date() {
+    return `${this.p_y}${this.p_m}${this.p_d}T${this.p_hh}${this.p_mm}${this.p_ss}`;
+  }
+
+  get p_refname() {
+    const {ref} = this;
+    if (!ref) {
+      return null;
+    }
+    return parsePath(ref).base;
+  }
+
+  get p_refext() {
+    const {ref} = this;
+    if (!ref) {
+      return null;
+    }
+    return parsePath(ref).ext;
+  }
+
+  get p_refhost() {
+    const {ref} = this;
+    if (!ref) {
+      return null;
+    }
+    return ref.host;
+  }
+
+  get p_refdomain() {
+    const {ref} = this;
+    if (!ref) {
+      return null;
+    }
+    return ref.domain;
+  }
+
+  get p_refsubdirs() {
+    const {ref} = this;
+    if (!ref) {
+      return null;
+    }
+    return parsePath(ref).path;
+  }
+
+  get p_refqstring() {
+    const {ref} = this;
+    if (!ref) {
+      return null;
+    }
+    const {search} = ref;
+    return search && search.substr(1).replace(/\/+/g, "-");
+  }
+
+  get p_refurl() {
+    return this.d.usableReferrer.substr(
+      this.d.uReferrer.protocol.length + 2);
+  }
+
+  get p_hh() {
+    return DATE_FORMATTER.format(this.d.startDate.getHours());
+  }
+
+  get p_mm() {
+    return DATE_FORMATTER.format(this.d.startDate.getMinutes());
+  }
+
+  get p_ss() {
+    return DATE_FORMATTER.format(this.d.startDate.getSeconds());
+  }
+
+  get p_d() {
+    return DATE_FORMATTER.format(this.d.startDate.getDate());
+  }
+
+  get p_m() {
+    return DATE_FORMATTER.format(this.d.startDate.getMonth() + 1);
+  }
+
+  get p_y() {
+    return DATE_FORMATTER.format(this.d.startDate.getFullYear());
+  }
+
+  toString() {
+    const {mask, subfolder} = this.d;
+    // eslint-disable-next-line @typescript-eslint/no-this-alias
+    const self: any = this;
+    const baseMask = subfolder ? `${subfolder}/${mask}` : mask;
+    return sanitizePath(baseMask.replace(REPLACE_EXPR, function(type: string) {
+      let prop = type.substr(1, type.length - 2);
+      const flat = prop.startsWith("flat");
+      if (flat) {
+        prop = prop.substr(4);
+      }
+      prop = `p_${prop}`;
+      let rv = (prop in self) ?
+        (self[prop] || "").trim() :
+        type;
+      if (flat) {
+        rv = rv.replace(/[/\\]+/g, "-");
+      }
+      return rv.replace(/\/{2,}/g, "/");
+    }));
+  }
+}
+
+export const SUPPORTED =
+  Object.keys(Object.getOwnPropertyDescriptors(Renamer.prototype)).
+    filter(k => k.startsWith("p_")).
+    map(k => k.slice(2));
+
+function makeHTMLMap() {
+  const e = document.createElement("section");
+  e.className = "renamer-map";
+
+  const head = document.createElement("h2");
+  head.className = "renamer-head";
+  head.textContent = _("renamer-tags");
+  e.appendChild(head);
+
+  const tags = SUPPORTED;
+  const mid = Math.ceil(tags.length / 2);
+  for (const half of [tags.slice(0, mid), tags.slice(mid)]) {
+    const cont = document.createElement("div");
+    cont.className = "renamer-half";
+    for (const k of half) {
+      const tag = document.createElement("code");
+      tag.className = "renamer-tag";
+      tag.textContent = `*${k}*`;
+      cont.appendChild(tag);
+
+      const label = document.createElement("label");
+      label.className = "renamer-label";
+      label.textContent = _(`renamer-${k}`);
+      cont.appendChild(label);
+    }
+    e.appendChild(cont);
+  }
+  const info = document.createElement("em");
+  info.className = "renamer-info";
+  info.textContent = _("renamer-info");
+  e.appendChild(info);
+  return e;
+}
+
+export function hookButton(maskButton: HTMLElement) {
+  let maskMap: HTMLElement;
+  maskButton.addEventListener("click", (evt: MouseEvent) => {
+    evt.preventDefault();
+    evt.stopPropagation();
+
+    const {top, right} = maskButton.getBoundingClientRect();
+    if (!maskMap) {
+      maskMap = makeHTMLMap();
+      document.body.appendChild(maskMap);
+      maskMap.classList.add("hidden");
+    }
+    maskMap.classList.toggle("hidden");
+    if (!maskMap.classList.contains("hidden")) {
+      const maskRect = maskMap.getBoundingClientRect();
+      maskMap.style.top = `${top - maskRect.height - 10}px`;
+      maskMap.style.left = `${right - maskRect.width}px`;
+    }
+  });
+}

extensions/downthemall/lib/manager/scheduler.ts

@@ -0,0 +1,69 @@
+"use strict";
+// License: MIT
+
+import { QUEUED } from "./state";
+import { Limits } from "./limits";
+import { filterInSitu } from "../util";
+// eslint-disable-next-line no-unused-vars
+import { Download } from "./download";
+
+const REFILTER_COUNT = 50;
+
+function queuedFilter(d: Download) {
+  return d.state === QUEUED && !d.removed;
+}
+
+export class Scheduler {
+  private runCount: number;
+
+  private readonly queue: Download[];
+
+  constructor(queue: Download[]) {
+    this.queue = Array.from(queue).filter(queuedFilter);
+    this.runCount = 0;
+  }
+
+  async next(running: Iterable<Download>) {
+    if (!this.queue.length) {
+      return null;
+    }
+
+    if (this.runCount > REFILTER_COUNT) {
+      filterInSitu(this.queue, queuedFilter);
+      if (!this.queue.length) {
+        return null;
+      }
+    }
+
+    const hosts = Object.create(null);
+    for (const d of running) {
+      const {domain} = d.uURL;
+      if (domain in hosts) {
+        hosts[domain]++;
+      }
+      else {
+        hosts[domain] = 1;
+      }
+    }
+
+    await Limits.load();
+    for (const d of this.queue) {
+      if (d.state !== QUEUED || d.removed) {
+        continue;
+      }
+      const {domain} = d.uURL;
+      const limit = Limits.getConcurrentFor(domain);
+      const cur = hosts[domain] || 0;
+      if (limit <= cur) {
+        continue;
+      }
+      this.runCount++;
+      return d;
+    }
+    return null;
+  }
+
+  destroy() {
+    this.queue.length = 0;
+  }
+}

extensions/downthemall/lib/manager/state.ts

@@ -0,0 +1,16 @@
+"use strict";
+// License: MIT
+
+export const QUEUED = 1 << 0;
+export const RUNNING = 1 << 1;
+export const FINISHING = 1 << 2;
+export const PAUSED = 1 << 3;
+export const DONE = 1 << 4;
+export const CANCELED = 1 << 5;
+export const MISSING = 1 << 6;
+export const RETRYING = 1 << 7;
+
+export const RESUMABLE = PAUSED | CANCELED | RETRYING;
+export const FORCABLE = PAUSED | QUEUED | CANCELED | RETRYING;
+export const PAUSEABLE = QUEUED | CANCELED | RUNNING | RETRYING;
+export const CANCELABLE = QUEUED | RUNNING | PAUSED | DONE | MISSING | RETRYING;

extensions/downthemall/lib/memoize.ts

@@ -0,0 +1,143 @@
+"use strict";
+// License: MIT
+
+const DEFAULT_LIMIT = 3000;
+
+let memoes: any[] = [];
+
+export function filterCaches(c: any) {
+  if (!c) {
+    return false;
+  }
+  c.clear();
+  return true;
+}
+
+export function clearCaches() {
+  memoes = memoes.filter(filterCaches);
+}
+
+type MemoizeFun<T> = (...args: any[]) => T;
+
+/**
+ * Decorate a function with a memoization wrapper, with a limited-size cache
+ * to reduce peak memory utilization.
+ *
+ * The memoized function may have any number of arguments, but they must be
+ * be serializable.  It's safest to use this only on functions that accept
+ * primitives.
+ *
+ * A memoized function is not thread-safe, but so is JS,  nor re-entrant-safe!
+ *
+ * @param {Function} func The function to be memoized
+ * @param {Number} [limit] Optional. Cache size (default: 3000)
+ * @param {Number} [numArgs] Options. Number of arguments the function expects
+ * (default: func.length)
+ * @returns {Function} Memoized function
+ */
+export function memoize<T>(
+    func: MemoizeFun<T>, limit?: number, numArgs?: number): MemoizeFun<T> {
+  const climit = limit && limit > 0 ? limit : DEFAULT_LIMIT;
+  numArgs = numArgs || func.length;
+
+  const cache = new Map();
+  memoes.push(cache);
+  const keylist: any[] = [];
+  const args: any[] = [];
+  let key; let result;
+
+  switch (numArgs) {
+  case 0:
+    throw new Error("memoize does not support functions without arguments");
+
+  case 1:
+    return function memoizeOne(a: any) {
+      key = a.spec || a;
+      if (cache.has(key)) {
+        return cache.get(key);
+      }
+
+      result = func(a);
+      cache.set(key, result);
+      if (keylist.push(key) > climit) {
+        cache.delete(keylist.shift());
+      }
+      return result;
+    };
+
+  case 2:
+    return function memoizeTwo(a: any, b: any) {
+      args[0] = a; args[1] = b;
+      key = JSON.stringify(args);
+      args.length = 0;
+
+      if (cache.has(key)) {
+        return cache.get(key);
+      }
+
+      const result = func(a, b);
+      cache.set(key, result);
+      if (keylist.push(key) > climit) {
+        cache.delete(keylist.shift());
+      }
+      return result;
+    };
+
+  case 3:
+    return function memoizeThree(a: any, b: any, c: any) {
+      args[0] = a; args[1] = b; args[2] = c;
+      key = JSON.stringify(args);
+      args.length = 0;
+
+      if (cache.has(key)) {
+        return cache.get(key);
+      }
+
+      const result = func(a, b, c);
+      cache.set(key, result);
+      if (keylist.push(key) > climit) {
+        cache.delete(keylist.shift());
+      }
+      return result;
+    };
+
+  case 4:
+    return function memoizeFour(a: any, b: any, c: any, d: any) {
+      args[0] = a; args[1] = b; args[2] = c; args[3] = d;
+      key = JSON.stringify(args);
+      args.length = 0;
+
+      if (cache.has(key)) {
+        return cache.get(key);
+      }
+
+      const result = func(a, b, c, d);
+      cache.set(key, result);
+      if (keylist.push(key) > climit) {
+        cache.delete(keylist.shift());
+      }
+      return result;
+    };
+
+  default:
+    return function(...args: any[]) {
+      const key = JSON.stringify(args);
+
+      if (cache.has(key)) {
+        return cache.get(key);
+      }
+
+      const result = func(...args);
+      cache.set(key, result);
+      if (keylist.push(key) > climit) {
+        cache.delete(keylist.shift());
+      }
+      return result;
+    };
+  }
+}
+
+
+export const identity = memoize(function(o: any) {
+  return o;
+});

extensions/downthemall/lib/mime.ts

@@ -0,0 +1,65 @@
+"use strict";
+// License: MIT
+
+import mime from "../data/mime.json";
+
+export class MimeInfo {
+  public readonly type: string;
+
+  public readonly extensions: Set<string>;
+
+  public readonly major: string;
+
+  public readonly minor: string;
+
+  public readonly primary: string;
+
+  constructor(type: string, extensions: string[]) {
+    this.type = type;
+    const [major, minor] = type.split("/", 2);
+    this.major = major;
+    this.minor = minor;
+    [this.primary] = extensions;
+    this.extensions = new Set(extensions);
+    Object.freeze(this);
+  }
+}
+
+export const MimeDB = new class MimeDB {
+  private readonly mimeToExts: Map<string, MimeInfo>;
+
+  private readonly registeredExtensions: Set<string>;
+
+  constructor() {
+    const exts = new Map<string, string[]>();
+    for (const [prim, more] of Object.entries(mime.e)) {
+      let toadd = more;
+      if (!Array.isArray(toadd)) {
+        toadd = [toadd];
+      }
+      toadd.unshift(prim);
+      exts.set(prim, toadd);
+    }
+    this.mimeToExts = new Map(Array.from(
+      Object.entries(mime.m),
+      ([mime, prim]) => [mime, new MimeInfo(mime, exts.get(prim) || [prim])]
+    ));
+    const all = Array.from(
+      this.mimeToExts.values(),
+      m => Array.from(m.extensions, e => e.toLowerCase()));
+    this.registeredExtensions = new Set(all.flat());
+  }
+
+  getPrimary(mime: string) {
+    const info = this.mimeToExts.get(mime.trim().toLocaleLowerCase());
+    return info ? info.primary : "";
+  }
+
+  getMime(mime: string) {
+    return this.mimeToExts.get(mime.trim().toLocaleLowerCase());
+  }
+
+  hasExtension(ext: string) {
+    return this.registeredExtensions.has(ext.toLowerCase());
+  }
+}();

extensions/downthemall/lib/notifications.ts

@@ -0,0 +1,78 @@
+"use strict";
+// License: MIT
+
+import { extension, notifications } from "./browser";
+
+import {EventEmitter} from "./events";
+
+const DEFAULTS = {
+  type: "basic",
+  iconUrl: extension.getURL("/style/icon64.png"),
+  title: "DownThemAll!",
+  message: "message",
+};
+
+const TIMEOUT = 4000;
+
+let gid = 1;
+
+export class Notification extends EventEmitter {
+  private notification: any;
+
+  private readonly generated: boolean;
+
+  constructor(id: string | null, options = {}) {
+    super();
+
+    this.generated = !id;
+    id = id || `DownThemAll-notification${++gid}`;
+    if (typeof options === "string") {
+      options = {message: options};
+    }
+    options = Object.assign(Object.assign({}, DEFAULTS), options);
+
+    this.opened = this.opened.bind(this);
+    this.closed = this.closed.bind(this);
+    this.clicked = this.clicked.bind(this);
+
+    this.notification = notifications.create(id, options);
+
+    this.notification.then(this.opened).catch(console.error);
+    notifications.onClosed.addListener(this.closed);
+    notifications.onClicked.addListener(this.clicked);
+    notifications.onButtonClicked.addListener(this.clicked);
+  }
+
+  opened(notification: any) {
+    this.notification = notification;
+    this.emit("opened", this);
+    if (this.generated) {
+      setTimeout(() => {
+        notifications.clear(notification);
+      }, TIMEOUT);
+    }
+  }
+
+  clicked(notification: any, button?: number) {
+    // We can only be clicked, when we were opened, at which point the
+    // notification id is available
+    if (notification !== this.notification) {
+      return;
+    }
+    if (typeof button === "number") {
+      this.emit("button", this, button);
+      return;
+    }
+    this.emit("clicked", this);
+    console.log("clicked", notification);
+  }
+
+  async closed(notification: any) {
+    if (notification !== await this.notification) {
+      return;
+    }
+    notifications.onClosed.removeListener(this.closed);
+    notifications.onClicked.removeListener(this.clicked);
+    this.emit("closed", this);
+  }
+}

extensions/downthemall/lib/objectoverlay.ts

@@ -0,0 +1,103 @@
+"use strict";
+// License: MIT
+
+import { storage } from "./browser";
+
+function toJSON(overlay: any, object: any) {
+  const result: any = {};
+  for (const key of Object.keys(overlay)) {
+    const val: any = overlay[key];
+    if (val !== object[val]) {
+      result[key] = val;
+    }
+  }
+  return result;
+}
+
+function isOverridden(overlay: any, object: any, name: string) {
+  return name in overlay && name in object && overlay[name] !== object[name];
+}
+
+class Handler {
+  private readonly base: any;
+
+  constructor(base: any) {
+    this.base = base;
+  }
+
+  "has"(target: any, name: string) {
+    if (name === "toJSON") {
+      return true;
+    }
+    if (name === "isOverridden") {
+      return true;
+    }
+    return name in target || name in this.base;
+  }
+
+  "get"(target: any, name: string) {
+    if (name === "toJSON") {
+      return toJSON.bind(null, target, this.base);
+    }
+    if (name === "isOverridden") {
+      return isOverridden.bind(null, target, this.base);
+    }
+    if (name === "reset") {
+      return () => {
+        Object.keys(target).forEach(k => delete target[k]);
+      };
+    }
+    if (name in target) {
+      return target[name];
+    }
+    if (name in this.base) {
+      return this.base[name];
+    }
+    return null;
+  }
+
+  getOwnPropertyDescriptor(target: any, name: string) {
+    let res = Object.getOwnPropertyDescriptor(target, name);
+    if (!res) {
+      res = Object.getOwnPropertyDescriptor(this.base, name);
+    }
+    if (res) {
+      res.enumerable = res.writable = res.configurable = true;
+    }
+    return res;
+  }
+
+  "set"(target: any, name: string, value: any) {
+    target[name] = value;
+    return true;
+  }
+
+  ownKeys(target: any) {
+    const result = Object.keys(target);
+    result.push(...Object.keys(this.base));
+    return Array.from(new Set(result));
+  }
+}
+
+export function overlay(top: object) {
+  return new Proxy(top, new Handler(this));
+}
+
+export async function loadOverlay(
+    storageKey: string, sync: boolean, defaults: object) {
+  const bottom = Object.freeze(defaults);
+  const top = await storage[sync ? "sync" : "local"].get(storageKey);
+  return overlay.call(bottom, top[storageKey] || {});
+}
+
+export interface Overlayable {
+  overlay: Function;
+}
+
+Object.defineProperty(Object.prototype, "overlay", {
+  value: overlay
+});
+
+Object.defineProperty(Object, "loadOverlay", {
+  value: loadOverlay
+});

extensions/downthemall/lib/prefs.ts

@@ -0,0 +1,103 @@
+"use strict";
+// License: MIT
+
+import DEFAULT_PREFS from "../data/prefs.json";
+import { EventEmitter } from "./events";
+import { loadOverlay } from "./objectoverlay";
+import { storage } from "./browser";
+
+const PREFS = Symbol("PREFS");
+const PREF_STORAGE = "prefs";
+const TIMEOUT_SAVE = 100;
+
+export const Prefs = new class extends EventEmitter {
+  private [PREFS]: any;
+
+  private scheduled: any;
+
+  constructor() {
+    super();
+    this.save = this.save.bind(this);
+    this[PREFS] = loadOverlay(
+      PREF_STORAGE, false, DEFAULT_PREFS).then(r => {
+      storage.onChanged.addListener((changes: any, area: string) => {
+        if (area !== "local" || !("prefs" in changes)) {
+          return;
+        }
+        for (const [k, v] of Object.entries(changes.prefs.newValue)) {
+          if (JSON.stringify(r[k]) === JSON.stringify(v)) {
+            continue;
+          }
+          r[k] = v;
+          this.scheduleSave();
+          this.emit(k, this, k, v);
+        }
+      });
+      return this[PREFS] = r;
+    }).catch(ex => {
+      console.error("Failed to load prefs", ex.toString(), ex.stack);
+      this[PREFS] = null;
+      throw ex;
+    });
+  }
+
+  async "get"(key: string, defaultValue?: any) {
+    const prefs = await this[PREFS];
+    return prefs[key] || defaultValue;
+  }
+
+  *[Symbol.iterator]() {
+    yield *Object.keys(this[PREFS]);
+  }
+
+  async "set"(key: string, value: any) {
+    if (typeof key === "undefined" || typeof value === "undefined") {
+      throw Error("Tried to set undefined to a pref, probably a bug");
+    }
+    const prefs = await this[PREFS];
+    prefs[key] = value;
+    this.scheduleSave();
+    this.emit(key, this, key, value);
+  }
+
+  async reset(key: string) {
+    if (typeof key === "undefined") {
+      throw Error("Tried to set undefined to a pref, probably a bug");
+    }
+    const prefs = await this[PREFS];
+    delete prefs[key];
+    this.scheduleSave();
+    this.emit(key, this, key, prefs[key]);
+  }
+
+  scheduleSave() {
+    if (this.scheduled) {
+      return;
+    }
+    this.scheduled = setTimeout(this.save, TIMEOUT_SAVE);
+  }
+
+  async save() {
+    this.scheduled = 0;
+    const prefs = (await this[PREFS]).toJSON();
+    await storage.local.set({prefs});
+  }
+}();
+
+export class PrefWatcher {
+  public readonly name: string;
+
+  public value: any;
+
+  constructor(name: string, defaultValue?: any) {
+    this.name = name;
+    this.value = defaultValue;
+    this.changed = this.changed.bind(this);
+    Prefs.on(name, this.changed);
+    Prefs.get(name, defaultValue).then(val => this.changed(Prefs, name, val));
+  }
+
+  changed(prefs: any, key: string, value: any) {
+    this.value = value;
+  }
+}

extensions/downthemall/lib/pserializer.ts

@@ -0,0 +1,131 @@
+"use strict";
+// License: MIT
+
+const RUNNING = Symbol();
+const LIMIT = Symbol();
+const ITEMS = Symbol();
+
+function nothing() { /* ignored */ }
+
+type Wrapped<T> = (...args: any[]) => Promise<T>;
+
+interface Item {
+  readonly ctx: any;
+  readonly fn: Function;
+  readonly args: any[];
+  readonly resolve: Function;
+  readonly reject: Function;
+}
+
+function scheduleDirect<T>(ctx: any, fn: Function, ...args: any[]): Promise<T> {
+  try {
+    const p = Promise.resolve(fn.call(ctx, ...args));
+    this[RUNNING]++;
+    p.finally(this._next).catch(nothing);
+    return p;
+  }
+  catch (ex) {
+    return Promise.reject(ex);
+  }
+}
+
+function scheduleForLater<T>(
+    head: boolean, ctx: any, fn: Function, ...args: any[]): Promise<T> {
+  const rv = new Promise<T>((resolve, reject) => {
+    const item = { ctx, fn, args, resolve, reject };
+    this[ITEMS][head ? "unshift" : "push"](item);
+  });
+  return rv;
+}
+
+function scheduleInternal<T>(
+    head: boolean, ctx: any, fn: Function, ...args: any[]): Promise<T> {
+  if (this[RUNNING] < this.limit) {
+    return scheduleDirect.call(this, ctx, fn, ...args);
+  }
+  return scheduleForLater.call(this, head, ctx, fn, ...args);
+}
+
+export class PromiseSerializer {
+  private [LIMIT]: number;
+
+  private [RUNNING]: number;
+
+  private [ITEMS]: Item[];
+
+  private readonly _next: () => void;
+
+  constructor(limit: number) {
+    this[LIMIT] = Math.max(limit || 5, 1);
+    this[ITEMS] = [];
+    this[RUNNING] = 0;
+    this._next = this.next.bind(this);
+    Object.seal(this);
+  }
+
+  get limit() {
+    return this[LIMIT];
+  }
+
+  get running() {
+    return this[RUNNING];
+  }
+
+  get scheduled() {
+    return this[ITEMS].length;
+  }
+
+  get total() {
+    return this.scheduled + this.running;
+  }
+
+  static wrapNew<T>(limit: number, ctx: any, fn: Function): Wrapped<T> {
+    return new PromiseSerializer(limit).wrap(ctx, fn);
+  }
+
+  wrap<T>(ctx: any, fn: Function): Wrapped<T> {
+    const rv = this.scheduleWithContext.bind(this, ctx, fn);
+    Object.defineProperty(rv, "prepend", {
+      value: this.prependWithContext.bind(this, ctx, fn)
+    });
+    return rv;
+  }
+
+  schedule<T>(fn: Wrapped<T>, ...args: any[]): Promise<T> {
+    return this.scheduleWithContext(null, fn, ...args);
+  }
+
+  scheduleWithContext<T>(ctx: any, fn: Wrapped<T>, ...args: any[]): Promise<T> {
+    return scheduleInternal.call(this, false, ctx, fn, ...args);
+  }
+
+  prepend<T>(fn: Wrapped<T>, ...args: any[]): Promise<T> {
+    return this.prependWithContext(null, fn, ...args);
+  }
+
+  prependWithContext<T>(ctx: any, fn: Wrapped<T>, ...args: any[]): Promise<T> {
+    return scheduleInternal.call(this, true, ctx, fn, ...args);
+  }
+
+  next() {
+    this[RUNNING]--;
+    const item = this[ITEMS].shift();
+    if (!item) {
+      return;
+    }
+    try {
+      const p = Promise.resolve(item.fn.call(item.ctx, ...item.args));
+      this[RUNNING]++;
+      item.resolve(p);
+      p.finally(this._next).catch(nothing);
+    }
+    catch (ex) {
+      try {
+        item.reject(ex);
+      }
+      finally {
+        this.next();
+      }
+    }
+  }
+}

extensions/downthemall/lib/recentlist.ts

@@ -0,0 +1,124 @@
+"use strict";
+// License: MIT
+
+import { none } from "./util";
+import { storage } from "./browser";
+
+const LIST = Symbol("saved-list");
+
+function unique(e: string) {
+  if (typeof e !== "string" || this.has(e)) {
+    return false;
+  }
+  this.add(e);
+  return true;
+}
+
+export class RecentList {
+  public readonly pref: string;
+
+  public readonly defaults: string[];
+
+  public readonly limit: number;
+
+  private _inited: any;
+
+  private [LIST]: string[];
+
+  constructor(pref: string, defaults: string[] = []) {
+    if (!pref) {
+      throw Error("Invalid pref");
+    }
+    defaults = defaults || [];
+    if (!Array.isArray(defaults)) {
+      throw new Error("Invalid defaults");
+    }
+    this.pref = `savedlist-${pref}`;
+    this.defaults = Array.from(defaults);
+    this[LIST] = [];
+    this.limit = 15;
+  }
+
+  get values() {
+    return Array.from(this[LIST]);
+  }
+
+  get current() {
+    return this[LIST][0] || "";
+  }
+
+  async _init() {
+    const {[this.pref]: saved = []} = await storage.local.get(this.pref) || [];
+    this[LIST] = [...saved, ...this.defaults].
+      filter(unique, new Set()).
+      slice(0, this.limit);
+  }
+
+  init() {
+    if (!this._inited) {
+      this._inited = this._init();
+      this._inited.then(() => {
+        this.init = none;
+      });
+    }
+    return this._inited;
+  }
+
+  async reset() {
+    this[LIST] = Array.from(this.defaults);
+    await this.save();
+  }
+
+  async push(value: string) {
+    if (value === null || typeof value === "undefined") {
+      throw new Error("Invalid value");
+    }
+    const list = this[LIST];
+    const idx = list.indexOf(value);
+    if (idx === 0) {
+      return;
+    }
+    if (idx > 0) {
+      list.splice(idx, 1);
+    }
+    list.unshift(value);
+    while (list.length > 10) {
+      list.pop();
+    }
+    await this.save();
+    return;
+  }
+
+  async save() {
+    await storage.local.set({[this.pref]: this[LIST]});
+  }
+
+  *[Symbol.iterator]() {
+    yield *this[LIST];
+  }
+}
+
+export const MASK = new RecentList("mask", [
+  "*name*.*ext*",
+  "*num*_*name*.*ext*",
+  "*url*-*name*.*ext*",
+  "downthemall/*y*-*m*/*name*.*ext*",
+  "*name* (*text*).*ext*"
+]);
+MASK.init().catch(console.error);
+
+export const FASTFILTER = new RecentList("fastfilter", [
+  "",
+  "/\\.mp3$/",
+  "/\\.(html|htm|rtf|doc|pdf)$/",
+  "http://www.website.com/subdir/*.*",
+  "http://www.website.com/subdir/pre*.???",
+  "*.z??, *.css, *.html"
+]);
+FASTFILTER.init().catch(console.error);
+
+export const SUBFOLDER = new RecentList("subfolder", [
+  "",
+  "downthemall",
+]);
+SUBFOLDER.init().catch(console.error);

extensions/downthemall/lib/select.ts

@@ -0,0 +1,239 @@
+"use strict";
+// License: MIT
+
+// eslint-disable-next-line no-unused-vars
+import { Bus, Port } from "./bus";
+import { Prefs } from "./prefs";
+import { Promised, timeout } from "./util";
+import { donate, openPrefs, openUrls } from "./windowutils";
+// eslint-disable-next-line no-unused-vars
+import { filters, FAST, Filter } from "./filters";
+import { WindowStateTracker } from "./windowstatetracker";
+import { windows, CHROME } from "./browser";
+// eslint-disable-next-line no-unused-vars
+import { BaseItem } from "./item";
+
+interface BaseMatchedItem extends BaseItem {
+  sidx?: number;
+  matched?: string | null;
+  prevMatched?: string | null;
+}
+
+export interface ItemDelta {
+  idx: number;
+  matched?: string | null;
+}
+
+function computeSelection(
+    filters: Filter[],
+    items: BaseMatchedItem[],
+    onlyFast: boolean): ItemDelta[] {
+  let ws = items.map((item, idx: number) => {
+    item.idx = item.idx || idx;
+    item.sidx = item.sidx || idx;
+    const {matched = null} = item;
+    item.prevMatched = matched;
+    item.matched = null;
+    return item;
+  });
+  for (const filter of filters) {
+    ws = ws.filter(item => {
+      if (filter.matchItem(item)) {
+        if (filter.id === FAST) {
+          item.matched = "fast";
+        }
+        else if (!onlyFast && typeof filter.id === "string") {
+          item.matched = filter.id;
+        }
+        else {
+          item.matched = null;
+        }
+      }
+      return !item.matched;
+    });
+  }
+  return items.filter(item => item.prevMatched !== item.matched).map(item => {
+    return {
+      idx: item.sidx as number,
+      matched: item.matched
+    };
+  });
+}
+
+function *computeActiveFiltersGen(
+    filters: Filter[], activeOverrides: Map<string, boolean>) {
+  for (const filter of filters) {
+    if (typeof filter.id !== "string") {
+      continue;
+    }
+    const override = activeOverrides.get(filter.id);
+    if (typeof override === "boolean") {
+      if (override) {
+        yield filter;
+      }
+      continue;
+    }
+    if (filter.active) {
+      yield filter;
+    }
+  }
+}
+
+function computeActiveFilters(
+    filters: Filter[], activeOverrides: Map<string, boolean>) {
+  return Array.from(computeActiveFiltersGen(filters, activeOverrides));
+}
+
+function filtersToDescs(filters: Filter[]) {
+  return filters.map(f => f.descriptor);
+}
+
+export async function select(links: BaseItem[], media: BaseItem[]) {
+  const fm = await filters();
+  const tracker = new WindowStateTracker("select", {
+    minWidth: 700,
+    minHeight: 500,
+  });
+  await tracker.init();
+  const windowOptions = tracker.getOptions({
+    url: "/windows/select.html",
+    type: "popup",
+  });
+  const window = await windows.create(windowOptions);
+  tracker.track(window.id);
+  try {
+    if (!CHROME) {
+      windows.update(window.id, tracker.getOptions({}));
+    }
+    const port = await Promise.race<Port>([
+      new Promise<Port>(resolve => Bus.oncePort("select", port => {
+        resolve(port);
+        return true;
+      })),
+      timeout<Port>(5 * 1000)]);
+    if (!port.isSelf) {
+      throw Error("Invalid sender connected");
+    }
+    tracker.track(window.id, port);
+
+    const overrides = new Map();
+    let fast: Filter | null = null;
+    let onlyFast: false;
+    try {
+      fast = await fm.getFastFilter();
+    }
+    catch (ex) {
+      // ignored
+    }
+
+    const sendFilters = function(delta = false) {
+      const {linkFilters, mediaFilters} = fm;
+      const alink = computeActiveFilters(linkFilters, overrides);
+      const amedia = computeActiveFilters(mediaFilters, overrides);
+      const sactiveFilters = new Set<any>();
+      [alink, amedia].forEach(
+        a => a.forEach(filter => sactiveFilters.add(filter.id)));
+      const activeFilters = Array.from(sactiveFilters);
+      const linkFilterDescs = filtersToDescs(linkFilters);
+      const mediaFilterDescs = filtersToDescs(mediaFilters);
+      port.post("filters", {linkFilterDescs, mediaFilterDescs, activeFilters});
+
+      if (fast) {
+        alink.unshift(fast);
+        amedia.unshift(fast);
+      }
+      const deltaLinks = computeSelection(alink, links, onlyFast);
+      const deltaMedia = computeSelection(amedia, media, onlyFast);
+      if (delta) {
+        port.post("item-delta", {deltaLinks, deltaMedia});
+      }
+    };
+
+    const done = new Promised();
+
+    port.on("disconnect", () => {
+      done.reject(new Error("Prematurely disconnected"));
+    });
+
+    port.on("cancel", () => {
+      done.reject(new Error("User canceled"));
+    });
+
+    port.on("queue", (msg: any) => {
+      done.resolve(msg);
+    });
+
+    port.on("filter-changed", (spec: any) => {
+      overrides.set(spec.id, spec.value);
+      sendFilters(true);
+    });
+
+    port.on("fast-filter", ({fastFilter}) => {
+      if (fastFilter) {
+        try {
+          fast = fm.getFastFilterFor(fastFilter);
+        }
+        catch (ex) {
+          console.error(ex);
+          fast = null;
+        }
+      }
+      else {
+        fast = null;
+      }
+      sendFilters(true);
+    });
+    port.on("onlyfast", ({fast}) => {
+      onlyFast = fast;
+      sendFilters(true);
+    });
+
+    port.on("donate", () => {
+      donate();
+    });
+    port.on("prefs", () => {
+      openPrefs();
+    });
+
+    port.on("openUrls", ({urls, incognito}) => {
+      openUrls(urls, incognito);
+    });
+
+    try {
+      fm.on("changed", () => sendFilters(true));
+      sendFilters(false);
+      const type = await Prefs.get("last-type", "links");
+      port.post("items", {type, links, media});
+      const {items, options} = await done;
+      const selectedIndexes = new Set<number>(items);
+      const selectedList = (options.type === "links" ? links : media);
+      const selectedItems = selectedList.filter(
+        (item: BaseItem, idx: number) => selectedIndexes.has(idx));
+      for (const [filter, override] of overrides) {
+        const f = fm.get(filter);
+        if (f) {
+          f.active = override;
+        }
+      }
+      await fm.save();
+      return {items: selectedItems, options};
+    }
+    finally {
+      fm.off("changed", sendFilters);
+    }
+  }
+  finally {
+    try {
+      await tracker.finalize();
+    }
+    catch (ex) {
+      // window might be gone; ignored
+    }
+    try {
+      await windows.remove(window.id);
+    }
+    catch (ex) {
+      // window might be gone; ignored
+    }
+  }
+}

extensions/downthemall/lib/single.ts

@@ -0,0 +1,77 @@
+"use strict";
+// License: MIT
+
+// eslint-disable-next-line no-unused-vars
+import { Bus, Port } from "./bus";
+import { WindowStateTracker } from "./windowstatetracker";
+import { Promised, timeout } from "./util";
+import { donate } from "./windowutils";
+import { windows, CHROME } from "./browser";
+// eslint-disable-next-line no-unused-vars
+import { BaseItem } from "./item";
+
+export async function single(item: BaseItem | null) {
+  const tracker = new WindowStateTracker("single", {
+    minWidth: 700,
+    minHeight: 460
+  });
+  await tracker.init();
+  const windowOptions = tracker.getOptions({
+    url: "/windows/single.html",
+    type: "popup",
+  });
+  const window = await windows.create(windowOptions);
+  tracker.track(window.id);
+  try {
+    if (!CHROME) {
+      windows.update(window.id, tracker.getOptions({}));
+    }
+    const port: Port = await Promise.race<Port>([
+      new Promise<Port>(resolve => Bus.oncePort("single", port => {
+        resolve(port);
+        return true;
+      })),
+      timeout<Port>(5 * 1000)]);
+    if (!port.isSelf) {
+      throw Error("Invalid sender connected");
+    }
+    tracker.track(window.id, port);
+
+    const done = new Promised();
+
+    port.on("disconnect", () => {
+      done.reject(new Error("Prematurely disconnected"));
+    });
+
+    port.on("queue", msg => {
+      done.resolve(msg);
+    });
+
+    port.on("cancel", () => {
+      done.reject(new Error("User canceled"));
+    });
+
+    port.on("donate", () => {
+      donate();
+    });
+
+    if (item) {
+      port.post("item", {item});
+    }
+    return await done;
+  }
+  finally {
+    try {
+      await tracker.finalize();
+    }
+    catch (ex) {
+      // window might be gone; ignored
+    }
+    try {
+      await windows.remove(window.id);
+    }
+    catch (ex) {
+      // window might be gone; ignored
+    }
+  }
+}

extensions/downthemall/lib/sorting.ts

@@ -0,0 +1,213 @@
+"use strict";
+// License: MIT
+
+const RE_TOKENIZE = /(0x[0-9a-f]+|[+-]?[0-9]+(?:\.[0-9]*(?:e[+-]?[0-9]+)?)?|\d+)/i;
+const RE_HEX = /^0x[0-9a-z]+$/i;
+const RE_TRIMMORE = /\s+/g;
+
+type KeyFunc<T> = (v: T) => any;
+type CompareFunc<T> = (a: T, b: T) => number;
+
+/**
+ * Compare two values using the usual less-than-greater-than rules
+ * @param {*} a First value
+ * @param {*} b Second value
+ * @returns {number} Comparision result
+ */
+export function defaultCompare(a: any, b: any) {
+  return a < b ? -1 : (a > b ? 1 : 0);
+}
+
+
+function parseToken(chunk: string) {
+  chunk = chunk.replace(RE_TRIMMORE, " ").trim();
+  if (RE_HEX.test(chunk)) {
+    return parseInt(chunk.slice(2), 16);
+  }
+  const val = parseFloat(chunk);
+  return Number.isNaN(val) ? chunk : val;
+}
+
+function filterTokens(str: string) {
+  return str && str.trim();
+}
+
+function tokenize(val: any) {
+  if (typeof val === "number") {
+    return [[`${val}`], [val]];
+  }
+  const tokens = `${val}`.split(RE_TOKENIZE).filter(filterTokens);
+  const numeric = tokens.map(parseToken);
+  return [tokens, numeric];
+}
+
+
+/**
+ * Natural Sort algorithm for es6
+ * @param {*} a First term
+ * @param {*} b Second term
+ * @returns {Number} Comparison result
+ */
+export function naturalCompare(a: any, b: any): number {
+  const [xTokens, xNumeric] = tokenize(a);
+  const [yTokens, yNumeric] = tokenize(b);
+
+  // natural sorting through split numeric strings and default strings
+  const {length: xTokenLen} = xTokens;
+  const {length: yTokenLen} = yTokens;
+  const maxLen = Math.min(xTokenLen, yTokenLen);
+  for (let i = 0; i < maxLen; ++i) {
+    // find floats not starting with '0', string or 0 if not defined
+    const xnum = xNumeric[i];
+    const ynum = yNumeric[i];
+    const xtype = typeof xnum;
+    const xisnum = xtype === "number";
+    const ytype = typeof ynum;
+    const sameType = xtype === ytype;
+    if (!sameType) {
+      // Proper numbers go first.
+      // We already checked sameType above, so we know only one is a number.
+      return xisnum ? -1 : 1;
+    }
+
+    // sametype follows...
+    if (xisnum) {
+      // both are numbers
+      // Compare the numbers and if they are the same, the tokens too
+      const res = defaultCompare(xnum, ynum) ||
+          defaultCompare(xTokens[i], yTokens[i]);
+      if (!res) {
+        continue;
+      }
+      return res;
+    }
+
+    // both must be stringey
+    // Compare the actual tokens.
+    const res = defaultCompare(xTokens[i], yTokens[i]);
+    if (!res) {
+      continue;
+    }
+    return res;
+  }
+  return defaultCompare(xTokenLen, yTokenLen);
+}
+
+/**
+ * Natural Sort algorithm for es6, case-insensitive version
+ * @param {*} a First term
+ * @param {*} b Second term
+ * @returns {Number} Comparison result
+ */
+export function naturalCaseCompare(a: any, b: any) {
+  return naturalCompare(`${a}`.toUpperCase(), `${b}`.toUpperCase());
+}
+
+/**
+ * Array-enabled compare: If both operands are an array, compare individual
+ * elements up to the length of the smaller array. If all elements match,
+ * consider the array with fewer items smaller
+ * @param {*} a First item to compare (either PoD or Array)
+ * @param {*} b Second item to compare (either PoD or Array)
+ * @param {cmpf} [cmp] Compare function or default_compare
+ * @returns {number} Comparison result
+ */
+export function arrayCompare(a: any, b: any, cmp: CompareFunc<any>): number {
+  cmp = cmp || defaultCompare;
+  if (Array.isArray(a) && Array.isArray(b)) {
+    const {length: alen} = a;
+    const {length: blen} = b;
+    const len = Math.min(alen, blen);
+    for (let i = 0; i < len; ++i) {
+      const rv = arrayCompare(a[i], b[i], cmp);
+      if (rv) {
+        return rv;
+      }
+    }
+    return defaultCompare(alen, blen);
+  }
+  return cmp(a, b);
+}
+
+interface MapValue {
+  key: any;
+  index: number;
+  value: any;
+}
+
+function mappedCompare(
+    fn: CompareFunc<any>, a: MapValue, b: MapValue): number {
+  const {key: ka} = a;
+  const {key: kb} = b;
+  return arrayCompare(ka, kb, fn) ||
+  /* stable */ defaultCompare(a.index, b.index);
+}
+
+/**
+ * Tranform a given value into a key for sorting. Keys can be either PoDs or
+ * an array of PoDs.
+ * @callback keyfn
+ * @param {*} item Array item to map
+ * @returns {*} Key for sorting
+ */
+
+/**
+ * Compare to items with each other, returning <0, 0, >0.
+ * @callback cmpfn
+ * @param {*} item Array item to map
+ * @returns {number} Comparision result
+ */
+
+/**
+ * Sort an array by a given key function and comparision function.
+ * This sort is stable, but and in-situ
+ * @param {*[]} arr Array to be sorted
+ * @param {keyfn} [key] How to make keys. If ommitted, use value as key.
+ * @param {cmpfn} [cmp] How to compare keys. If omitted, use default cmp.
+ * @returns {*[]} New sorted array
+ */
+export function sort<T>(arr: T[], key?: KeyFunc<T>, cmp?: CompareFunc<T>) {
+  cmp = cmp || defaultCompare;
+  const carr = arr as unknown as MapValue[];
+  if (key) {
+    arr.forEach((value, index) => {
+      carr[index] = {value, key: key(value), index};
+    });
+  }
+  else {
+    arr.forEach((value, index) => {
+      carr[index] = {value, key: value, index};
+    });
+  }
+  arr.sort(mappedCompare.bind(null, cmp));
+  carr.forEach((i, idx) => {
+    arr[idx] = i.value;
+  });
+  return arr;
+}
+
+/**
+ * Sort an array by a given key function and comparision function.
+ * This sort is stable, but NOT in-situ, it will rather leave the
+ * original array untoched and return a sorted copy.
+ * @param {*[]} arr Array to be sorted
+ * @param {keyfn} [key] How to make keys. If ommitted, use value as key.
+ * @param {cmpfn} [cmp] How to compare keys. If omitted, use default cmp.
+ * @returns {*[]} New sorted array
+ */
+export function sorted<T>(arr: T[], key?: KeyFunc<T>, cmp?: CompareFunc<T>) {
+  cmp = cmp || defaultCompare;
+  let carr: MapValue[];
+  if (key) {
+    carr = arr.map((value, index) => {
+      return {value, key: key(value), index};
+    });
+  }
+  else {
+    carr = arr.map((value, index) => {
+      return {value, key: value, index};
+    });
+  }
+  carr.sort(mappedCompare.bind(null, cmp));
+  return carr.map(v => v.value);
+}

extensions/downthemall/lib/textlinks.ts

@@ -0,0 +1,112 @@
+"use strict";
+// License: MIT
+
+import { textlinks as rtextlinks } from "../data/xregexps.json";
+
+const SCHEME_DEFAULT = "https";
+
+// Link matcher
+const regLinks = new RegExp(rtextlinks.source, rtextlinks.flags);
+// Match more exactly or more than 3 dots.
+// Links are then assumed "cropped" and will be ignored.
+const regShortened = /\.{3,}/;
+// http cleanup
+const regHttp = /^h(?:x+|tt)?p(s?)/i;
+// ftp cleanup
+const regFtp = /^f(?:x+|t)p/i;
+// www (sans protocol) match
+const regWWW = /^www/i;
+// Right-trim (sanitize) link
+const regDTrim = /[<>._-]+$|#.*?$/g;
+
+function mapper(e: string) {
+  try {
+    if (regShortened.test(e)) {
+      return null;
+    }
+    if (regWWW.test(e)) {
+      if (e.indexOf("/") < 0) {
+        e = `${SCHEME_DEFAULT}://${e}/`;
+      }
+      else {
+        e = `${SCHEME_DEFAULT}://${e}`;
+      }
+    }
+    return e.replace(regHttp, "http$1").
+      replace(regFtp, "ftp").
+      replace(regDTrim, "");
+  }
+  catch (ex) {
+    return null;
+  }
+}
+
+/**
+ * Minimal Link representation (partially) implementing DOMElement
+ *
+ * @param {string} url URL (href) of the Links
+ * @param {string} title Optional. Title/description
+ * @see DOMElement
+ */
+export class FakeLink {
+  public readonly src: string;
+
+  public readonly href: string;
+
+  public readonly title: string;
+
+  public readonly fake: boolean;
+
+  public childNodes: readonly Node[];
+
+  constructor (url: string, title?: string) {
+    this.src = this.href = url;
+    if (title) {
+      this.title = title;
+    }
+    this.fake = true;
+    Object.freeze(this);
+  }
+
+  hasAttribute(attr: string) {
+    return (attr in this);
+  }
+
+  getAttribute(attr: string) {
+    // eslint-disable-next-line @typescript-eslint/no-this-alias
+    const self: any = this;
+    return (attr in self) ? self[attr] : null;
+  }
+
+  toString() {
+    return this.href;
+  }
+}
+
+FakeLink.prototype.childNodes = Object.freeze([]);
+
+/**
+ * Parses a text looking for any URLs with supported protocols
+ *
+ * @param {string} text Text to parse
+ * @param {boolean} [fakeLinks]
+ *   Whether an array of plain text links will be returned or
+ *   an array of FakeLinks.
+ * @returns {string[]} results
+ */
+export function getTextLinks(text: string, fakeLinks = false) {
+  const rv: any = text.match(regLinks);
+  if (!rv) {
+    return [];
+  }
+  let i; let k; let e;
+  for (i = 0, k = 0, e = rv.length; i < e; i++) {
+    const a = mapper(rv[i]);
+    if (a) {
+      rv[k] = fakeLinks ? new FakeLink(a) : a;
+      k += 1;
+    }
+  }
+  rv.length = k; // truncate
+  return rv;
+}

extensions/downthemall/lib/util.ts

@@ -0,0 +1,380 @@
+"use strict";
+// License: MIT
+
+import * as psl from "psl";
+import { identity, memoize } from "./memoize";
+import { IPReg } from "./ipreg";
+export { debounce } from "../uikit/lib/util";
+
+export class Promised {
+  private promise: Promise<any>;
+
+  resolve: (value?: any) => void;
+
+  reject: (reason?: any) => void;
+
+  constructor() {
+    this.promise = new Promise((resolve, reject) => {
+      this.resolve = resolve;
+      this.reject = reject;
+    });
+  }
+
+  then(
+      resolve: (value: any) => any,
+      reject: (reason: any) => PromiseLike<never>) {
+    return this.promise.then(resolve).catch(reject);
+  }
+}
+
+export function timeout<T>(to: number) {
+  return new Promise<T>((resolve, reject) => {
+    setTimeout(() => reject(new Error("timeout")), to);
+  });
+}
+
+export function lazy<T>(object: any, name: string, fun: (...any: any[]) => T) {
+  Object.defineProperty(object, name, {
+    get() {
+      const value = fun();
+      Object.defineProperty(object, name, {
+        value,
+        enumerable: true, writable: true, configurable: true
+      });
+      return value;
+    },
+    enumerable: true, configurable: true
+  });
+  return object;
+}
+
+export function none() { /* ignored */ }
+
+export function sanitizePathGeneric(path: string) {
+  return path.
+    replace(/:+/g, "ː").
+    replace(/\?+/g, "_").
+    replace(/\*+/g, "_").
+    replace(/<+/g, "◄").
+    replace(/>+/g, "▶").
+    replace(/"+/g, "'").
+    replace(/\|+/g, "¦").
+    replace(/#+/g, "♯").
+    replace(/[.\s]+$/g, "").
+    trim();
+}
+
+const REG_TRIMMORE = /^[\s.]+|[\s.]+$/g;
+const REG_RESERVED = new RegExp(
+  `^(?:${"CON, PRN, AUX, NUL, COM1, COM2, COM3, COM4, COM5, COM6, COM7, COM8, COM9, LPT1, LPT2, LPT3, LPT4, LPT5, LPT6, LPT7, LPT8, LPT9".split(", ").join("|")})(?:\\..*)$`,
+  "i");
+
+export function sanitizePathWindows(path: string) {
+  path = path.
+    replace(/:+/g, "ː").
+    replace(/\?+/g, "_").
+    replace(/\*+/g, "_").
+    replace(/<+/g, "◄").
+    replace(/>+/g, "▶").
+    replace(/"+/g, "'").
+    replace(/\|+/g, "¦").
+    replace(/#+/g, "♯").
+    replace(/[.\s]+$/g, "").
+    replace(REG_TRIMMORE, "");
+  // Legacy file names
+  if (REG_RESERVED.test(path)) {
+    path = `!${path}`;
+  }
+  return path;
+}
+
+// Cannot use browser.runtime here
+export const IS_WIN = typeof navigator !== "undefined" &&
+  navigator.platform &&
+  navigator.platform.includes("Win");
+
+
+export const sanitizePath = identity(
+  IS_WIN ? sanitizePathWindows : sanitizePathGeneric);
+
+export class PathInfo {
+  private baseField: string;
+
+  private extField: string;
+
+  private pathField: string;
+
+  private nameField: string;
+
+  private fullField: string;
+
+  constructor(base: string, ext: string, path: string) {
+    this.baseField = base;
+    this.extField = ext;
+    this.pathField = path;
+    this.update();
+  }
+
+  get base() {
+    return this.baseField;
+  }
+
+  set base(nv) {
+    this.baseField = sanitizePath(nv);
+    this.update();
+  }
+
+  get ext() {
+    return this.extField;
+  }
+
+  set ext(nv) {
+    this.extField = sanitizePath(nv);
+    this.update();
+  }
+
+  get name() {
+    return this.nameField;
+  }
+
+  get path() {
+    return this.pathField;
+  }
+
+  set path(nv) {
+    this.pathField = sanitizePath(nv);
+    this.update();
+  }
+
+  get full() {
+    return this.fullField;
+  }
+
+  private update() {
+    this.nameField = this.extField ? `${this.baseField}.${this.extField}` : this.baseField;
+    this.fullField = this.pathField ? `${this.pathField}/${this.nameField}` : this.nameField;
+  }
+
+  clone() {
+    return new PathInfo(this.baseField, this.extField, this.pathField);
+  }
+}
+
+// XXX cleanup + test
+export const parsePath = memoize(function parsePath(
+    path: string | URL): PathInfo {
+  if (path instanceof URL) {
+    path = decodeURIComponent(path.pathname);
+  }
+  path = path.trim().replace(/\\/g, "/");
+  const pieces = path.split("/").
+    map((e: string) => sanitizePath(e)).
+    filter((e: string) => e && e !== ".");
+
+  const name = path.endsWith("/") ? "" : pieces.pop() || "";
+  const idx = name.lastIndexOf(".");
+  let base = name;
+  let ext = "";
+  if (idx >= 0) {
+    base = sanitizePath(name.substr(0, idx));
+    ext = sanitizePath(name.substr(idx + 1));
+  }
+
+  for (let i = 0; i < pieces.length;) {
+    if (pieces[i] !== "..") {
+      ++i;
+      continue;
+    }
+    if (i === 0) {
+      throw Error("Invalid traversal");
+    }
+    pieces.slice(i - 1, 2);
+  }
+
+  path = pieces.join("/");
+  return new PathInfo(base, ext, path);
+});
+
+export class CoalescedUpdate<T> extends Set<T> {
+  private readonly to: number;
+
+  private readonly cb: Function;
+
+  private triggerTimer: any;
+
+  constructor(to: number, cb: Function) {
+    super();
+    this.to = to;
+    this.cb = cb;
+    this.triggerTimer = 0;
+    this.trigger = this.trigger.bind(this);
+    Object.seal(this);
+  }
+
+  add(s: T) {
+    if (!this.triggerTimer) {
+      this.triggerTimer = setTimeout(this.trigger, this.to);
+    }
+    return super.add(s);
+  }
+
+  trigger() {
+    this.triggerTimer = 0;
+    if (!this.size) {
+      return;
+    }
+    const a = Array.from(this);
+    this.clear();
+    this.cb(a);
+  }
+}
+
+export const hostToDomain = memoize(psl.get, 1000);
+
+export interface URLd extends URL {
+  domain: string;
+}
+
+Object.defineProperty(URL.prototype, "domain", {
+  get() {
+    try {
+      const {hostname} = this;
+      return IPReg.test(hostname) ?
+        hostname :
+        hostToDomain(hostname) || hostname;
+    }
+    catch (ex) {
+      console.error(ex);
+      return this.host;
+    }
+  },
+  enumerable: true,
+  configurable: false,
+});
+
+/**
+ * Filter arrays in-situ. Like Array.filter, but in place
+ *
+ * @param {Array} arr
+ * @param {Function} cb
+ * @param {Object} tp
+ * @returns {Array} Filtered array (identity)
+ */
+export function filterInSitu<T>(
+    arr: (T | null | undefined)[], cb: (value: T) => boolean, tp?: any) {
+  tp = tp || null;
+  let i; let k; let e;
+  const carr = arr as unknown as T[];
+  for (i = 0, k = 0, e = arr.length; i < e; i++) {
+    const a = arr[i]; // replace filtered items
+    if (!a) {
+      continue;
+    }
+    if (cb.call(tp, a, i, arr)) {
+      carr[k] = a;
+      k += 1;
+    }
+  }
+  carr.length = k; // truncate
+  return carr;
+}
+
+/**
+ * Map arrays in-situ. Like Array.map, but in place.
+ * @param {Array} arr
+ * @param {Function} cb
+ * @param {Object} tp
+ * @returns {Array} Mapped array (identity)
+ */
+export function mapInSitu<TRes, T>(arr: T[], cb: (value: T) => TRes, tp?: any) {
+  tp = tp || null;
+  const carr = arr as unknown as TRes[];
+
+  for (let i = 0, e = arr.length; i < e; i++) {
+    carr[i] = cb.call(tp, arr[i], i, arr);
+  }
+  return carr;
+}
+
+/**
+ * Filters and then maps an array in-situ
+ * @param {Array} arr
+ * @param {Function} filterStep
+ * @param {Function} mapStep
+ * @param {Object} tp
+ * @returns {Array} Filtered and mapped array (identity)
+ */
+export function filterMapInSitu<TRes, T>(
+    arr: T[],
+    filterStep: (value: T) => boolean,
+    mapStep: (value: T) => TRes,
+    tp?: any) {
+  tp = tp || null;
+  const carr = arr as unknown as TRes[];
+
+  let i; let k; let e;
+  for (i = 0, k = 0, e = arr.length; i < e; i++) {
+    const a = arr[i]; // replace filtered items
+    if (a && filterStep.call(tp, a, i, arr)) {
+      carr[k] = mapStep.call(tp, a, i, arr);
+      k += 1;
+    }
+  }
+  carr.length = k; // truncate
+  return carr;
+}
+
+/**
+ * Map and then filter an array in place
+ *
+ * @param {Array} arr
+ * @param {Function} mapStep
+ * @param {Function} filterStep
+ * @param {Object} tp
+ * @returns {Array} Mapped and filtered array (identity)
+ */
+export function mapFilterInSitu<TRes, T>(
+    arr: T[],
+    mapStep: (value: T) => TRes | null | undefined,
+    filterStep: (value: T) => boolean,
+    tp?: any) {
+  tp = tp || null;
+  const carr = arr as unknown as TRes[];
+
+  let i; let k; let e;
+  for (i = 0, k = 0, e = arr.length; i < e; i++) {
+    const a = carr[k] = mapStep.call(tp, arr[i], i, arr);
+    if (a && filterStep.call(tp, a, i, arr)) {
+      k += 1;
+    }
+  }
+  carr.length = k; // truncate
+  return carr;
+}
+
+/**
+ * Get a random integer
+ * @param {Number} min
+ * @param {Number} max
+ * @returns {Number}
+ */
+export function randint(min: number, max: number) {
+  return Math.floor(Math.random() * (max - min)) + min;
+}
+
+
+export function validateSubFolder(folder: string) {
+  if (!folder) {
+    return;
+  }
+  folder = folder.replace(/[/\\]+/g, "/");
+  if (folder.startsWith("/")) {
+    throw new Error("error.noabsolutepath");
+  }
+  if (/^[a-z]:\//i.test(folder)) {
+    throw new Error("error.noabsolutepath");
+  }
+  if (/^\.+\/|\/\.+\/|\/\.+$/g.test(folder)) {
+    throw new Error("error.nodotsinpath");
+  }
+}

extensions/downthemall/lib/uuid.ts

@@ -0,0 +1,45 @@
+/* eslint-disable no-magic-numbers */
+"use strict";
+// License: MIT
+
+const random = (function() {
+try {
+  window.crypto.getRandomValues(new Uint8Array(1));
+  return function(size: number) {
+    const buf = new Uint8Array(size);
+    crypto.getRandomValues(buf);
+    return buf;
+  };
+}
+catch (ex) {
+  // eslint-disable-next-line @typescript-eslint/no-var-requires
+  const cr = require("crypto");
+
+  return function(size: number) {
+    const buf = new Uint8Array(size);
+    cr.randomFillSync(buf);
+    return buf;
+  };
+}
+})();
+
+const UUID_BYTES = 16;
+
+const HEX_MAP = new Map(Array.from(new Uint8Array(256)).map((e, i) => {
+  return [i, i.toString(16).slice(-2).padStart(2, "0")];
+}));
+
+const hex = HEX_MAP.get.bind(HEX_MAP);
+
+export default function uuid() {
+  const vals = random(UUID_BYTES);
+  vals[6] = (vals[6] & 0x0f) + 64;
+  const h = Array.from(vals).map(hex);
+  return [
+    h.slice(0, 4).join(""),
+    h.slice(4, 6).join(""),
+    h.slice(6, 8).join(""),
+    h.slice(8, 10).join(""),
+    h.slice(10).join(""),
+  ].join("-");
+}

extensions/downthemall/lib/windowstatetracker.ts

@@ -0,0 +1,142 @@
+"use strict";
+// License: MIT
+
+import { Prefs } from "./prefs";
+import { windows } from "./browser";
+// eslint-disable-next-line no-unused-vars
+import { Port } from "./bus";
+
+
+const VALID_WINDOW_STATES = Object.freeze(new Set(["normal", "maximized"]));
+
+interface Constraints {
+  minWidth: number;
+  minHeight: number;
+  left?: number;
+  top?: number;
+}
+
+export class WindowStateTracker {
+  private width: number;
+
+  private height: number;
+
+  private readonly minWidth: number;
+
+  private readonly minHeight: number;
+
+  private left: number;
+
+  private top: number;
+
+  private state: string;
+
+  private readonly key: string;
+
+  private windowId: number;
+
+  constructor(windowType: string, constraints: Constraints) {
+    // eslint-disable-next-line no-magic-numbers
+    const {minWidth = 500, minHeight = 400, left = -1, top = -1} = constraints;
+    this.width = this.minWidth = minWidth;
+    this.height = this.minHeight = minHeight;
+    this.left = left;
+    this.top = top;
+    this.state = "normal";
+    this.key = `window-state-${windowType}`;
+    this.update = this.update.bind(this);
+  }
+
+  async init() {
+    const initialState = await Prefs.get(this.key);
+    if (initialState) {
+      Object.assign(this, initialState);
+    }
+    this.validate();
+  }
+
+  getOptions(options: any) {
+    const result = Object.assign(options, {
+      state: this.state,
+    });
+    if (result.state !== "maximized") {
+      result.width = this.width;
+      result.height = this.height;
+      if (this.top >= 0) {
+        result.top = this.top;
+        result.left = this.left;
+      }
+    }
+    return result;
+  }
+
+  validate() {
+    this.width = Math.max(this.minWidth, this.width) || this.minWidth;
+    this.height = Math.max(this.minHeight, this.height) || this.minHeight;
+    this.top = Math.max(-1, this.top) || -1;
+    this.left = Math.max(-1, this.left) || -1;
+    this.state = VALID_WINDOW_STATES.has(this.state) ? this.state : "normal";
+  }
+
+  async update() {
+    if (!this.windowId) {
+      return;
+    }
+    try {
+      const window = await windows.get(this.windowId);
+      if (!VALID_WINDOW_STATES.has(window.state)) {
+        return;
+      }
+      const previous = JSON.stringify(this);
+      this.width = window.width;
+      this.height = window.height;
+      this.left = window.left;
+      this.top = window.top;
+      this.state = window.state;
+      this.validate();
+      if (previous === JSON.stringify(this)) {
+        // Nothing changed
+        return;
+      }
+      await this.save();
+    }
+    catch {
+      // ignored
+    }
+  }
+
+  track(windowId: number, port?: Port) {
+    if (port) {
+      port.on("resized", this.update);
+      port.on("unload", e => this.finalize(e));
+      port.on("disconnect", this.finalize.bind(this));
+    }
+    this.windowId = windowId;
+  }
+
+  async finalize(state?: any) {
+    if (state) {
+      this.left = state.left;
+      this.top = state.top;
+    }
+    await this.update();
+    this.windowId = 0;
+    if (state) {
+      await this.save();
+    }
+  }
+
+  async save() {
+    await Prefs.set(this.key, this.toJSON());
+  }
+
+  toJSON() {
+    return {
+      width: this.width,
+      height: this.height,
+      top: this.top,
+      left: this.left,
+      state: this.state,
+    };
+  }
+}