Refactor cookie handling

Switch to jettison for JSON marshaling Create an HTTP UI Use goembed for assets
Load and save queue and history files
2020-12-21 01:15:16 -08:00 · 2020-12-13 01:05:17 -08:00 · 2020-12-09 16:17:07 -08:00
20 changed files with 1180 additions and 214 deletions
--- a/assets/bootstrap-table.min.css
+++ b/assets/bootstrap-table.min.css
--- a/assets/bootstrap-table.min.js
+++ b/assets/bootstrap-table.min.js
--- a/assets/bootstrap.min.css
+++ b/assets/bootstrap.min.css
--- a/assets/bootstrap.min.js
+++ b/assets/bootstrap.min.js
--- a/assets/downloads.js
+++ b/assets/downloads.js
@ -0,0 +1,85 @@
+function updateDownloads(downloads) {
+	var dls = [];
+	var i = 0;
+	var tbody = $("<tbody></tbody>").attr("id", "table")
+	for (var d of downloads) {
+		var row = $("<tr></tr>").addClass("download").attr("index", i)
+
+		var index = $("<th></th>").attr("scope", "row").text(i)
+		var filename = $("<td></td>").text(d.subdir + "/" + d.filename).addClass("lastlog")
+		var url = $("<td></td>").text(d.url)
+		var status = $("<td></td>").addClass(d.status).css("cursor", "pointer").text(d.status + "\n" + d.error).on("click",{index: i, download: d}, evt => {
+				if (d.status != "Queue" || d.status != "Complete") {
+					setDownload([{
+						index: evt.data("index"),
+						status: "Queue",
+						priority: evt.data("d").priority
+					}])
+				}
+			})
+
+		var action = $("<td></td>").addClass("dropdown")
+		var button = $("<button>Actions</button>").addClass("btn btn-secondary dropdown-toggle").attr("type", "button").attr("data-toggle", "dropdown")
+		var menu = $("<div></div>").addClass("dropdown-menu")
+
+		var del = $('<a href="#">Delete</a>').addClass("dropdown-item").on("click",{index: i, download: d}, evt => {
+						if (confirm("confirm deletion of " + evt.data.download.filename)) {
+							deleteDownload([{
+								index: evt.data.index
+							}])
+						}
+					})
+		var resume = $('<a href="#">Resume</a>').addClass("dropdown-item").on("click",{index: i, download: d}, evt => {
+						setDownload([{
+							index: evt.data.index,
+							status: "Queue",
+							priority: evt.data.download.priority
+						}])
+					})
+		var stop = $('<a href="#">Stop</a>').addClass("dropdown-item").on("click",{index: i, download: d}, evt => {
+						setDownload([{
+							index: evt.data.index,
+							status: "Stop",
+							priority: evt.data.download.priority
+						}])
+					})
+		var na = $('<a href="#">N/A</a>').addClass("dropdown-item")
+
+		menu.append(del, resume, stop, na)
+		action.append(button, menu)
+
+		var progress = $("<td></td>").append(
+				$("<div></div>").addClass("progress").append(
+					$("<div></div>").addClass("progress-bar text-dark").css("width", (d.progress ?? "0") + '%').attr("id", "progress-" + i)
+					.text((d.progress ?? "0") + '%')
+				),
+			)
+
+
+		row.append(
+			index,
+			filename,
+			url,
+			status,
+			action,
+			progress
+		)
+		tbody.append(row)
+		i++;
+	}
+	$("#table").replaceWith(tbody)
+}
+
+function deleteDownload(data){
+	var xhr = new XMLHttpRequest();
+	xhr.open("DELETE", "http://gloader.narnian.us:8844/delete");
+	xhr.setRequestHeader("Content-Type", "application/javascript");
+	xhr.send(JSON.stringify(data));
+}
+
+function setDownload(data){
+	var xhr = new XMLHttpRequest();
+	xhr.open("POST", "http://gloader.narnian.us:8844/set");
+	xhr.setRequestHeader("Content-Type", "application/javascript");
+	xhr.send(JSON.stringify(data));
+}
--- a/assets/favicon.ico
+++ b/assets/favicon.ico
--- a/assets/footer.tmpl
+++ b/assets/footer.tmpl
@ -0,0 +1,9 @@
+
+</div>
+
+<script src="/jquery-3.2.1.slim.min.js"></script>
+<script src="/popper.min.js"></script>
+<script src="/bootstrap.min.js"></script>
+<script src="/bootstrap-table.min.js"></script>
+
+</html>
--- a/assets/header.tmpl
+++ b/assets/header.tmpl
@ -0,0 +1,78 @@
+<!DOCTYPE html>
+<html lang="en">
+<title>{{ .Hostname }} — gloader</title>
+<link rel="stylesheet" href="/bootstrap.min.css" />
+<link rel="stylesheet" href="/bootstrap-table.min.css" />
+<style type="text/css">
+.progress-bar:nth-child(5n) {
+        background-color: #337ab7;
+}
+.progress-bar:nth-child(5n+1) {
+        background-color: #5cb85c;
+}
+.progress-bar:nth-child(5n+2) {
+        background-color: #5bc0de;
+}
+.progress-bar:nth-child(5n+3) {
+        background-color: #f0ad4e;
+}
+.progress-bar:nth-child(5n+4) {
+        background-color: #d9534f;
+}
+.lastlog {
+        text-overflow: ellipsis;
+        white-space: nowrap;
+        overflow: hidden;
+}
+.Error {
+  color: red;
+}
+
+.container-fluid {
+  padding-left: 45px;
+}
+
+.navbar-default .navbar-text {
+  color: #777;
+}
+
+.navbar-default {
+  background-color: #f8f8f8;
+  border-color: #e7e7e7;
+}
+
+body {
+  font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
+}
+</style>
+
+  <nav class="navbar navbar-default">
+        <p style="margin-top: 0.25em; font-size: 18px"><a href="/">gokrazy</a><br>
+        <!-- <small style="font-size: 11px" class="text-muted">version { .BuildTimestamp }}</small> --></p>
+
+       <table class="navbar-text navbar-right" style="border-spacing: 10px 0; border-collapse: separate">
+    <tr>
+      <th>host</th>
+      <td>{{ .Hostname }}</td>
+    </tr>
+    <!-- <tr>
+      <th>kernel</th>
+      <td>{ .Kernel }}</td>
+    </tr>
+    { if (ne .Model "") }}
+    <tr>
+      <th>model</th>
+      <td>{ .Model }}</td>
+    </tr>
+    { end }}
+    { if .EEPROM }}
+    <tr>
+      <th>EEPROM<br>(SHA256)</th>
+      <td>{ shortenSHA256 .EEPROM.PieepromSHA256 }}<br>{ shortenSHA256 .EEPROM.VL805SHA256 }}</td>
+    </tr>
+    { end }} -->
+  </table>
+
+  </nav>
+
+  <div class="container-fluid">
--- a/assets/history.tmpl
+++ b/assets/history.tmpl
@ -0,0 +1,52 @@
+{{ template "header" . }}
+
+	<div >
+
+		<!-- <h1>services</h1> -->
+
+		<table data-toggle="table" class="table table-hover">
+			<tbody id="table">
+				<thead>
+					<tr>
+						<th data-sortable="true" scope="col" width="5%">Index</th>
+						<th data-sortable="true" scope="col">Filename</th>
+						<th data-sortable="true" scope="col">URL</th>
+						<th data-sortable="true" scope="col">Status</th>
+						<th scope="col">Actions</th>
+					</tr>
+				</thead>
+
+				{{ range $idx, $request := .History }}
+				<tr class="download">
+					<th scope="row">{{ $idx }}</th>
+					<td class="lastlog">
+						{{ printf "%s/%s" $request.Subdir $request.Filename }}
+					</td>
+					<td>
+						{{ $request.URL }}
+					</td>
+					<td>
+						{{ printf "%v" $request.Status }}
+						{{ if $request.Error }}
+							{{ $request.Error }}
+						{{ end }}
+					</td>
+					<td class="dropdown">
+						<button class="btn btn-secondary dropdown-toggle" type="button" data-toggle="dropdown">Actions</button>
+						<div class="dropdown-menu">
+							<a class="dropdown-item" href="#" onclick="if (confirm('confirm deletion of ' + {{ printf "%s/%s" $request.Subdir $request.Filename }})) { deleteDownload([{index: {{ $idx }}, history: true }])}">Delete</a>
+							<a class="dropdown-item" href="#" onclick='setDownload([{ index: {{ $idx }}, status: "Queue", priority: {{ $request.Priority }} }])'>Resume</a>
+							<a class="dropdown-item" href="#" onclick='setDownload([{ index: {{ $idx }}, status: "Stop", priority: {{ $request.Priority }} }])'>Stop</a>
+							<a class="dropdown-item" title="{{ printf "%s/%s" $request.Subdir $request.Filename }}" href="/get/{{ printf "%s/%s" $request.Subdir $request.Filename }}">Retrieve</a>
+						</div>
+					</td>
+				</tr>
+				{{ end }}
+			</tbody>
+		</table>
+
+	</div>
+
+<script src="/downloads.js"></script>
+
+{{ template "footer" . }}
--- a/assets/jquery-3.2.1.slim.min.js
+++ b/assets/jquery-3.2.1.slim.min.js
--- a/assets/popper.min.js
+++ b/assets/popper.min.js
--- a/assets/queue.tmpl
+++ b/assets/queue.tmpl
@ -0,0 +1,70 @@
+{{ template "header" . }}
+
+	<div >
+
+		<!-- <h1>services</h1> -->
+
+		<table data-toggle="table" class="table">
+			<thead>
+				<tr>
+					<th data-sortable="true" scope="col" width="5%">Index</th>
+					<th data-sortable="true" scope="col">Filename</th>
+					<th data-sortable="true" scope="col">URL</th>
+					<th data-sortable="true" scope="col">Status</th>
+					<th scope="col">Actions</th>
+					<th data-sortable="true" scope="col">Progress</th>
+				</tr>
+			</thead>
+			<tbody id="table">
+
+				{{ range $idx, $request := .Queue }}
+				<tr class="download">
+					<th scope="row">{{ $idx }}</th>
+					<td class="lastlog">
+						{{ printf "%s/%s" $request.Subdir $request.Filename }}
+					</td>
+					<td>
+						{{ $request.URL }}
+					</td>
+					<td>
+						{{ printf "%v" $request.Status }}
+						{{ if $request.Error }}
+							{{ $request.Error }}
+						{{ end }}
+					</td>
+					<td class="dropdown">
+						<button class="btn btn-secondary dropdown-toggle" type="button" data-toggle="dropdown">Actions</button>
+						<div class="dropdown-menu">
+							<a class="dropdown-item" href="#" onclick="if (confirm('confirm deletion of ' + {{ printf "%s/%s" $request.Subdir $request.Filename }})) { deleteDownload([{index: {{ $idx }} }])}">Delete</a>
+							<a class="dropdown-item" href="#" onclick='setDownload([{ index: {{ $idx }}, status: "Queue", priority: {{ $request.Priority }} }])'>Resume</a>
+							<a class="dropdown-item" href="#" onclick='setDownload([{ index: {{ $idx }}, status: "Stop", priority: {{ $request.Priority | js}} }])'>Stop</a>
+							<a class="dropdown-item" href="/get/{{ printf "%s/%s" $request.Subdir $request.Filename }}">Retrieve</a>
+						</div>
+					</td>
+					<td>
+						<div class="progress">
+							<div id="progress-{{ $idx }}" class="progress-bar text-dark" style="width: 0{{ $request.Progress }}%">0{{ $request.Progress }}%</div>
+						</div>
+					</td>
+				</tr>
+				{{ end }}
+			</tbody>
+		</table>
+
+	</div>
+
+<script src="/downloads.js"></script>
+<script type="text/javascript">
+	function setDownloads() {
+		var xhr = new XMLHttpRequest();
+		xhr.open("GET", "http://gloader.narnian.us:8844/queued");
+		xhr.setRequestHeader("Accept", "application/javascript");
+		xhr.onload = evt => {
+			updateDownloads(JSON.parse(evt.target.responseText));
+		};
+		xhr.send();
+	}
+	var interval = setInterval(setDownloads, 5 * 1000)
+</script>
+
+{{ template "footer" . }}
--- a/bundle.go
+++ b/bundle.go
@ -0,0 +1,3 @@
+package main
+
+//go:generate sh -c "go run goembed.go -package bundled -var assets assets/history.tmpl assets/downloads.js assets/header.tmpl assets/footer.tmpl assets/queue.tmpl assets/status.tmpl assets/favicon.ico assets/bootstrap.min.css assets/bootstrap-table.min.css assets/bootstrap-table.min.js assets/bootstrap.min.js assets/popper.min.js assets/jquery-3.2.1.slim.min.js > bundled/GENERATED_bundled.go && gofmt -w bundled/GENERATED_bundled.go"
--- a/bundled/bundled.go
+++ b/bundled/bundled.go
@ -0,0 +1,18 @@
+package bundled
+
+import (
+	"bytes"
+	"net/http"
+	"time"
+)
+
+func Asset(basename string) string {
+	return string(assets["assets/"+basename])
+}
+
+func HTTPHandlerFunc(basename string) http.Handler {
+	modTime := time.Now()
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.ServeContent(w, r, basename, modTime, bytes.NewReader(assets["assets/"+basename]))
+	})
+}
--- a/downloader.go
+++ b/downloader.go
@ -1,10 +1,16 @@
 package main

 import (
+	"bytes"
+	"context"
 	"encoding/json"
 	"errors"
 	"fmt"
+	"html/template"
+	"io"
 	"io/ioutil"
+	"log"
+	"math"
 	"mime"
 	"net"
 	"net/http"
@ -18,21 +24,23 @@ import (
 	"strings"
 	"time"

+	"git.narnian.us/lordwelch/gloader/bundled"
+
 	"github.com/cavaliercoder/grab"
 	"github.com/lordwelch/pathvalidate"
+	"github.com/wI2L/jettison"
+
 	"golang.org/x/net/publicsuffix"
 )

 var (
-	DefaultCookieJar          = newCookieJar()
-	DefaultGrabClient         = grab.NewClient()
 	DefaultMaxActiveDownloads = 4

 	ErrUnsupportedScheme = errors.New("unsupported scheme")
 )

-type Priority uint8
-type Status uint8
+type Priority int
+type Status int

 const (
 	Highest Priority = iota
@ -56,57 +64,154 @@ type Downloader struct {
 	DownloadDir        string
 	CompleteDir        string
 	InfoDir            string
-	Grab               *grab.Client
-	Jar                http.CookieJar
 	MaxActiveDownloads int
 	Server             *http.Server
-	downloads          RequestQueue
-	history            RequestQueue
+	Downloads          RequestQueue
+	History            RequestQueue
 	NewRequest         chan Request
 	requestDone        chan *Request
+	OnComplete         func(d *Downloader, r Request)
+	OnAdd              func(d *Downloader, r Request)
 }

 type Request struct {
-	URL           url.URL        `json:"url"`
+	URL           string         `json:"url"`
 	Cookies       []http.Cookie  `json:"cookies"`
 	ForceDownload bool           `json:"forceDownload"`
-	Status        Status         `json:"-"`
+	Status        Status         `json:"status"`
 	Priority      Priority       `json:"priority"`
-	Filepath      string         `json:"filepath"`
+	FilePath      string         `json:"filepath"`
+	Filename      string         `json:"filename"`
+	Subdir        string         `json:"subdir"`
 	TempPath      string         `json:"tempPath"`
 	Response      *grab.Response `json:"-"`
-	Error         error          `json:"-"`
-	CompletedDate time.Time
+	Error         string         `json:"error"`
+	Err           error          `json:"-"`
+	CompletedDate time.Time      `json:"completedDate"`
+	Jar           http.CookieJar `json:"-"`
+	Progress      string         `json:"progress,omitempty"`
+	grab          *grab.Client
 }

 type RequestQueue struct {
-	queue    []*Request
+	Queue    []*Request
 	URLSort  bool
 	DateSort bool
 }

+func (p Priority) MarshalJSON() ([]byte, error) {
+	var v string
+	switch p {
+	default:
+		v = "Medium"
+	case Medium:
+		v = "Medium"
+	case Low:
+		v = "Low"
+	case High:
+		v = "High"
+	case Highest:
+		v = "Highest"
+	}
+	return json.Marshal(v)
+}
+
+func (p *Priority) UnmarshalJSON(b []byte) error {
+	var (
+		v int
+		s string
+	)
+	if err := json.Unmarshal(b, &v); err == nil {
+		*p = Priority(v)
+		return nil
+	}
+	if err := json.Unmarshal(b, &s); err != nil {
+		return err
+	}
+	switch strings.ToLower(s) {
+	default:
+		*p = Medium
+	case "medium":
+		*p = Medium
+	case "low":
+		*p = Low
+	case "high":
+		*p = High
+	case "highest":
+		*p = Highest
+	}
+	return nil
+}
+
+func (s *Status) UnmarshalJSON(b []byte) error {
+	var v string
+	if err := json.Unmarshal(b, &v); err != nil {
+		return err
+	}
+	switch strings.ToLower(v) {
+	default:
+		*s = Queued
+	case "queued", "queue":
+		*s = Queued
+	case "complete", "completed":
+		*s = Complete
+	case "stop", "stopped":
+		*s = Stopped
+	case "download", "downloading":
+		*s = Downloading
+	case "error":
+		*s = Error
+	case "cancel", "canceled":
+		*s = Canceled
+	}
+	return nil
+}
+
+func (s Status) MarshalJSON() ([]byte, error) {
+	return json.Marshal(s.String())
+}
+
+func (s Status) String() string {
+	switch s {
+	default:
+		return "Queued"
+	case Queued:
+		return "Queued"
+	case Complete:
+		return "Complete"
+	case Stopped:
+		return "Stopped"
+	case Downloading:
+		return "Downloading"
+	case Error:
+		return "Error"
+	case Canceled:
+		return "Canceled"
+	}
+}
+
 func (rq RequestQueue) Less(i, j int) bool {
 	ii := 0
 	jj := 0
-	if rq.queue[i].ForceDownload {
+	if rq.Queue[i].ForceDownload {
 		ii = 1
 	}
-	if rq.queue[j].ForceDownload {
+	if rq.Queue[j].ForceDownload {
 		jj = 1
 	}
 	if ii < jj {
 		return true
 	}

-	if rq.queue[i].Priority < rq.queue[j].Priority {
+	if rq.Queue[i].Priority < rq.Queue[j].Priority {
 		return true
 	}

-	if rq.DateSort && rq.queue[i].CompletedDate.Before(rq.queue[j].CompletedDate) {
+	if rq.DateSort && rq.Queue[i].CompletedDate.Before(rq.Queue[j].CompletedDate) {
 		return true
 	}

-	if rq.URLSort && rq.queue[i].URL.String() < rq.queue[j].URL.String() {
+	if rq.URLSort && rq.Queue[i].URL < rq.Queue[j].URL {
 		return true
 	}

@ -114,44 +219,73 @@ func (rq RequestQueue) Less(i, j int) bool {
 }

 func (rq RequestQueue) Len() int {
-	return len(rq.queue)
+	return len(rq.Queue)
 }

 func (rq RequestQueue) Swap(i, j int) {
-	rq.queue[i], rq.queue[j] = rq.queue[j], rq.queue[i]
+	rq.Queue[i], rq.Queue[j] = rq.Queue[j], rq.Queue[i]
 }

 func (rq *RequestQueue) Pop(i int) *Request {
-	r := rq.queue[i]
-	copy(rq.queue[i:], rq.queue[i+1:])
-	rq.queue[len(rq.queue)-1] = nil
-	rq.queue = rq.queue[:len(rq.queue)-1]
+	r := rq.Queue[i]
+	copy(rq.Queue[i:], rq.Queue[i+1:])
+	rq.Queue[len(rq.Queue)-1] = nil
+	rq.Queue = rq.Queue[:len(rq.Queue)-1]
 	return r
 }

 func (rq *RequestQueue) remove(r *Request) {
-	for i, req := range rq.queue {
+	for i, req := range rq.Queue {
 		if req == r {
-			copy(rq.queue[i:], rq.queue[i+1:])
-			rq.queue[len(rq.queue)-1] = nil
-			rq.queue = rq.queue[:len(rq.queue)-1]
+			copy(rq.Queue[i:], rq.Queue[i+1:])
+			rq.Queue[len(rq.Queue)-1] = nil
+			rq.Queue = rq.Queue[:len(rq.Queue)-1]
 			break
 		}
 	}
 }

+func (rq *RequestQueue) updateStatus() {
+	for _, req := range rq.Queue {
+		if req.Response != nil {
+			req.Progress = fmt.Sprintf("%.2f", math.Abs(req.Response.Progress()*100))
+		}
+	}
+}
+
+func (r *Request) setError(err error) {
+	if err != nil {
+		r.Status = Error
+		r.Err = err
+		r.Error = err.Error()
+	} else {
+		r.Status = Paused
+		r.Err = nil
+		r.Error = ""
+	}
+}
+
+func (r Request) Delete() error {
+	var err, ret error
+	if r.Response != nil {
+		err = r.Response.Cancel()
+		if err != nil && err != context.Canceled {
+			ret = err
+		}
+	}
+	_ = os.Remove(r.TempPath)
+	err = os.Remove(r.FilePath)
+	if err != nil && err != os.ErrNotExist {
+		ret = err
+	}
+	return ret
+}
+
 func newCookieJar() http.CookieJar {
 	c, _ := cookiejar.New(&cookiejar.Options{PublicSuffixList: publicsuffix.List})
 	return c
 }

-func newDownloader() *Downloader {
-	return &Downloader{
-		Jar:  DefaultCookieJar,
-		Grab: DefaultGrabClient,
-	}
-}
-
 func (d *Downloader) Start(network, address string) {
 	var (
 		listener net.Listener
@ -178,11 +312,10 @@ func (d *Downloader) Start(network, address string) {
 			ReadTimeout:  2 * time.Minute,
 			WriteTimeout: 2 * time.Minute,
 		}
-
 	}

 	if d.DataDir == "" {
-		d.DataDir = "/perm/downloader"
+		d.DataDir = "/perm/gloader"
 	}

 	if d.DownloadDir == "" {
@ -193,26 +326,146 @@ func (d *Downloader) Start(network, address string) {
 		d.CompleteDir = path.Join(d.DataDir, "Complete")
 	}

-	fmt.Println(d.DataDir)
-	fmt.Println(d.DownloadDir)
-	fmt.Println(d.CompleteDir)
-	os.MkdirAll(d.DataDir, 0777)
-	os.MkdirAll(d.DownloadDir, 0777)
-	os.MkdirAll(d.CompleteDir, 0777)
+	log.Println(d.DataDir)
+	log.Println(d.DownloadDir)
+	log.Println(d.CompleteDir)
+	_ = os.MkdirAll(d.DataDir, 0777)
+	_ = os.MkdirAll(d.DownloadDir, 0777)
+	_ = os.MkdirAll(d.CompleteDir, 0777)

 	listener, err = net.Listen(network, address)
 	if err != nil {
 		panic(err)
 	}
-	fmt.Println("adding /add handler")
-	// mux.HandleFunc("/", d.UI)
-	mux.HandleFunc("/add", d.restAddDownload)
+	log.Println("adding http handlers")
+
+	d.initStatus(mux)
+	mux.HandleFunc("/add", d.restAddDownload)
+	mux.HandleFunc("/queued", d.restStatus(true))
+	mux.HandleFunc("/completed", d.restStatus(false))
+	mux.HandleFunc("/set", d.restSetDownloadStatus)
+	mux.HandleFunc("/delete", d.restDelete)
+	mux.Handle("/get/", http.StripPrefix("/get/", http.FileServer(http.Dir(d.CompleteDir))))
+
+	log.Println("starting main go routine")

-	fmt.Println("starting main go routine")
 	go d.download()

-	fmt.Println("serving http server")
-	d.Server.Serve(listener)
+	log.Println("serving http server")
+	_ = d.Server.Serve(listener)
+}
+
+func httpMethodNotAllowed(w http.ResponseWriter, r *http.Request, method string) {
+	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+	w.Header().Set("X-Content-Type-Options", "nosniff")
+	w.Header().Add("Allow", method)
+	w.WriteHeader(http.StatusMethodNotAllowed)
+	fmt.Fprintf(w, "HTTP Error 405 – Method Not Allowed\nOnly %s method is allowed\n", method)
+	log.Printf("HTTP Error 405 – Method Not Allowed\nOnly %s method is allowed\n", method)
+}
+
+func (d *Downloader) restDelete(w http.ResponseWriter, r *http.Request) {
+	var (
+		err   error
+		index []struct {
+			Index   int
+			History bool
+		}
+		ret []Request
+	)
+	if r.Method != http.MethodDelete {
+		httpMethodNotAllowed(w, r, http.MethodDelete)
+		return
+	}
+	err = json.NewDecoder(r.Body).Decode(&index)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+	sort.Slice(index, func(i, j int) bool {
+		return index[i].Index < index[j].Index
+	})
+	for x, i := range index {
+		if i.History {
+			if i.Index >= d.History.Len() || i.Index < 0 {
+				http.Error(w, fmt.Sprintf("slice index out of bounds. index: %d length of slice: %d", i.Index, d.Downloads.Len()), http.StatusBadRequest)
+				return
+			}
+			ret = append(ret, *d.History.Pop(i.Index - x))
+			ret[len(ret)-1].Delete()
+		} else {
+			if i.Index >= d.Downloads.Len() || i.Index < 0 {
+				http.Error(w, fmt.Sprintf("slice index out of bounds. index: %d length of slice: %d", i.Index, d.Downloads.Len()), http.StatusBadRequest)
+				return
+			}
+			ret = append(ret, *d.Downloads.Pop(i.Index - x))
+			ret[len(ret)-1].Delete()
+		}
+	}
+	d.syncDownloads()
+
+	v, err := jettison.MarshalOpts(ret, jettison.DenyList([]string{"cookies"}))
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	w.Header().Add("Content-Type", "application/json; charset=utf-8")
+	w.WriteHeader(http.StatusOK)
+	w.Write(v)
+}
+
+func (d *Downloader) restSetDownloadStatus(w http.ResponseWriter, r *http.Request) {
+	var (
+		err   error
+		index []struct {
+			Index    int
+			Status   Status
+			Priority Priority
+		}
+	)
+	if r.Method != http.MethodPost {
+		httpMethodNotAllowed(w, r, http.MethodPost)
+		return
+	}
+	err = json.NewDecoder(r.Body).Decode(&index)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+	for _, i := range index {
+		if i.Index >= d.Downloads.Len() || i.Index < 0 {
+			http.Error(w, fmt.Sprintf("slice index out of bounds. index: %d length of slice: %d", i.Index, d.Downloads.Len()), http.StatusBadRequest)
+			return
+		}
+		r := d.Downloads.Queue[i.Index]
+		r.Priority = i.Priority
+		r.Status = i.Status
+	}
+	d.syncDownloads()
+}
+
+func (d *Downloader) restStatus(q bool) func(http.ResponseWriter, *http.Request) {
+	return func(w http.ResponseWriter, r *http.Request) {
+		var queue = &d.History
+		if q {
+			d.Downloads.updateStatus()
+			queue = &d.Downloads
+		}
+
+		if r.Method != http.MethodGet {
+			httpMethodNotAllowed(w, r, http.MethodGet)
+			return
+		}
+		queue.updateStatus()
+		v, err := jettison.MarshalOpts(queue.Queue, jettison.DenyList([]string{"cookies"}))
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+		w.Header().Add("Content-Type", "application/json; charset=utf-8")
+		w.WriteHeader(http.StatusOK)
+		w.Write(v)
+	}
 }

 func (d *Downloader) restAddDownload(w http.ResponseWriter, r *http.Request) {
@ -221,12 +474,7 @@ func (d *Downloader) restAddDownload(w http.ResponseWriter, r *http.Request) {
 		err      error
 	)
 	if r.Method != http.MethodPost {
-		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
-		w.Header().Set("X-Content-Type-Options", "nosniff")
-		w.Header().Add("Allow", http.MethodPost)
-		w.WriteHeader(http.StatusMethodNotAllowed)
-		fmt.Fprintln(w, "HTTP Error 405 – Method Not Allowed\nOnly POST method is allowed")
-		fmt.Println("HTTP Error 405 – Method Not Allowed\nOnly POST method is allowed")
+		httpMethodNotAllowed(w, r, http.MethodPost)
 		return
 	}
 	// TODO fail only on individual requests
@ -236,36 +484,53 @@ func (d *Downloader) restAddDownload(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	for _, req := range requests {
-		req.TempPath = ""
-		fmt.Println("adding request", req.URL.String())
+		req.TempPath = "" // not allowed via REST API
+		req.FilePath = "" // not allowed via REST API
+		if req.Status != Paused {
+			req.Status = Queued
+		}
+		log.Println("adding request", req.URL)
 		d.NewRequest <- req
 	}
 	w.WriteHeader(http.StatusOK)
 }

-func (d Downloader) getContentDispsition(r Request) string {
+func (d Downloader) getNameFromHEAD(r Request) string {
 	var (
 		err error
 		re  *http.Response
 		p   map[string]string
 	)
+
+	r.insertCookies()
 	ht := &http.Client{
-		Jar:     d.Jar,
+		Jar:     r.Jar,
 		Timeout: 30 * time.Second,
+		Transport: &http.Transport{
+			Dial: (&net.Dialer{
+				Timeout:   5 * time.Second,
+				KeepAlive: 30 * time.Second,
+			}).Dial,
+			TLSHandshakeTimeout:   5 * time.Second,
+			ResponseHeaderTimeout: 5 * time.Second,
+			ExpectContinueTimeout: 1 * time.Second,
+		},
 	}
-	re, err = ht.Head(r.URL.String())
+	re, err = ht.Head(r.URL)
 	if err != nil {
 		return ""
 	}
+	if re.StatusCode < 200 || re.StatusCode > 299 {
+		return ""
+	}
 	re.Body.Close()
 	_, p, err = mime.ParseMediaType(re.Header.Get("Content-Disposition"))
-	if err != nil {
-		return ""
+	if err == nil {
+		if f, ok := p["filename"]; ok {
+			return f
+		}
 	}
-	if f, ok := p["filename"]; ok {
-		return f
-	}
-	return ""
+	return path.Base(re.Request.URL.Path)
 }

 // getFilename checks the provided filepath
@ -273,28 +538,29 @@ func (d Downloader) getContentDispsition(r Request) string {
 // if not set uses the basename of the url
 // and sanitizes the filename using github.com/lordwelch/pathvalidate
 func (d *Downloader) getFilename(r *Request) {
-	fmt.Println("Determining filename")
-	r.Filepath = filepath.Clean(r.Filepath)
-	if r.Filepath == "." {
-		fmt.Println("filename is empty, testing head request")
-		r.Filepath = d.getContentDispsition(*r)
-		fmt.Println("path from head request:", r.Filepath)
-		if r.Filepath == "" {
-			r.Filepath, _ = url.PathUnescape(filepath.Base(r.URL.Path))
+	log.Println("Determining filename")
+	r.Filename = filepath.Clean(r.Filename)
+	if r.Filename == "." {
+		log.Println("filename is empty, testing head request")
+		r.Filename = d.getNameFromHEAD(*r)
+		log.Println("path from head request:", r.Filename)
+		if r.Filename == "" {
+			u, _ := url.Parse(r.URL)
+			r.Filename, _ = url.PathUnescape(filepath.Base(u.Path))
 		}
 	}
-	r.Filepath, _ = pathvalidate.SanitizeFilename(r.Filepath, '_')
-	r.Filepath = filepath.Join(d.DownloadDir, r.Filepath)
+	r.Filename, _ = pathvalidate.SanitizeFilename(r.Filename, '_')
+	// r.Filename = filepath.Join(d.CompleteDir, r.Filename)

-	// if filepath.IsAbs(r.Filepath) { // should already exist
-	// 	dir, file := filepath.Split(r.Filepath)
+	// if filepath.IsAbs(r.Filename) { // should already exist
+	// 	dir, file := filepath.Split(r.Filename)
 	// 	// someone is trying to be sneaky (or someone changed the CompleteDir), change path to the correct dir
 	// 	if dir != filepath.Clean(d.CompleteDir) {
-	// 		r.Filepath = filepath.Join(d.CompleteDir, file)
+	// 		r.Filename = filepath.Join(d.CompleteDir, file)
 	// 	}
 	// 	return
 	// }
-	fmt.Println("result path:", r.Filepath)
+	log.Println("result path:", r.Filename)
 }

 func getNewFilename(dir, name string) string {
@ -302,15 +568,16 @@ func getNewFilename(dir, name string) string {
 		err   error
 		index = 1
 	)
-	fmt.Println("getfilename", dir, name)
+	log.Println("getfilename", dir, name)
 	ext := filepath.Ext(name)
 	base := strings.TrimSuffix(name, ext)
-	fmt.Println("stat", filepath.Join(dir, name))
+	log.Println("stat", filepath.Join(dir, name))
 	_, err = os.Stat(filepath.Join(dir, name))
 	for err == nil {
 		name = strings.TrimRight(base+"."+strconv.Itoa(index)+ext, ".")
-		fmt.Println("stat", filepath.Join(dir, name))
+		log.Println("stat", filepath.Join(dir, name))
 		_, err = os.Stat(filepath.Join(dir, name))
+		index++
 	}
 	if os.IsNotExist(err) {
 		return filepath.Join(dir, name)
@ -318,93 +585,120 @@ func getNewFilename(dir, name string) string {
 	panic(err) // other path error
 }

-func (d Downloader) getDownloadFilename(r *Request) {
+func (d Downloader) getTempFilename(r *Request) {
 	if r.TempPath == "" {
-		f, err := ioutil.TempFile(d.DownloadDir, filepath.Base(r.Filepath))
+		f, err := ioutil.TempFile(d.DownloadDir, filepath.Base(r.Filename))
 		if err != nil {
-			fmt.Printf("request for %v failed: %v", r.URL.String(), err)
+			log.Printf("request for %v failed: %v", r.URL, err)
 		}
+		r.TempPath = f.Name()
 		f.Close()
-		r.TempPath = filepath.Join(d.DownloadDir, f.Name())
 	}
-	f, err := os.OpenFile(r.Filepath, os.O_CREATE|os.O_EXCL, 0666)
+	os.MkdirAll(filepath.Dir(r.FilePath), 0o777)
+	f, err := os.OpenFile(r.Filename, os.O_CREATE|os.O_EXCL, 0666)
 	if err != nil {
 		return
 	}
 	f.Close()
 }

-func (d Downloader) SearchDownloads(u url.URL) int {
-	for i, req := range d.downloads.queue {
-		if req.URL.String() == u.String() {
+func (d Downloader) SearchDownloads(u string) int {
+	for i, req := range d.Downloads.Queue {
+		if req.URL == u {
 			return i
 		}
 	}
 	return -1
 }

-func (d Downloader) SearchHistory(u url.URL) int {
-	for i, req := range d.history.queue {
-		if req.URL.String() == u.String() {
+func (d Downloader) SearchHistory(u string) int {
+	for i, req := range d.History.Queue {
+		if req.URL == u {
 			return i
 		}
 	}
 	return -1
-
 }

-func (d Downloader) FindRequest(u url.URL) *Request {
+func (d Downloader) FindRequest(u string) *Request {
 	if i := d.SearchDownloads(u); i >= 0 {
-		return d.downloads.queue[i]
+		return d.Downloads.Queue[i]
 	}
 	if i := d.SearchHistory(u); i >= 0 {
-		return d.history.queue[i]
+		return d.History.Queue[i]
 	}
 	return nil
 }

 func (d *Downloader) addRequest(r *Request) {
-	fmt.Println("adding download for", r.URL.String())
+	log.Println("adding download for", r.URL)
 	req := d.FindRequest(r.URL)
 	d.getFilename(r)

 	if req != nil { // url alread added
-		fmt.Println("URL is already added", r.URL.String())
-		if fi, err := os.Stat(r.Filepath); filepath.Base(req.Filepath) == filepath.Base(r.Filepath) || (err == nil && fi.Name() == filepath.Base(r.Filepath) && fi.Size() != 0) { // filepath has been found, should this check for multiple downloads of the same url or let the download name increment automatically
-			fmt.Println("file already exists", r.Filepath)
-			//getNewFilename(d.CompleteDir, filepath.Base(r.Filepath))
-			d.validate(*r) // TODO, should also check to see if it seems like it is similar, (check first k to see if it is the same file?? leave option to user)
+		log.Println("URL is already added", r.URL)
+		return
+		// if fi, err := os.Stat(r.Filepath); filepath.Base(req.Filepath) == filepath.Base(r.Filepath) || (err == nil && fi.Name() == filepath.Base(r.Filepath) && fi.Size() != 0) { // filepath has been found, should this check for multiple downloads of the same url or let the download name increment automatically
+		// 	log.Println("file already exists", r.Filepath)
+		// 	d.validate(*r) // TODO, should also check to see if it seems like it is similar, (check first k to see if it is the same file?? leave option to user)
+		// 	return
+		// }
+	}
+	r.FilePath = getNewFilename(d.CompleteDir, filepath.Join(r.Subdir, r.Filename))
+	d.Downloads.Queue = append(d.Downloads.Queue, r)
+
+	if len(d.getRunningDownloads()) < d.MaxActiveDownloads {
+		d.startDownload(d.Downloads.Len() - 1)
+	}
+}
+
+// func (d *Downloader) validate(r Request) {
+// 	//TODO
+// }
+
+func (d *Downloader) startDownload(i int) {
+	var (
+		r   *Request
+		req *grab.Request
+		err error
+	)
+	r = d.Downloads.Queue[i]
+	r.insertCookies()
+	d.getTempFilename(r)
+	log.Println("starting download for", r.URL, "to", r.TempPath)
+	// d.Downloads.Queue = append(d.Downloads.Queue, r)
+	if r.Response == nil || r.Response.Err() != nil {
+		req, err = grab.NewRequest(r.TempPath, r.URL)
+		if err != nil {
+			r.setError(err)
 			return
 		}
-	} else { // new request, download link
-		r.Filepath = getNewFilename(d.CompleteDir, filepath.Base(r.Filepath))
-		d.downloads.queue = append(d.downloads.queue, r)
-	}
-	if len(d.getRunningDownloads()) < d.MaxActiveDownloads {
-		d.startDownload(r)
-	}
-}
-
-func (d *Downloader) validate(r Request) {
-	//TODO
-}
-
-func (d *Downloader) startDownload(r *Request) {
-	fmt.Println("starting download for", r.URL.String())
-	d.getDownloadFilename(r)
-	req, err := grab.NewRequest(r.TempPath, r.URL.String())
-	if err != nil {
-		r.Status = Error
-		r.Error = err
-		return
 	}
 	r.Status = Downloading
-	r.Response = d.Grab.Do(req)
+
+	if r.grab == nil {
+		r.grab = &grab.Client{
+			HTTPClient: &http.Client{
+				Jar: r.Jar,
+				Transport: &http.Transport{
+					Dial: (&net.Dialer{
+						Timeout:   10 * time.Second,
+						KeepAlive: 30 * time.Second,
+					}).Dial,
+					TLSHandshakeTimeout:   5 * time.Second,
+					ResponseHeaderTimeout: 5 * time.Second,
+					ExpectContinueTimeout: 1 * time.Second,
+				},
+			},
+		}
+	}
+
+	r.Response = r.grab.Do(req)
 	go func(r *Request) {
-		fmt.Println("wait for download")
-		fmt.Println(r.Response.IsComplete())
+		log.Println("wait for download")
+		log.Println(r.Response.IsComplete())
 		r.Response.Wait()
-		fmt.Println("download completed for", r.URL)
+		log.Println("download completed for", r.URL)
 		d.requestDone <- r
 	}(r)
 }
@ -413,7 +707,7 @@ func (d Downloader) getRunningDownloads() []*Request {
 	var (
 		running = make([]*Request, 0, d.MaxActiveDownloads)
 	)
-	for _, req := range d.downloads.queue {
+	for _, req := range d.Downloads.Queue {
 		if req.Status == Downloading && req.Response != nil {
 			running = append(running, req)
 		}
@ -425,61 +719,168 @@ func (d *Downloader) syncDownloads() {
 	if len(d.getRunningDownloads()) >= d.MaxActiveDownloads {
 		return
 	}
-	sort.Stable(d.downloads)
+	sort.Stable(d.Downloads)
+	var downloadsMaxed bool
 	// Start new downloads
-	for _, req := range d.downloads.queue {
-		if d.MaxActiveDownloads >= len(d.getRunningDownloads()) {
-			if req.Status == Queued {
-				d.startDownload(req)
+	for i, req := range d.Downloads.Queue {
+		switch req.Status {
+		case Queued:
+			if !downloadsMaxed && d.MaxActiveDownloads >= len(d.getRunningDownloads()) {
+				d.startDownload(i)
+			}
+		case Stopped:
+			if req.Response != nil {
+				var err = req.Response.Cancel()
+				if err != nil && err != context.Canceled {
+					req.setError(err)
+				}
+				req.Response = nil
+			}
+		case Downloading:
+			if req.Response == nil {
+				d.startDownload(i)
+			}
+		case Error:
+			if req.Response != nil && req.Err == nil {
+				var err = req.Response.Err()
+				var err2 = req.Response.Cancel()
+				if err != nil && err2 != context.Canceled {
+					err = err2
+				}
+				req.setError(err)
+			}
+		case Canceled:
+			if req.Response != nil {
+				err := req.Response.Cancel()
+				if err != nil && err != context.Canceled {
+					req.setError(err)
+				}
+				req.Response = nil
 			}
 		}
 	}

 	// Clean completed/canceled downloads
-	for i := 0; i < d.downloads.Len(); i++ {
-		if d.downloads.queue[i].Status == Complete || d.downloads.queue[i].Status == Canceled {
-			d.history.queue = append(d.history.queue, d.downloads.Pop(i))
+	for i := 0; i < d.Downloads.Len(); i++ {
+		if d.Downloads.Queue[i].Status == Complete || d.Downloads.Queue[i].Status == Canceled {
+			d.History.Queue = append(d.History.Queue, d.Downloads.Pop(i))
 			i--
 		}
 	}
+	d.Downloads.updateStatus()
 }

 func (d *Downloader) requestCompleted(r *Request) {
 	if r.Response.Err() == nil {
-		fmt.Println("removing from downloads")
-		d.downloads.remove(r)
+		log.Println("removing from downloads")
+		d.Downloads.remove(r)
 		r.Status = Complete
-		fmt.Println(r.TempPath, "!=", r.Filepath)
-		if r.TempPath != r.Filepath {
-			fmt.Println("renaming download to the completed dir")
-			os.Rename(r.TempPath, r.Filepath)
+		log.Println(r.TempPath, "!=", r.FilePath)
+		if r.TempPath != r.FilePath {
+			log.Println("renaming download to the completed dir")
+			os.Rename(r.TempPath, r.FilePath)
 		}
-		d.history.queue = append(d.history.queue, r)
+		d.History.Queue = append(d.History.Queue, r)
 	} else {
-		r.Status = Error
-		r.Error = r.Response.Err()
-		fmt.Println("fucking error:", r.Error)
+		r.setError(r.Response.Err())
+		log.Println("fucking error:", r.Err)
 	}
 }

 func (d *Downloader) download() {
 	for {
 		select {
-		case TIME := <-time.After(10 * time.Second):
-			fmt.Println(TIME)
-			for _, req := range d.downloads.queue {
-				fmt.Println(req.URL)
-				fmt.Println(req.Status)
-				fmt.Println(req.Response.ETA())
-			}
+		case <-time.After(10 * time.Second):
 			d.syncDownloads()

 		case r := <-d.NewRequest:
+			r.Progress = "" // not allowed when adding
 			d.addRequest(&r)
+			if d.OnAdd != nil {
+				d.OnAdd(d, r)
+			}

 		case r := <-d.requestDone:
-			fmt.Println("finishing request for", r.URL)
+			log.Println("finishing request for", r.URL)
 			d.requestCompleted(r)
+			if d.OnComplete != nil {
+				d.OnComplete(d, *r)
+			}
 		}
 	}
 }
+
+func (r *Request) insertCookies() {
+	if r.Jar == nil {
+		r.Jar = newCookieJar()
+	}
+	u, _ := url.Parse(r.URL)
+	for i, v := range r.Cookies {
+		r.Jar.SetCookies(&url.URL{
+			Scheme: u.Scheme,
+			Path:   v.Path,
+			Host:   v.Domain,
+		}, []*http.Cookie{&r.Cookies[i]})
+	}
+}
+
+func (d *Downloader) initStatus(mux *http.ServeMux) {
+	commonTmpls := template.New("root").Funcs(map[string]interface{}{
+		"shortenSHA256": func(hash string) string {
+			if len(hash) > 10 {
+				return hash[:10]
+			}
+			return hash
+		},
+	})
+	commonTmpls = template.Must(commonTmpls.New("header").Parse(bundled.Asset("header.tmpl")))
+	commonTmpls = template.Must(commonTmpls.New("footer").Parse(bundled.Asset("footer.tmpl")))
+
+	queueTmpl := template.Must(template.Must(commonTmpls.Clone()).New("queueTmpl").Parse(bundled.Asset("queue.tmpl")))
+	historyTmpl := template.Must(template.Must(commonTmpls.Clone()).New("historyTmpl").Parse(bundled.Asset("history.tmpl")))
+
+	for _, fn := range []string{
+		"favicon.ico",
+		"bootstrap.min.js",
+		"bootstrap.min.css",
+		"bootstrap-table.min.js",
+		"bootstrap-table.min.css",
+		"popper.min.js",
+		"jquery-3.2.1.slim.min.js",
+		"downloads.js",
+	} {
+		mux.Handle("/"+fn, bundled.HTTPHandlerFunc(fn))
+	}
+	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		var buf bytes.Buffer
+		d.Downloads.updateStatus()
+		if err := queueTmpl.Execute(&buf, struct {
+			Queue    []*Request
+			History  []*Request
+			Hostname string
+		}{
+			Queue:    d.Downloads.Queue,
+			History:  d.History.Queue,
+			Hostname: "gloader",
+		}); err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+		}
+		io.Copy(w, &buf)
+	})
+	mux.HandleFunc("/history", func(w http.ResponseWriter, r *http.Request) {
+		var buf bytes.Buffer
+		d.Downloads.updateStatus()
+		if err := historyTmpl.Execute(&buf, struct {
+			Queue    []*Request
+			History  []*Request
+			Hostname string
+		}{
+			Queue:    d.Downloads.Queue,
+			History:  d.History.Queue,
+			Hostname: "gloader",
+		}); err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+		}
+		io.Copy(w, &buf)
+	})
+}
--- a/go.mod
+++ b/go.mod
@ -8,5 +8,6 @@ require (
 	github.com/cavaliercoder/grab v2.0.0+incompatible
 	github.com/lordwelch/pathvalidate v0.0.0-20201012043703-54efa7ea1308
 	github.com/u-root/u-root v7.0.0+incompatible
+	github.com/wI2L/jettison v0.7.1
 	golang.org/x/net v0.0.0-20201209123823-ac852fbbde11
 )
--- a/go.sum
+++ b/go.sum
@ -1,6 +1,5 @@
 github.com/lordwelch/pathvalidate v0.0.0-20201012043703-54efa7ea1308 h1:CkcsZK6QYg59rc92eqU2h+FRjWltCIiplmEwIB05jfM=
 github.com/lordwelch/pathvalidate v0.0.0-20201012043703-54efa7ea1308/go.mod h1:4I4r5Y/LkH+34KACiudU+Q27ooz7xSDyVEuWAVKeJEQ=
-github.com/u-root/u-root v1.0.0 h1:3hJy0CG3mXIZtWRE+yrghG/3H0v8L1qEeZBlPr5nS9s=
 github.com/u-root/u-root v7.0.0+incompatible h1:u+KSS04pSxJGI5E7WE4Bs9+Zd75QjFv+REkjy/aoAc8=
 github.com/u-root/u-root v7.0.0+incompatible/go.mod h1:RYkpo8pTHrNjW08opNd/U6p/RJE7K0D8fXO0d47+3YY=
 golang.org/x/net v0.0.0-20201209123823-ac852fbbde11 h1:lwlPPsmjDKK0J6eG6xDWd5XPehI0R024zxjDnw3esPA=
--- a/goembed.go
+++ b/goembed.go
@ -0,0 +1,187 @@
+// +build ignore
+
+// goembed generates a Go source file from an input file.
+package main
+
+import (
+	"bufio"
+	"bytes"
+	"compress/gzip"
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"text/template"
+	"unicode/utf8"
+)
+
+var (
+	packageFlag = flag.String("package", "", "Go package name")
+	varFlag     = flag.String("var", "", "Go var name")
+	gzipFlag    = flag.Bool("gzip", false, "Whether to gzip contents")
+)
+
+func main() {
+	flag.Parse()
+
+	fmt.Printf("package %s\n\n", *packageFlag)
+
+	if *gzipFlag {
+		err := gzipPrologue.Execute(os.Stdout, map[string]interface{}{
+			"Args":    flag.Args(),
+			"VarName": *varFlag,
+		})
+		if err != nil {
+			log.Fatal(err)
+		}
+	}
+
+	if flag.NArg() > 0 {
+		fmt.Println("// Table of contents")
+		fmt.Printf("var %v = map[string][]byte{\n", *varFlag)
+		for i, filename := range flag.Args() {
+			fmt.Printf("\t%q: %s_%d,\n", filename, *varFlag, i)
+		}
+		fmt.Println("}")
+
+		// Using a separate variable for each []byte, instead of
+		// combining them into a single map literal, enables a storage
+		// optimization: the compiler places the data directly in the
+		// program's noptrdata section instead of the heap.
+		for i, filename := range flag.Args() {
+			if err := oneVar(fmt.Sprintf("%s_%d", *varFlag, i), filename); err != nil {
+				log.Fatal(err)
+			}
+		}
+	} else {
+		if err := oneVarReader(*varFlag, os.Stdin); err != nil {
+			log.Fatal(err)
+		}
+	}
+}
+
+func oneVar(varName, filename string) error {
+	f, err := os.Open(filename)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	return oneVarReader(varName, f)
+}
+
+func oneVarReader(varName string, r io.Reader) error {
+	// Generate []byte(<big string constant>) instead of []byte{<list of byte values>}.
+	// The latter causes a memory explosion in the compiler (60 MB of input chews over 9 GB RAM).
+	// Doing a string conversion avoids some of that, but incurs a slight startup cost.
+	if !*gzipFlag {
+		fmt.Printf(`var %s = []byte("`, varName)
+	} else {
+		var buf bytes.Buffer
+		gzw, _ := gzip.NewWriterLevel(&buf, gzip.BestCompression)
+		if _, err := io.Copy(gzw, r); err != nil {
+			return err
+		}
+		if err := gzw.Close(); err != nil {
+			return err
+		}
+		fmt.Printf("var %s []byte // set in init\n\n", varName)
+		fmt.Printf(`var %s_gzip = []byte("`, varName)
+		r = &buf
+	}
+
+	bufw := bufio.NewWriter(os.Stdout)
+	if _, err := io.Copy(&writer{w: bufw}, r); err != nil {
+		return err
+	}
+	if err := bufw.Flush(); err != nil {
+		return err
+	}
+	fmt.Println(`")`)
+	return nil
+}
+
+type writer struct {
+	w io.Writer
+}
+
+func (w *writer) Write(data []byte) (n int, err error) {
+	n = len(data)
+
+	for err == nil && len(data) > 0 {
+		// https://golang.org/ref/spec#String_literals: "Within the quotes, any
+		// character may appear except newline and unescaped double quote. The
+		// text between the quotes forms the value of the literal, with backslash
+		// escapes interpreted as they are in rune literals […]."
+		switch b := data[0]; b {
+		case '\\':
+			_, err = w.w.Write([]byte(`\\`))
+		case '"':
+			_, err = w.w.Write([]byte(`\"`))
+		case '\n':
+			_, err = w.w.Write([]byte(`\n`))
+
+		case '\x00':
+			// https://golang.org/ref/spec#Source_code_representation: "Implementation
+			// restriction: For compatibility with other tools, a compiler may
+			// disallow the NUL character (U+0000) in the source text."
+			_, err = w.w.Write([]byte(`\x00`))
+
+		default:
+			// https://golang.org/ref/spec#Source_code_representation: "Implementation
+			// restriction: […] A byte order mark may be disallowed anywhere else in
+			// the source."
+			const byteOrderMark = '\uFEFF'
+
+			if r, size := utf8.DecodeRune(data); r != utf8.RuneError && r != byteOrderMark {
+				_, err = w.w.Write(data[:size])
+				data = data[size:]
+				continue
+			}
+
+			_, err = fmt.Fprintf(w.w, `\x%02x`, b)
+		}
+		data = data[1:]
+	}
+
+	return n - len(data), err
+}
+
+var gzipPrologue = template.Must(template.New("").Parse(`
+import (
+	"bytes"
+	"compress/gzip"
+	"io/ioutil"
+)
+func init() {
+	var (
+		r *gzip.Reader
+		err error
+	)
+{{ if gt (len .Args) 0 }}
+{{ range $idx, $var := .Args }}
+{{ $n := printf "%s_%d" $.VarName $idx }}
+	r, err = gzip.NewReader(bytes.NewReader({{ $n }}_gzip))
+	if err != nil {
+		panic(err)
+	}
+	{{ $n }}, err = ioutil.ReadAll(r)
+	{{ $.VarName }}["{{ $var }}"] = {{ $n }}
+	r.Close()
+	if err != nil {
+		panic(err)
+	}
+{{ end }}
+{{ else }}
+	r, err = gzip.NewReader(bytes.NewReader({{ .VarName }}_gzip))
+	if err != nil {
+		panic(err)
+	}
+	{{ .VarName }}, err = ioutil.ReadAll(r)
+	r.Close()
+	if err != nil {
+		panic(err)
+	}
+{{ end }}
+}
+`))
--- a/grab/client.go
+++ b/grab/client.go
@ -267,6 +267,7 @@ func (c *Client) validateLocal(resp *Response) stateFunc {

 	if expectedSize >= 0 && expectedSize < resp.fi.Size() {
 		// remote size is known, is smaller than local size and we want to resume
+		fmt.Fprintln(os.Stderr, "validate\n")
 		resp.err = ErrBadLength
 		return c.closeResponse
 	}
@ -395,6 +396,7 @@ func (c *Client) readResponse(resp *Response) stateFunc {
 		// remote size is known
 		resp.sizeUnsafe += resp.bytesResumed
 		if resp.Request.Size > 0 && resp.Request.Size != resp.sizeUnsafe {
+			fmt.Fprintln(os.Stderr, "response\n")
 			resp.err = ErrBadLength
 			return c.closeResponse
 		}
@ -527,6 +529,7 @@ func (c *Client) copyFile(resp *Response) stateFunc {
 		discoveredSize := resp.bytesResumed + bytesCopied
 		atomic.StoreInt64(&resp.sizeUnsafe, discoveredSize)
 		if resp.Request.Size > 0 && resp.Request.Size != discoveredSize {
+			fmt.Fprintln(os.Stderr, "file\n")
 			resp.err = ErrBadLength
 			return c.closeResponse
 		}
--- a/main.go
+++ b/main.go
@ -1,6 +1,8 @@
 package main

 import (
+	"bufio"
+	"encoding/json"
 	"errors"
 	"flag"
 	"fmt"
@ -30,11 +32,75 @@ func main() {
 		log.Println(err)
 		os.Exit(1)
 	}
-	d := newDownloader()
-	d.DataDir = filepath.Join(gloaderHome, "data")
+	d := &Downloader{
+		OnAdd:      save,
+		OnComplete: save,
+		DataDir:    filepath.Join(gloaderHome, "data"),
+	}
+	loadQueue(d)
 	d.Start("tcp", ":8844")
 }

+func save(d *Downloader, r Request) {
+	var (
+		content []byte
+		err     error
+	)
+	content, err = json.Marshal(d.History.Queue)
+	if err != nil {
+		log.Println(err)
+		return
+	}
+	err = ioutil.WriteFile(filepath.Join(gloaderHome, "history.json"), content, 0o666)
+	if err != nil {
+		log.Println(err)
+		return
+	}
+	content, err = json.Marshal(d.Downloads.Queue)
+	if err != nil {
+		log.Println(err)
+		return
+	}
+	err = ioutil.WriteFile(filepath.Join(gloaderHome, "queue.json"), content, 0o666)
+	if err != nil {
+		log.Println(err)
+		return
+	}
+}
+
+func loadQueue(d *Downloader) {
+	var (
+		f       io.ReadCloser
+		err     error
+		decoder *json.Decoder
+	)
+	f, err = os.Open(filepath.Join(gloaderHome, "history.json"))
+	if err != nil {
+		log.Println(err)
+		return
+	}
+	decoder = json.NewDecoder(bufio.NewReader(f))
+	err = decoder.Decode(&d.History.Queue)
+	if err != nil {
+		log.Println(err)
+		return
+	}
+	f.Close()
+
+	f, err = os.Open(filepath.Join(gloaderHome, "queue.json"))
+	if err != nil {
+		log.Println(err)
+		return
+	}
+	decoder = json.NewDecoder(bufio.NewReader(f))
+	err = decoder.Decode(&d.Downloads.Queue)
+	if err != nil {
+		log.Println(err)
+		return
+	}
+	f.Close()
+}
+
 func mount() error {
 	var (
 		partUUIDb []byte
@ -72,15 +138,16 @@ func mount() error {
 		}
 		_, err = folder.Readdir(1)
 		if errors.Is(err, io.EOF) {
-			fmt.Printf("mount %s %s", partUUID, dataDir)
+			log.Printf("mount %s %s\n", partUUID, dataDir)
 			dev = findPartUUID(partUUID)
-			if err != nil {
-				return fmt.Errorf("error mounting datadir: %w", err)
-			}
 			err = syscall.Mount(dev.Path, dataDir, "ext4", 0, "")
 			if err != nil {
 				return fmt.Errorf("error mounting datadir: %w", err)
 			}
+			err = syscall.Mount(dev.Path, dataDir, "ext4", syscall.MS_SHARED|syscall.MS_REMOUNT, "")
+			if err != nil {
+				return fmt.Errorf("error remounting datadir: %w", err)
+			}
 			return nil
 		}
 		return fmt.Errorf("error mounting datadir: %w", err)
@ -88,56 +155,6 @@ func mount() error {
 	return fmt.Errorf("error mounting datadir: data dir %s is not a directory", dataDir)
 }

-// func findPartUUID(uuid string) (string, error) {
-// 	var dev string
-// 	err := filepath.Walk("/sys/block", func(path string, info os.FileInfo, err error) error {
-// 		if err != nil {
-// 			log.Printf("findPartUUID: %v", err)
-// 			return nil
-// 		}
-// 		if info.Mode()&os.ModeSymlink == 0 {
-// 			return nil
-// 		}
-// 		devname := "/dev/" + filepath.Base(path)
-// 		f, err := os.Open(devname)
-// 		if err != nil {
-// 			log.Printf("findPartUUID: %v", err)
-// 			return nil
-// 		}
-// 		defer f.Close()
-// 		if _, err := f.Seek(440, io.SeekStart); err != nil {
-// 			var se syscall.Errno
-// 			if errors.As(err, &se) && se == syscall.EINVAL {
-// 				// Seek()ing empty loop devices results in EINVAL.
-// 				return nil
-// 			}
-// 			log.Printf("findPartUUID: %v(%T)", err, err.(*os.PathError).Err)
-// 			return nil
-// 		}
-// 		var diskSig struct {
-// 			ID      uint32
-// 			Trailer uint16
-// 		}
-// 		if err := binary.Read(f, binary.LittleEndian, &diskSig); err != nil {
-// 			log.Printf("findPartUUID: %v", err)
-// 			return nil
-// 		}
-// 		if fmt.Sprintf("%08x", diskSig.ID) == uuid && diskSig.Trailer == 0 {
-// 			dev = devname
-// 			// TODO: abort early with sentinel error code
-// 			return nil
-// 		}
-// 		return nil
-// 	})
-// 	if err != nil {
-// 		return "", err
-// 	}
-// 	if dev == "" {
-// 		return "", fmt.Errorf("PARTUUID=%s not found", uuid)
-// 	}
-// 	return dev, nil
-// }
-
 type part struct {
 	UUID string
 	Path string