lordwelch/gloader
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: lordwelch:v0.0.1
Branches Tags
lordwelch:master
lordwelch:v0.0.7 lordwelch:v0.0.6 lordwelch:v0.0.5 lordwelch:v0.0.4 lordwelch:v0.0.3 lordwelch:v0.0.2 lordwelch:v0.0.1
...
compare: lordwelch:v0.0.5
Branches Tags
lordwelch:master
lordwelch:v0.0.7 lordwelch:v0.0.6 lordwelch:v0.0.5 lordwelch:v0.0.4 lordwelch:v0.0.3 lordwelch:v0.0.2 lordwelch:v0.0.1

4 Commits
v0.0.1 ... v0.0.5

Author SHA1 Message Date
lordwelch
36e79aa907 Refactor cookie handling 
Switch to jettison for JSON marshaling
Create an HTTP UI
Use goembed for assets
 2020-12-21 01:15:16 -08:00
lordwelch
bd0c3d2cc6 Load and save queue and history files 
Switch to using an additional filename and sub-directory field
Allow status in json decode/encode
Switch to using string for url instead of url.URL
use log instead of fmt for logging
Add basic status handlers for the queue and history
Add HTTP timeouts
Implement cookie handling
Ignore TempPath and FilePath when adding URLs, they are absolute paths
Ignore Status when adding URLs and status is not Paused
When determining the filename use the path from the final redirect
Use the correct TempPath when downloading
Actually add requests to the queue before starting them
 2020-12-13 01:05:17 -08:00
lordwelch
b1079eab13 Remove superfluous err check 2020-12-09 16:17:07 -08:00
lordwelch
1112d98ca9 Implement new findPartUUID that uses github.com/u-root/u-root/pkg/mount/gpt 2020-12-09 16:06:05 -08:00
20 changed files with 1205 additions and 194 deletions
Expand all files Collapse all files

10
assets/bootstrap-table.min.css vendored Normal file
View File

File diff suppressed because one or more lines are too long

10
assets/bootstrap-table.min.js vendored Normal file
View File

File diff suppressed because one or more lines are too long

7
assets/bootstrap.min.css vendored Normal file
View File

File diff suppressed because one or more lines are too long

7
assets/bootstrap.min.js vendored Normal file
View File

File diff suppressed because one or more lines are too long

85
assets/downloads.js Normal file
View File

@ -0,0 +1,85 @@
function updateDownloads(downloads) {
var dls = [];
var i = 0;
var tbody = $("<tbody></tbody>").attr("id", "table")
for (var d of downloads) {
var row = $("<tr></tr>").addClass("download").attr("index", i)
var index = $("<th></th>").attr("scope", "row").text(i)
var filename = $("<td></td>").text(d.subdir + "/" + d.filename).addClass("lastlog")
var url = $("<td></td>").text(d.url)
var status = $("<td></td>").addClass(d.status).css("cursor", "pointer").text(d.status + "\n" + d.error).on("click",{index: i, download: d}, evt => {
if (d.status != "Queue" || d.status != "Complete") {
setDownload([{
index: evt.data("index"),
status: "Queue",
priority: evt.data("d").priority
}])
}
})
var action = $("<td></td>").addClass("dropdown")
var button = $("<button>Actions</button>").addClass("btn btn-secondary dropdown-toggle").attr("type", "button").attr("data-toggle", "dropdown")
var menu = $("<div></div>").addClass("dropdown-menu")
var del = $('<a href="#">Delete</a>').addClass("dropdown-item").on("click",{index: i, download: d}, evt => {
if (confirm("confirm deletion of " + evt.data.download.filename)) {
deleteDownload([{
index: evt.data.index
}])
}
})
var resume = $('<a href="#">Resume</a>').addClass("dropdown-item").on("click",{index: i, download: d}, evt => {
setDownload([{
index: evt.data.index,
status: "Queue",
priority: evt.data.download.priority
}])
})
var stop = $('<a href="#">Stop</a>').addClass("dropdown-item").on("click",{index: i, download: d}, evt => {
setDownload([{
index: evt.data.index,
status: "Stop",
priority: evt.data.download.priority
}])
})
var na = $('<a href="#">N/A</a>').addClass("dropdown-item")
menu.append(del, resume, stop, na)
action.append(button, menu)
var progress = $("<td></td>").append(
$("<div></div>").addClass("progress").append(
$("<div></div>").addClass("progress-bar text-dark").css("width", (d.progress ?? "0") + '%').attr("id", "progress-" + i)
.text((d.progress ?? "0") + '%')
),
)
row.append(
index,
filename,
url,
status,
action,
progress
)
tbody.append(row)
i++;
}
$("#table").replaceWith(tbody)
}
function deleteDownload(data){
var xhr = new XMLHttpRequest();
xhr.open("DELETE", "http://gloader.narnian.us:8844/delete");
xhr.setRequestHeader("Content-Type", "application/javascript");
xhr.send(JSON.stringify(data));
}
function setDownload(data){
var xhr = new XMLHttpRequest();
xhr.open("POST", "http://gloader.narnian.us:8844/set");
xhr.setRequestHeader("Content-Type", "application/javascript");
xhr.send(JSON.stringify(data));
}

BIN
assets/favicon.ico Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 5.3 KiB

9
assets/footer.tmpl Normal file
View File

@ -0,0 +1,9 @@
</div>
<script src="/jquery-3.2.1.slim.min.js"></script>
<script src="/popper.min.js"></script>
<script src="/bootstrap.min.js"></script>
<script src="/bootstrap-table.min.js"></script>
</html>

78
assets/header.tmpl Normal file
View File

@ -0,0 +1,78 @@
<!DOCTYPE html>
<html lang="en">
<title>{{ .Hostname }} — gloader</title>
<link rel="stylesheet" href="/bootstrap.min.css" />
<link rel="stylesheet" href="/bootstrap-table.min.css" />
<style type="text/css">
.progress-bar:nth-child(5n) {
background-color: #337ab7;
}
.progress-bar:nth-child(5n+1) {
background-color: #5cb85c;
}
.progress-bar:nth-child(5n+2) {
background-color: #5bc0de;
}
.progress-bar:nth-child(5n+3) {
background-color: #f0ad4e;
}
.progress-bar:nth-child(5n+4) {
background-color: #d9534f;
}
.lastlog {
text-overflow: ellipsis;
white-space: nowrap;
overflow: hidden;
}
.Error {
color: red;
}
.container-fluid {
padding-left: 45px;
}
.navbar-default .navbar-text {
color: #777;
}
.navbar-default {
background-color: #f8f8f8;
border-color: #e7e7e7;
}
body {
font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
}
</style>
<nav class="navbar navbar-default">
<p style="margin-top: 0.25em; font-size: 18px"><a href="/">gokrazy</a><br>
<!-- <small style="font-size: 11px" class="text-muted">version { .BuildTimestamp }}</small> --></p>
<table class="navbar-text navbar-right" style="border-spacing: 10px 0; border-collapse: separate">
<tr>
<th>host</th>
<td>{{ .Hostname }}</td>
</tr>
<!-- <tr>
<th>kernel</th>
<td>{ .Kernel }}</td>
</tr>
{ if (ne .Model "") }}
<tr>
<th>model</th>
<td>{ .Model }}</td>
</tr>
{ end }}
{ if .EEPROM }}
<tr>
<th>EEPROM<br>(SHA256)</th>
<td>{ shortenSHA256 .EEPROM.PieepromSHA256 }}<br>{ shortenSHA256 .EEPROM.VL805SHA256 }}</td>
</tr>
{ end }} -->
</table>
</nav>
<div class="container-fluid">

52
assets/history.tmpl Normal file
View File

@ -0,0 +1,52 @@
{{ template "header" . }}
<div >
<!-- <h1>services</h1> -->
<table data-toggle="table" class="table table-hover">
<tbody id="table">
<thead>
<tr>
<th data-sortable="true" scope="col" width="5%">Index</th>
<th data-sortable="true" scope="col">Filename</th>
<th data-sortable="true" scope="col">URL</th>
<th data-sortable="true" scope="col">Status</th>
<th scope="col">Actions</th>
</tr>
</thead>
{{ range $idx, $request := .History }}
<tr class="download">
<th scope="row">{{ $idx }}</th>
<td class="lastlog">
{{ printf "%s/%s" $request.Subdir $request.Filename }}
</td>
<td>
{{ $request.URL }}
</td>
<td>
{{ printf "%v" $request.Status }}
{{ if $request.Error }}
{{ $request.Error }}
{{ end }}
</td>
<td class="dropdown">
<button class="btn btn-secondary dropdown-toggle" type="button" data-toggle="dropdown">Actions</button>
<div class="dropdown-menu">
<a class="dropdown-item" href="#" onclick="if (confirm('confirm deletion of ' + {{ printf "%s/%s" $request.Subdir $request.Filename }})) { deleteDownload([{index: {{ $idx }}, history: true }])}">Delete</a>
<a class="dropdown-item" href="#" onclick='setDownload([{ index: {{ $idx }}, status: "Queue", priority: {{ $request.Priority }} }])'>Resume</a>
<a class="dropdown-item" href="#" onclick='setDownload([{ index: {{ $idx }}, status: "Stop", priority: {{ $request.Priority }} }])'>Stop</a>
<a class="dropdown-item" title="{{ printf "%s/%s" $request.Subdir $request.Filename }}" href="/get/{{ printf "%s/%s" $request.Subdir $request.Filename }}">Retrieve</a>
</div>
</td>
</tr>
{{ end }}
</tbody>
</table>
</div>
<script src="/downloads.js"></script>
{{ template "footer" . }}

4
assets/jquery-3.2.1.slim.min.js vendored Normal file
View File

File diff suppressed because one or more lines are too long

5
assets/popper.min.js vendored Normal file
View File

File diff suppressed because one or more lines are too long

70
assets/queue.tmpl Normal file
View File

@ -0,0 +1,70 @@
{{ template "header" . }}
<div >
<!-- <h1>services</h1> -->
<table data-toggle="table" class="table">
<thead>
<tr>
<th data-sortable="true" scope="col" width="5%">Index</th>
<th data-sortable="true" scope="col">Filename</th>
<th data-sortable="true" scope="col">URL</th>
<th data-sortable="true" scope="col">Status</th>
<th scope="col">Actions</th>
<th data-sortable="true" scope="col">Progress</th>
</tr>
</thead>
<tbody id="table">
{{ range $idx, $request := .Queue }}
<tr class="download">
<th scope="row">{{ $idx }}</th>
<td class="lastlog">
{{ printf "%s/%s" $request.Subdir $request.Filename }}
</td>
<td>
{{ $request.URL }}
</td>
<td>
{{ printf "%v" $request.Status }}
{{ if $request.Error }}
{{ $request.Error }}
{{ end }}
</td>
<td class="dropdown">
<button class="btn btn-secondary dropdown-toggle" type="button" data-toggle="dropdown">Actions</button>
<div class="dropdown-menu">
<a class="dropdown-item" href="#" onclick="if (confirm('confirm deletion of ' + {{ printf "%s/%s" $request.Subdir $request.Filename }})) { deleteDownload([{index: {{ $idx }} }])}">Delete</a>
<a class="dropdown-item" href="#" onclick='setDownload([{ index: {{ $idx }}, status: "Queue", priority: {{ $request.Priority }} }])'>Resume</a>
<a class="dropdown-item" href="#" onclick='setDownload([{ index: {{ $idx }}, status: "Stop", priority: {{ $request.Priority | js}} }])'>Stop</a>
<a class="dropdown-item" href="/get/{{ printf "%s/%s" $request.Subdir $request.Filename }}">Retrieve</a>
</div>
</td>
<td>
<div class="progress">
<div id="progress-{{ $idx }}" class="progress-bar text-dark" style="width: 0{{ $request.Progress }}%">0{{ $request.Progress }}%</div>
</div>
</td>
</tr>
{{ end }}
</tbody>
</table>
</div>
<script src="/downloads.js"></script>
<script type="text/javascript">
function setDownloads() {
var xhr = new XMLHttpRequest();
xhr.open("GET", "http://gloader.narnian.us:8844/queued");
xhr.setRequestHeader("Accept", "application/javascript");
xhr.onload = evt => {
updateDownloads(JSON.parse(evt.target.responseText));
};
xhr.send();
}
var interval = setInterval(setDownloads, 5 * 1000)
</script>
{{ template "footer" . }}

3
bundle.go Normal file
View File

@ -0,0 +1,3 @@
package main
//go:generate sh -c "go run goembed.go -package bundled -var assets assets/history.tmpl assets/downloads.js assets/header.tmpl assets/footer.tmpl assets/queue.tmpl assets/status.tmpl assets/favicon.ico assets/bootstrap.min.css assets/bootstrap-table.min.css assets/bootstrap-table.min.js assets/bootstrap.min.js assets/popper.min.js assets/jquery-3.2.1.slim.min.js > bundled/GENERATED_bundled.go && gofmt -w bundled/GENERATED_bundled.go"

18
bundled/bundled.go Normal file
View File

@ -0,0 +1,18 @@
package bundled
import (
"bytes"
"net/http"
"time"
)
func Asset(basename string) string {
return string(assets["assets/"+basename])
}
func HTTPHandlerFunc(basename string) http.Handler {
modTime := time.Now()
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
http.ServeContent(w, r, basename, modTime, bytes.NewReader(assets["assets/"+basename]))
})
}

715
downloader.go
View File

@ -1,10 +1,16 @@
package main package main
import ( import (
"bytes"
"context"
"encoding/json" "encoding/json"
"errors" "errors"
"fmt" "fmt"
"html/template"
"io"
"io/ioutil" "io/ioutil"
"log"
"math"
"mime" "mime"
"net" "net"
"net/http" "net/http"
@ -18,21 +24,23 @@ import (
"strings" "strings"
"time" "time"
"git.narnian.us/lordwelch/gloader/bundled"
"github.com/cavaliercoder/grab" "github.com/cavaliercoder/grab"
"github.com/lordwelch/pathvalidate" "github.com/lordwelch/pathvalidate"
"github.com/wI2L/jettison"
"golang.org/x/net/publicsuffix" "golang.org/x/net/publicsuffix"
) )
var ( var (
DefaultCookieJar = newCookieJar()
DefaultGrabClient = grab.NewClient()
DefaultMaxActiveDownloads = 4 DefaultMaxActiveDownloads = 4
ErrUnsupportedScheme = errors.New("unsupported scheme") ErrUnsupportedScheme = errors.New("unsupported scheme")
) )
type Priority uint8 type Priority int
type Status uint8 type Status int
const ( const (
Highest Priority = iota Highest Priority = iota
@ -56,57 +64,154 @@ type Downloader struct {
DownloadDir string DownloadDir string
CompleteDir string CompleteDir string
InfoDir string InfoDir string
Grab *grab.Client
Jar http.CookieJar
MaxActiveDownloads int MaxActiveDownloads int
Server *http.Server Server *http.Server
downloads RequestQueue Downloads RequestQueue
history RequestQueue History RequestQueue
NewRequest chan Request NewRequest chan Request
requestDone chan *Request requestDone chan *Request
OnComplete func(d *Downloader, r Request)
OnAdd func(d *Downloader, r Request)
} }
type Request struct { type Request struct {
URL url.URL `json:"url"` URL string `json:"url"`
Cookies []http.Cookie `json:"cookies"` Cookies []http.Cookie `json:"cookies"`
ForceDownload bool `json:"forceDownload"` ForceDownload bool `json:"forceDownload"`
Status Status `json:"-"` Status Status `json:"status"`
Priority Priority `json:"priority"` Priority Priority `json:"priority"`
Filepath string `json:"filepath"` FilePath string `json:"filepath"`
Filename string `json:"filename"`
Subdir string `json:"subdir"`
TempPath string `json:"tempPath"` TempPath string `json:"tempPath"`
Response *grab.Response `json:"-"` Response *grab.Response `json:"-"`
Error error `json:"-"` Error string `json:"error"`
CompletedDate time.Time Err error `json:"-"`
CompletedDate time.Time `json:"completedDate"`
Jar http.CookieJar `json:"-"`
Progress string `json:"progress,omitempty"`
grab *grab.Client
} }
type RequestQueue struct { type RequestQueue struct {
queue []*Request Queue []*Request
URLSort bool URLSort bool
DateSort bool DateSort bool
} }
func (p Priority) MarshalJSON() ([]byte, error) {
var v string
switch p {
default:
v = "Medium"
case Medium:
v = "Medium"
case Low:
v = "Low"
case High:
v = "High"
case Highest:
v = "Highest"
}
return json.Marshal(v)
}
func (p *Priority) UnmarshalJSON(b []byte) error {
var (
v int
s string
)
if err := json.Unmarshal(b, &v); err == nil {
*p = Priority(v)
return nil
}
if err := json.Unmarshal(b, &s); err != nil {
return err
}
switch strings.ToLower(s) {
default:
*p = Medium
case "medium":
*p = Medium
case "low":
*p = Low
case "high":
*p = High
case "highest":
*p = Highest
}
return nil
}
func (s *Status) UnmarshalJSON(b []byte) error {
var v string
if err := json.Unmarshal(b, &v); err != nil {
return err
}
switch strings.ToLower(v) {
default:
*s = Queued
case "queued", "queue":
*s = Queued
case "complete", "completed":
*s = Complete
case "stop", "stopped":
*s = Stopped
case "download", "downloading":
*s = Downloading
case "error":
*s = Error
case "cancel", "canceled":
*s = Canceled
}
return nil
}
func (s Status) MarshalJSON() ([]byte, error) {
return json.Marshal(s.String())
}
func (s Status) String() string {
switch s {
default:
return "Queued"
case Queued:
return "Queued"
case Complete:
return "Complete"
case Stopped:
return "Stopped"
case Downloading:
return "Downloading"
case Error:
return "Error"
case Canceled:
return "Canceled"
}
}
func (rq RequestQueue) Less(i, j int) bool { func (rq RequestQueue) Less(i, j int) bool {
ii := 0 ii := 0
jj := 0 jj := 0
if rq.queue[i].ForceDownload { if rq.Queue[i].ForceDownload {
ii = 1 ii = 1
} }
if rq.queue[j].ForceDownload { if rq.Queue[j].ForceDownload {
jj = 1 jj = 1
} }
if ii < jj { if ii < jj {
return true return true
} }
if rq.queue[i].Priority < rq.queue[j].Priority { if rq.Queue[i].Priority < rq.Queue[j].Priority {
return true return true
} }
if rq.DateSort && rq.queue[i].CompletedDate.Before(rq.queue[j].CompletedDate) { if rq.DateSort && rq.Queue[i].CompletedDate.Before(rq.Queue[j].CompletedDate) {
return true return true
} }
if rq.URLSort && rq.queue[i].URL.String() < rq.queue[j].URL.String() { if rq.URLSort && rq.Queue[i].URL < rq.Queue[j].URL {
return true return true
} }
@ -114,44 +219,73 @@ func (rq RequestQueue) Less(i, j int) bool {
} }
func (rq RequestQueue) Len() int { func (rq RequestQueue) Len() int {
return len(rq.queue) return len(rq.Queue)
} }
func (rq RequestQueue) Swap(i, j int) { func (rq RequestQueue) Swap(i, j int) {
rq.queue[i], rq.queue[j] = rq.queue[j], rq.queue[i] rq.Queue[i], rq.Queue[j] = rq.Queue[j], rq.Queue[i]
} }
func (rq *RequestQueue) Pop(i int) *Request { func (rq *RequestQueue) Pop(i int) *Request {
r := rq.queue[i] r := rq.Queue[i]
copy(rq.queue[i:], rq.queue[i+1:]) copy(rq.Queue[i:], rq.Queue[i+1:])
rq.queue[len(rq.queue)-1] = nil rq.Queue[len(rq.Queue)-1] = nil
rq.queue = rq.queue[:len(rq.queue)-1] rq.Queue = rq.Queue[:len(rq.Queue)-1]
return r return r
} }
func (rq *RequestQueue) remove(r *Request) { func (rq *RequestQueue) remove(r *Request) {
for i, req := range rq.queue { for i, req := range rq.Queue {
if req == r { if req == r {
copy(rq.queue[i:], rq.queue[i+1:]) copy(rq.Queue[i:], rq.Queue[i+1:])
rq.queue[len(rq.queue)-1] = nil rq.Queue[len(rq.Queue)-1] = nil
rq.queue = rq.queue[:len(rq.queue)-1] rq.Queue = rq.Queue[:len(rq.Queue)-1]
break break
} }
} }
} }
func (rq *RequestQueue) updateStatus() {
for _, req := range rq.Queue {
if req.Response != nil {
req.Progress = fmt.Sprintf("%.2f", math.Abs(req.Response.Progress()*100))
}
}
}
func (r *Request) setError(err error) {
if err != nil {
r.Status = Error
r.Err = err
r.Error = err.Error()
} else {
r.Status = Paused
r.Err = nil
r.Error = ""
}
}
func (r Request) Delete() error {
var err, ret error
if r.Response != nil {
err = r.Response.Cancel()
if err != nil && err != context.Canceled {
ret = err
}
}
_ = os.Remove(r.TempPath)
err = os.Remove(r.FilePath)
if err != nil && err != os.ErrNotExist {
ret = err
}
return ret
}
func newCookieJar() http.CookieJar { func newCookieJar() http.CookieJar {
c, _ := cookiejar.New(&cookiejar.Options{PublicSuffixList: publicsuffix.List}) c, _ := cookiejar.New(&cookiejar.Options{PublicSuffixList: publicsuffix.List})
return c return c
} }
func newDownloader() *Downloader {
return &Downloader{
Jar: DefaultCookieJar,
Grab: DefaultGrabClient,
}
}
func (d *Downloader) Start(network, address string) { func (d *Downloader) Start(network, address string) {
var ( var (
listener net.Listener listener net.Listener
@ -178,11 +312,10 @@ func (d *Downloader) Start(network, address string) {
ReadTimeout: 2 * time.Minute, ReadTimeout: 2 * time.Minute,
WriteTimeout: 2 * time.Minute, WriteTimeout: 2 * time.Minute,
} }
} }
if d.DataDir == "" { if d.DataDir == "" {
d.DataDir = "/perm/downloader" d.DataDir = "/perm/gloader"
} }
if d.DownloadDir == "" { if d.DownloadDir == "" {
@ -193,26 +326,146 @@ func (d *Downloader) Start(network, address string) {
d.CompleteDir = path.Join(d.DataDir, "Complete") d.CompleteDir = path.Join(d.DataDir, "Complete")
} }
fmt.Println(d.DataDir) log.Println(d.DataDir)
fmt.Println(d.DownloadDir) log.Println(d.DownloadDir)
fmt.Println(d.CompleteDir) log.Println(d.CompleteDir)
os.MkdirAll(d.DataDir, 0777) _ = os.MkdirAll(d.DataDir, 0777)
os.MkdirAll(d.DownloadDir, 0777) _ = os.MkdirAll(d.DownloadDir, 0777)
os.MkdirAll(d.CompleteDir, 0777) _ = os.MkdirAll(d.CompleteDir, 0777)
listener, err = net.Listen(network, address) listener, err = net.Listen(network, address)
if err != nil { if err != nil {
panic(err) panic(err)
} }
fmt.Println("adding /add handler") log.Println("adding http handlers")
// mux.HandleFunc("/", d.UI)
mux.HandleFunc("/add", d.restAddDownload) d.initStatus(mux)
mux.HandleFunc("/add", d.restAddDownload)
mux.HandleFunc("/queued", d.restStatus(true))
mux.HandleFunc("/completed", d.restStatus(false))
mux.HandleFunc("/set", d.restSetDownloadStatus)
mux.HandleFunc("/delete", d.restDelete)
mux.Handle("/get/", http.StripPrefix("/get/", http.FileServer(http.Dir(d.CompleteDir))))
log.Println("starting main go routine")
fmt.Println("starting main go routine")
go d.download() go d.download()
fmt.Println("serving http server") log.Println("serving http server")
d.Server.Serve(listener) _ = d.Server.Serve(listener)
}
func httpMethodNotAllowed(w http.ResponseWriter, r *http.Request, method string) {
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
w.Header().Set("X-Content-Type-Options", "nosniff")
w.Header().Add("Allow", method)
w.WriteHeader(http.StatusMethodNotAllowed)
fmt.Fprintf(w, "HTTP Error 405 – Method Not Allowed\nOnly %s method is allowed\n", method)
log.Printf("HTTP Error 405 – Method Not Allowed\nOnly %s method is allowed\n", method)
}
func (d *Downloader) restDelete(w http.ResponseWriter, r *http.Request) {
var (
err error
index []struct {
Index int
History bool
}
ret []Request
)
if r.Method != http.MethodDelete {
httpMethodNotAllowed(w, r, http.MethodDelete)
return
}
err = json.NewDecoder(r.Body).Decode(&index)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
sort.Slice(index, func(i, j int) bool {
return index[i].Index < index[j].Index
})
for x, i := range index {
if i.History {
if i.Index >= d.History.Len() || i.Index < 0 {
http.Error(w, fmt.Sprintf("slice index out of bounds. index: %d length of slice: %d", i.Index, d.Downloads.Len()), http.StatusBadRequest)
return
}
ret = append(ret, *d.History.Pop(i.Index - x))
ret[len(ret)-1].Delete()
} else {
if i.Index >= d.Downloads.Len() || i.Index < 0 {
http.Error(w, fmt.Sprintf("slice index out of bounds. index: %d length of slice: %d", i.Index, d.Downloads.Len()), http.StatusBadRequest)
return
}
ret = append(ret, *d.Downloads.Pop(i.Index - x))
ret[len(ret)-1].Delete()
}
}
d.syncDownloads()
v, err := jettison.MarshalOpts(ret, jettison.DenyList([]string{"cookies"}))
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
w.Header().Add("Content-Type", "application/json; charset=utf-8")
w.WriteHeader(http.StatusOK)
w.Write(v)
}
func (d *Downloader) restSetDownloadStatus(w http.ResponseWriter, r *http.Request) {
var (
err error
index []struct {
Index int
Status Status
Priority Priority
}
)
if r.Method != http.MethodPost {
httpMethodNotAllowed(w, r, http.MethodPost)
return
}
err = json.NewDecoder(r.Body).Decode(&index)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
for _, i := range index {
if i.Index >= d.Downloads.Len() || i.Index < 0 {
http.Error(w, fmt.Sprintf("slice index out of bounds. index: %d length of slice: %d", i.Index, d.Downloads.Len()), http.StatusBadRequest)
return
}
r := d.Downloads.Queue[i.Index]
r.Priority = i.Priority
r.Status = i.Status
}
d.syncDownloads()
}
func (d *Downloader) restStatus(q bool) func(http.ResponseWriter, *http.Request) {
return func(w http.ResponseWriter, r *http.Request) {
var queue = &d.History
if q {
d.Downloads.updateStatus()
queue = &d.Downloads
}
if r.Method != http.MethodGet {
httpMethodNotAllowed(w, r, http.MethodGet)
return
}
queue.updateStatus()
v, err := jettison.MarshalOpts(queue.Queue, jettison.DenyList([]string{"cookies"}))
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
w.Header().Add("Content-Type", "application/json; charset=utf-8")
w.WriteHeader(http.StatusOK)
w.Write(v)
}
} }
func (d *Downloader) restAddDownload(w http.ResponseWriter, r *http.Request) { func (d *Downloader) restAddDownload(w http.ResponseWriter, r *http.Request) {
@ -221,12 +474,7 @@ func (d *Downloader) restAddDownload(w http.ResponseWriter, r *http.Request) {
err error err error
) )
if r.Method != http.MethodPost { if r.Method != http.MethodPost {
w.Header().Set("Content-Type", "text/plain; charset=utf-8") httpMethodNotAllowed(w, r, http.MethodPost)
w.Header().Set("X-Content-Type-Options", "nosniff")
w.Header().Add("Allow", http.MethodPost)
w.WriteHeader(http.StatusMethodNotAllowed)
fmt.Fprintln(w, "HTTP Error 405 – Method Not Allowed\nOnly POST method is allowed")
fmt.Println("HTTP Error 405 – Method Not Allowed\nOnly POST method is allowed")
return return
} }
// TODO fail only on individual requests // TODO fail only on individual requests
@ -236,36 +484,53 @@ func (d *Downloader) restAddDownload(w http.ResponseWriter, r *http.Request) {
return return
} }
for _, req := range requests { for _, req := range requests {
req.TempPath = "" req.TempPath = "" // not allowed via REST API
fmt.Println("adding request", req.URL.String()) req.FilePath = "" // not allowed via REST API
if req.Status != Paused {
req.Status = Queued
}
log.Println("adding request", req.URL)
d.NewRequest <- req d.NewRequest <- req
} }
w.WriteHeader(http.StatusOK) w.WriteHeader(http.StatusOK)
} }
func (d Downloader) getContentDispsition(r Request) string { func (d Downloader) getNameFromHEAD(r Request) string {
var ( var (
err error err error
re *http.Response re *http.Response
p map[string]string p map[string]string
) )
r.insertCookies()
ht := &http.Client{ ht := &http.Client{
Jar: d.Jar, Jar: r.Jar,
Timeout: 30 * time.Second, Timeout: 30 * time.Second,
Transport: &http.Transport{
Dial: (&net.Dialer{
Timeout: 5 * time.Second,
KeepAlive: 30 * time.Second,
}).Dial,
TLSHandshakeTimeout: 5 * time.Second,
ResponseHeaderTimeout: 5 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
},
} }
re, err = ht.Head(r.URL.String()) re, err = ht.Head(r.URL)
if err != nil { if err != nil {
return "" return ""
} }
if re.StatusCode < 200 || re.StatusCode > 299 {
return ""
}
re.Body.Close() re.Body.Close()
_, p, err = mime.ParseMediaType(re.Header.Get("Content-Disposition")) _, p, err = mime.ParseMediaType(re.Header.Get("Content-Disposition"))
if err != nil { if err == nil {
return "" if f, ok := p["filename"]; ok {
return f
}
} }
if f, ok := p["filename"]; ok { return path.Base(re.Request.URL.Path)
return f
}
return ""
} }
// getFilename checks the provided filepath // getFilename checks the provided filepath
@ -273,28 +538,29 @@ func (d Downloader) getContentDispsition(r Request) string {
// if not set uses the basename of the url // if not set uses the basename of the url
// and sanitizes the filename using github.com/lordwelch/pathvalidate // and sanitizes the filename using github.com/lordwelch/pathvalidate
func (d *Downloader) getFilename(r *Request) { func (d *Downloader) getFilename(r *Request) {
fmt.Println("Determining filename") log.Println("Determining filename")
r.Filepath = filepath.Clean(r.Filepath) r.Filename = filepath.Clean(r.Filename)
if r.Filepath == "." { if r.Filename == "." {
fmt.Println("filename is empty, testing head request") log.Println("filename is empty, testing head request")
r.Filepath = d.getContentDispsition(*r) r.Filename = d.getNameFromHEAD(*r)
fmt.Println("path from head request:", r.Filepath) log.Println("path from head request:", r.Filename)
if r.Filepath == "" { if r.Filename == "" {
r.Filepath, _ = url.PathUnescape(filepath.Base(r.URL.Path)) u, _ := url.Parse(r.URL)
r.Filename, _ = url.PathUnescape(filepath.Base(u.Path))
} }
} }
r.Filepath, _ = pathvalidate.SanitizeFilename(r.Filepath, '_') r.Filename, _ = pathvalidate.SanitizeFilename(r.Filename, '_')
r.Filepath = filepath.Join(d.DownloadDir, r.Filepath) // r.Filename = filepath.Join(d.CompleteDir, r.Filename)
// if filepath.IsAbs(r.Filepath) { // should already exist // if filepath.IsAbs(r.Filename) { // should already exist
// dir, file := filepath.Split(r.Filepath) // dir, file := filepath.Split(r.Filename)
// // someone is trying to be sneaky (or someone changed the CompleteDir), change path to the correct dir // // someone is trying to be sneaky (or someone changed the CompleteDir), change path to the correct dir
// if dir != filepath.Clean(d.CompleteDir) { // if dir != filepath.Clean(d.CompleteDir) {
// r.Filepath = filepath.Join(d.CompleteDir, file) // r.Filename = filepath.Join(d.CompleteDir, file)
// } // }
// return // return
// } // }
fmt.Println("result path:", r.Filepath) log.Println("result path:", r.Filename)
} }
func getNewFilename(dir, name string) string { func getNewFilename(dir, name string) string {
@ -302,15 +568,16 @@ func getNewFilename(dir, name string) string {
err error err error
index = 1 index = 1
) )
fmt.Println("getfilename", dir, name) log.Println("getfilename", dir, name)
ext := filepath.Ext(name) ext := filepath.Ext(name)
base := strings.TrimSuffix(name, ext) base := strings.TrimSuffix(name, ext)
fmt.Println("stat", filepath.Join(dir, name)) log.Println("stat", filepath.Join(dir, name))
_, err = os.Stat(filepath.Join(dir, name)) _, err = os.Stat(filepath.Join(dir, name))
for err == nil { for err == nil {
name = strings.TrimRight(base+"."+strconv.Itoa(index)+ext, ".") name = strings.TrimRight(base+"."+strconv.Itoa(index)+ext, ".")
fmt.Println("stat", filepath.Join(dir, name)) log.Println("stat", filepath.Join(dir, name))
_, err = os.Stat(filepath.Join(dir, name)) _, err = os.Stat(filepath.Join(dir, name))
index++
} }
if os.IsNotExist(err) { if os.IsNotExist(err) {
return filepath.Join(dir, name) return filepath.Join(dir, name)
@ -318,93 +585,120 @@ func getNewFilename(dir, name string) string {
panic(err) // other path error panic(err) // other path error
} }
func (d Downloader) getDownloadFilename(r *Request) { func (d Downloader) getTempFilename(r *Request) {
if r.TempPath == "" { if r.TempPath == "" {
f, err := ioutil.TempFile(d.DownloadDir, filepath.Base(r.Filepath)) f, err := ioutil.TempFile(d.DownloadDir, filepath.Base(r.Filename))
if err != nil { if err != nil {
fmt.Printf("request for %v failed: %v", r.URL.String(), err) log.Printf("request for %v failed: %v", r.URL, err)
} }
r.TempPath = f.Name()
f.Close() f.Close()
r.TempPath = filepath.Join(d.DownloadDir, f.Name())
} }
f, err := os.OpenFile(r.Filepath, os.O_CREATE|os.O_EXCL, 0666) os.MkdirAll(filepath.Dir(r.FilePath), 0o777)
f, err := os.OpenFile(r.Filename, os.O_CREATE|os.O_EXCL, 0666)
if err != nil { if err != nil {
return return
} }
f.Close() f.Close()
} }
func (d Downloader) SearchDownloads(u url.URL) int { func (d Downloader) SearchDownloads(u string) int {
for i, req := range d.downloads.queue { for i, req := range d.Downloads.Queue {
if req.URL.String() == u.String() { if req.URL == u {
return i return i
} }
} }
return -1 return -1
} }
func (d Downloader) SearchHistory(u url.URL) int { func (d Downloader) SearchHistory(u string) int {
for i, req := range d.history.queue { for i, req := range d.History.Queue {
if req.URL.String() == u.String() { if req.URL == u {
return i return i
} }
} }
return -1 return -1
} }
func (d Downloader) FindRequest(u url.URL) *Request { func (d Downloader) FindRequest(u string) *Request {
if i := d.SearchDownloads(u); i >= 0 { if i := d.SearchDownloads(u); i >= 0 {
return d.downloads.queue[i] return d.Downloads.Queue[i]
} }
if i := d.SearchHistory(u); i >= 0 { if i := d.SearchHistory(u); i >= 0 {
return d.history.queue[i] return d.History.Queue[i]
} }
return nil return nil
} }
func (d *Downloader) addRequest(r *Request) { func (d *Downloader) addRequest(r *Request) {
fmt.Println("adding download for", r.URL.String()) log.Println("adding download for", r.URL)
req := d.FindRequest(r.URL) req := d.FindRequest(r.URL)
d.getFilename(r) d.getFilename(r)
if req != nil { // url alread added if req != nil { // url alread added
fmt.Println("URL is already added", r.URL.String()) log.Println("URL is already added", r.URL)
if fi, err := os.Stat(r.Filepath); filepath.Base(req.Filepath) == filepath.Base(r.Filepath) || (err == nil && fi.Name() == filepath.Base(r.Filepath) && fi.Size() != 0) { // filepath has been found, should this check for multiple downloads of the same url or let the download name increment automatically return
fmt.Println("file already exists", r.Filepath) // if fi, err := os.Stat(r.Filepath); filepath.Base(req.Filepath) == filepath.Base(r.Filepath) || (err == nil && fi.Name() == filepath.Base(r.Filepath) && fi.Size() != 0) { // filepath has been found, should this check for multiple downloads of the same url or let the download name increment automatically
//getNewFilename(d.CompleteDir, filepath.Base(r.Filepath)) // log.Println("file already exists", r.Filepath)
d.validate(*r) // TODO, should also check to see if it seems like it is similar, (check first k to see if it is the same file?? leave option to user) // d.validate(*r) // TODO, should also check to see if it seems like it is similar, (check first k to see if it is the same file?? leave option to user)
// return
// }
}
r.FilePath = getNewFilename(d.CompleteDir, filepath.Join(r.Subdir, r.Filename))
d.Downloads.Queue = append(d.Downloads.Queue, r)
if len(d.getRunningDownloads()) < d.MaxActiveDownloads {
d.startDownload(d.Downloads.Len() - 1)
}
}
// func (d *Downloader) validate(r Request) {
// //TODO
// }
func (d *Downloader) startDownload(i int) {
var (
r *Request
req *grab.Request
err error
)
r = d.Downloads.Queue[i]
r.insertCookies()
d.getTempFilename(r)
log.Println("starting download for", r.URL, "to", r.TempPath)
// d.Downloads.Queue = append(d.Downloads.Queue, r)
if r.Response == nil || r.Response.Err() != nil {
req, err = grab.NewRequest(r.TempPath, r.URL)
if err != nil {
r.setError(err)
return return
} }
} else { // new request, download link
r.Filepath = getNewFilename(d.CompleteDir, filepath.Base(r.Filepath))
d.downloads.queue = append(d.downloads.queue, r)
}
if len(d.getRunningDownloads()) < d.MaxActiveDownloads {
d.startDownload(r)
}
}
func (d *Downloader) validate(r Request) {
//TODO
}
func (d *Downloader) startDownload(r *Request) {
fmt.Println("starting download for", r.URL.String())
d.getDownloadFilename(r)
req, err := grab.NewRequest(r.TempPath, r.URL.String())
if err != nil {
r.Status = Error
r.Error = err
return
} }
r.Status = Downloading r.Status = Downloading
r.Response = d.Grab.Do(req)
if r.grab == nil {
r.grab = &grab.Client{
HTTPClient: &http.Client{
Jar: r.Jar,
Transport: &http.Transport{
Dial: (&net.Dialer{
Timeout: 10 * time.Second,
KeepAlive: 30 * time.Second,
}).Dial,
TLSHandshakeTimeout: 5 * time.Second,
ResponseHeaderTimeout: 5 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
},
},
}
}
r.Response = r.grab.Do(req)
go func(r *Request) { go func(r *Request) {
fmt.Println("wait for download") log.Println("wait for download")
fmt.Println(r.Response.IsComplete()) log.Println(r.Response.IsComplete())
r.Response.Wait() r.Response.Wait()
fmt.Println("download completed for", r.URL) log.Println("download completed for", r.URL)
d.requestDone <- r d.requestDone <- r
}(r) }(r)
} }
@ -413,7 +707,7 @@ func (d Downloader) getRunningDownloads() []*Request {
var ( var (
running = make([]*Request, 0, d.MaxActiveDownloads) running = make([]*Request, 0, d.MaxActiveDownloads)
) )
for _, req := range d.downloads.queue { for _, req := range d.Downloads.Queue {
if req.Status == Downloading && req.Response != nil { if req.Status == Downloading && req.Response != nil {
running = append(running, req) running = append(running, req)
} }
@ -425,61 +719,168 @@ func (d *Downloader) syncDownloads() {
if len(d.getRunningDownloads()) >= d.MaxActiveDownloads { if len(d.getRunningDownloads()) >= d.MaxActiveDownloads {
return return
} }
sort.Stable(d.downloads) sort.Stable(d.Downloads)
var downloadsMaxed bool
// Start new downloads // Start new downloads
for _, req := range d.downloads.queue { for i, req := range d.Downloads.Queue {
if d.MaxActiveDownloads >= len(d.getRunningDownloads()) { switch req.Status {
if req.Status == Queued { case Queued:
d.startDownload(req) if !downloadsMaxed && d.MaxActiveDownloads >= len(d.getRunningDownloads()) {
d.startDownload(i)
}
case Stopped:
if req.Response != nil {
var err = req.Response.Cancel()
if err != nil && err != context.Canceled {
req.setError(err)
}
req.Response = nil
}
case Downloading:
if req.Response == nil {
d.startDownload(i)
}
case Error:
if req.Response != nil && req.Err == nil {
var err = req.Response.Err()
var err2 = req.Response.Cancel()
if err != nil && err2 != context.Canceled {
err = err2
}
req.setError(err)
}
case Canceled:
if req.Response != nil {
err := req.Response.Cancel()
if err != nil && err != context.Canceled {
req.setError(err)
}
req.Response = nil
} }
} }
} }
// Clean completed/canceled downloads // Clean completed/canceled downloads
for i := 0; i < d.downloads.Len(); i++ { for i := 0; i < d.Downloads.Len(); i++ {
if d.downloads.queue[i].Status == Complete || d.downloads.queue[i].Status == Canceled { if d.Downloads.Queue[i].Status == Complete || d.Downloads.Queue[i].Status == Canceled {
d.history.queue = append(d.history.queue, d.downloads.Pop(i)) d.History.Queue = append(d.History.Queue, d.Downloads.Pop(i))
i-- i--
} }
} }
d.Downloads.updateStatus()
} }
func (d *Downloader) requestCompleted(r *Request) { func (d *Downloader) requestCompleted(r *Request) {
if r.Response.Err() == nil { if r.Response.Err() == nil {
fmt.Println("removing from downloads") log.Println("removing from downloads")
d.downloads.remove(r) d.Downloads.remove(r)
r.Status = Complete r.Status = Complete
fmt.Println(r.TempPath, "!=", r.Filepath) log.Println(r.TempPath, "!=", r.FilePath)
if r.TempPath != r.Filepath { if r.TempPath != r.FilePath {
fmt.Println("renaming download to the completed dir") log.Println("renaming download to the completed dir")
os.Rename(r.TempPath, r.Filepath) os.Rename(r.TempPath, r.FilePath)
} }
d.history.queue = append(d.history.queue, r) d.History.Queue = append(d.History.Queue, r)
} else { } else {
r.Status = Error r.setError(r.Response.Err())
r.Error = r.Response.Err() log.Println("fucking error:", r.Err)
fmt.Println("fucking error:", r.Error)
} }
} }
func (d *Downloader) download() { func (d *Downloader) download() {
for { for {
select { select {
case TIME := <-time.After(10 * time.Second): case <-time.After(10 * time.Second):
fmt.Println(TIME)
for _, req := range d.downloads.queue {
fmt.Println(req.URL)
fmt.Println(req.Status)
fmt.Println(req.Response.ETA())
}
d.syncDownloads() d.syncDownloads()
case r := <-d.NewRequest: case r := <-d.NewRequest:
r.Progress = "" // not allowed when adding
d.addRequest(&r) d.addRequest(&r)
if d.OnAdd != nil {
d.OnAdd(d, r)
}
case r := <-d.requestDone: case r := <-d.requestDone:
fmt.Println("finishing request for", r.URL) log.Println("finishing request for", r.URL)
d.requestCompleted(r) d.requestCompleted(r)
if d.OnComplete != nil {
d.OnComplete(d, *r)
}
} }
} }
} }
func (r *Request) insertCookies() {
if r.Jar == nil {
r.Jar = newCookieJar()
}
u, _ := url.Parse(r.URL)
for i, v := range r.Cookies {
r.Jar.SetCookies(&url.URL{
Scheme: u.Scheme,
Path: v.Path,
Host: v.Domain,
}, []*http.Cookie{&r.Cookies[i]})
}
}
func (d *Downloader) initStatus(mux *http.ServeMux) {
commonTmpls := template.New("root").Funcs(map[string]interface{}{
"shortenSHA256": func(hash string) string {
if len(hash) > 10 {
return hash[:10]
}
return hash
},
})
commonTmpls = template.Must(commonTmpls.New("header").Parse(bundled.Asset("header.tmpl")))
commonTmpls = template.Must(commonTmpls.New("footer").Parse(bundled.Asset("footer.tmpl")))
queueTmpl := template.Must(template.Must(commonTmpls.Clone()).New("queueTmpl").Parse(bundled.Asset("queue.tmpl")))
historyTmpl := template.Must(template.Must(commonTmpls.Clone()).New("historyTmpl").Parse(bundled.Asset("history.tmpl")))
for _, fn := range []string{
"favicon.ico",
"bootstrap.min.js",
"bootstrap.min.css",
"bootstrap-table.min.js",
"bootstrap-table.min.css",
"popper.min.js",
"jquery-3.2.1.slim.min.js",
"downloads.js",
} {
mux.Handle("/"+fn, bundled.HTTPHandlerFunc(fn))
}
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
var buf bytes.Buffer
d.Downloads.updateStatus()
if err := queueTmpl.Execute(&buf, struct {
Queue []*Request
History []*Request
Hostname string
}{
Queue: d.Downloads.Queue,
History: d.History.Queue,
Hostname: "gloader",
}); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
}
io.Copy(w, &buf)
})
mux.HandleFunc("/history", func(w http.ResponseWriter, r *http.Request) {
var buf bytes.Buffer
d.Downloads.updateStatus()
if err := historyTmpl.Execute(&buf, struct {
Queue []*Request
History []*Request
Hostname string
}{
Queue: d.Downloads.Queue,
History: d.History.Queue,
Hostname: "gloader",
}); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
}
io.Copy(w, &buf)
})
}

2
go.mod
View File

@ -7,5 +7,7 @@ replace github.com/cavaliercoder/grab => ./grab
require ( require (
github.com/cavaliercoder/grab v2.0.0+incompatible github.com/cavaliercoder/grab v2.0.0+incompatible
github.com/lordwelch/pathvalidate v0.0.0-20201012043703-54efa7ea1308 github.com/lordwelch/pathvalidate v0.0.0-20201012043703-54efa7ea1308
github.com/u-root/u-root v7.0.0+incompatible
github.com/wI2L/jettison v0.7.1
golang.org/x/net v0.0.0-20201209123823-ac852fbbde11 golang.org/x/net v0.0.0-20201209123823-ac852fbbde11
) )

2
go.sum
View File

@ -1,5 +1,7 @@
github.com/lordwelch/pathvalidate v0.0.0-20201012043703-54efa7ea1308 h1:CkcsZK6QYg59rc92eqU2h+FRjWltCIiplmEwIB05jfM= github.com/lordwelch/pathvalidate v0.0.0-20201012043703-54efa7ea1308 h1:CkcsZK6QYg59rc92eqU2h+FRjWltCIiplmEwIB05jfM=
github.com/lordwelch/pathvalidate v0.0.0-20201012043703-54efa7ea1308/go.mod h1:4I4r5Y/LkH+34KACiudU+Q27ooz7xSDyVEuWAVKeJEQ= github.com/lordwelch/pathvalidate v0.0.0-20201012043703-54efa7ea1308/go.mod h1:4I4r5Y/LkH+34KACiudU+Q27ooz7xSDyVEuWAVKeJEQ=
github.com/u-root/u-root v7.0.0+incompatible h1:u+KSS04pSxJGI5E7WE4Bs9+Zd75QjFv+REkjy/aoAc8=
github.com/u-root/u-root v7.0.0+incompatible/go.mod h1:RYkpo8pTHrNjW08opNd/U6p/RJE7K0D8fXO0d47+3YY=
golang.org/x/net v0.0.0-20201209123823-ac852fbbde11 h1:lwlPPsmjDKK0J6eG6xDWd5XPehI0R024zxjDnw3esPA= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11 h1:lwlPPsmjDKK0J6eG6xDWd5XPehI0R024zxjDnw3esPA=
golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

187
goembed.go Normal file
View File

@ -0,0 +1,187 @@
// +build ignore
// goembed generates a Go source file from an input file.
package main
import (
"bufio"
"bytes"
"compress/gzip"
"flag"
"fmt"
"io"
"log"
"os"
"text/template"
"unicode/utf8"
)
var (
packageFlag = flag.String("package", "", "Go package name")
varFlag = flag.String("var", "", "Go var name")
gzipFlag = flag.Bool("gzip", false, "Whether to gzip contents")
)
func main() {
flag.Parse()
fmt.Printf("package %s\n\n", *packageFlag)
if *gzipFlag {
err := gzipPrologue.Execute(os.Stdout, map[string]interface{}{
"Args": flag.Args(),
"VarName": *varFlag,
})
if err != nil {
log.Fatal(err)
}
}
if flag.NArg() > 0 {
fmt.Println("// Table of contents")
fmt.Printf("var %v = map[string][]byte{\n", *varFlag)
for i, filename := range flag.Args() {
fmt.Printf("\t%q: %s_%d,\n", filename, *varFlag, i)
}
fmt.Println("}")
// Using a separate variable for each []byte, instead of
// combining them into a single map literal, enables a storage
// optimization: the compiler places the data directly in the
// program's noptrdata section instead of the heap.
for i, filename := range flag.Args() {
if err := oneVar(fmt.Sprintf("%s_%d", *varFlag, i), filename); err != nil {
log.Fatal(err)
}
}
} else {
if err := oneVarReader(*varFlag, os.Stdin); err != nil {
log.Fatal(err)
}
}
}
func oneVar(varName, filename string) error {
f, err := os.Open(filename)
if err != nil {
return err
}
defer f.Close()
return oneVarReader(varName, f)
}
func oneVarReader(varName string, r io.Reader) error {
// Generate []byte(<big string constant>) instead of []byte{<list of byte values>}.
// The latter causes a memory explosion in the compiler (60 MB of input chews over 9 GB RAM).
// Doing a string conversion avoids some of that, but incurs a slight startup cost.
if !*gzipFlag {
fmt.Printf(`var %s = []byte("`, varName)
} else {
var buf bytes.Buffer
gzw, _ := gzip.NewWriterLevel(&buf, gzip.BestCompression)
if _, err := io.Copy(gzw, r); err != nil {
return err
}
if err := gzw.Close(); err != nil {
return err
}
fmt.Printf("var %s []byte // set in init\n\n", varName)
fmt.Printf(`var %s_gzip = []byte("`, varName)
r = &buf
}
bufw := bufio.NewWriter(os.Stdout)
if _, err := io.Copy(&writer{w: bufw}, r); err != nil {
return err
}
if err := bufw.Flush(); err != nil {
return err
}
fmt.Println(`")`)
return nil
}
type writer struct {
w io.Writer
}
func (w *writer) Write(data []byte) (n int, err error) {
n = len(data)
for err == nil && len(data) > 0 {
// https://golang.org/ref/spec#String_literals: "Within the quotes, any
// character may appear except newline and unescaped double quote. The
// text between the quotes forms the value of the literal, with backslash
// escapes interpreted as they are in rune literals […]."
switch b := data[0]; b {
case '\\':
_, err = w.w.Write([]byte(`\\`))
case '"':
_, err = w.w.Write([]byte(`\"`))
case '\n':
_, err = w.w.Write([]byte(`\n`))
case '\x00':
// https://golang.org/ref/spec#Source_code_representation: "Implementation
// restriction: For compatibility with other tools, a compiler may
// disallow the NUL character (U+0000) in the source text."
_, err = w.w.Write([]byte(`\x00`))
default:
// https://golang.org/ref/spec#Source_code_representation: "Implementation
// restriction: […] A byte order mark may be disallowed anywhere else in
// the source."
const byteOrderMark = '\uFEFF'
if r, size := utf8.DecodeRune(data); r != utf8.RuneError && r != byteOrderMark {
_, err = w.w.Write(data[:size])
data = data[size:]
continue
}
_, err = fmt.Fprintf(w.w, `\x%02x`, b)
}
data = data[1:]
}
return n - len(data), err
}
var gzipPrologue = template.Must(template.New("").Parse(`
import (
"bytes"
"compress/gzip"
"io/ioutil"
)
func init() {
var (
r *gzip.Reader
err error
)
{{ if gt (len .Args) 0 }}
{{ range $idx, $var := .Args }}
{{ $n := printf "%s_%d" $.VarName $idx }}
r, err = gzip.NewReader(bytes.NewReader({{ $n }}_gzip))
if err != nil {
panic(err)
}
{{ $n }}, err = ioutil.ReadAll(r)
{{ $.VarName }}["{{ $var }}"] = {{ $n }}
r.Close()
if err != nil {
panic(err)
}
{{ end }}
{{ else }}
r, err = gzip.NewReader(bytes.NewReader({{ .VarName }}_gzip))
if err != nil {
panic(err)
}
{{ .VarName }}, err = ioutil.ReadAll(r)
r.Close()
if err != nil {
panic(err)
}
{{ end }}
}
`))

3
grab/client.go
View File

@ -267,6 +267,7 @@ func (c *Client) validateLocal(resp *Response) stateFunc {
if expectedSize >= 0 && expectedSize < resp.fi.Size() { if expectedSize >= 0 && expectedSize < resp.fi.Size() {
// remote size is known, is smaller than local size and we want to resume // remote size is known, is smaller than local size and we want to resume
fmt.Fprintln(os.Stderr, "validate\n")
resp.err = ErrBadLength resp.err = ErrBadLength
return c.closeResponse return c.closeResponse
} }
@ -395,6 +396,7 @@ func (c *Client) readResponse(resp *Response) stateFunc {
// remote size is known // remote size is known
resp.sizeUnsafe += resp.bytesResumed resp.sizeUnsafe += resp.bytesResumed
if resp.Request.Size > 0 && resp.Request.Size != resp.sizeUnsafe { if resp.Request.Size > 0 && resp.Request.Size != resp.sizeUnsafe {
fmt.Fprintln(os.Stderr, "response\n")
resp.err = ErrBadLength resp.err = ErrBadLength
return c.closeResponse return c.closeResponse
} }
@ -527,6 +529,7 @@ func (c *Client) copyFile(resp *Response) stateFunc {
discoveredSize := resp.bytesResumed + bytesCopied discoveredSize := resp.bytesResumed + bytesCopied
atomic.StoreInt64(&resp.sizeUnsafe, discoveredSize) atomic.StoreInt64(&resp.sizeUnsafe, discoveredSize)
if resp.Request.Size > 0 && resp.Request.Size != discoveredSize { if resp.Request.Size > 0 && resp.Request.Size != discoveredSize {
fmt.Fprintln(os.Stderr, "file\n")
resp.err = ErrBadLength resp.err = ErrBadLength
return c.closeResponse return c.closeResponse
} }

132
main.go
View File

@ -1,7 +1,8 @@
package main package main
import ( import (
"encoding/binary" "bufio"
"encoding/json"
"errors" "errors"
"flag" "flag"
"fmt" "fmt"
@ -10,8 +11,11 @@ import (
"log" "log"
"os" "os"
"path/filepath" "path/filepath"
"strconv"
"strings" "strings"
"syscall" "syscall"
"github.com/u-root/u-root/pkg/mount/gpt"
) )
var ( var (
@ -28,16 +32,80 @@ func main() {
log.Println(err) log.Println(err)
os.Exit(1) os.Exit(1)
} }
d := newDownloader() d := &Downloader{
d.DataDir = filepath.Join(gloaderHome, "data") OnAdd: save,
OnComplete: save,
DataDir: filepath.Join(gloaderHome, "data"),
}
loadQueue(d)
d.Start("tcp", ":8844") d.Start("tcp", ":8844")
} }
func save(d *Downloader, r Request) {
var (
content []byte
err error
)
content, err = json.Marshal(d.History.Queue)
if err != nil {
log.Println(err)
return
}
err = ioutil.WriteFile(filepath.Join(gloaderHome, "history.json"), content, 0o666)
if err != nil {
log.Println(err)
return
}
content, err = json.Marshal(d.Downloads.Queue)
if err != nil {
log.Println(err)
return
}
err = ioutil.WriteFile(filepath.Join(gloaderHome, "queue.json"), content, 0o666)
if err != nil {
log.Println(err)
return
}
}
func loadQueue(d *Downloader) {
var (
f io.ReadCloser
err error
decoder *json.Decoder
)
f, err = os.Open(filepath.Join(gloaderHome, "history.json"))
if err != nil {
log.Println(err)
return
}
decoder = json.NewDecoder(bufio.NewReader(f))
err = decoder.Decode(&d.History.Queue)
if err != nil {
log.Println(err)
return
}
f.Close()
f, err = os.Open(filepath.Join(gloaderHome, "queue.json"))
if err != nil {
log.Println(err)
return
}
decoder = json.NewDecoder(bufio.NewReader(f))
err = decoder.Decode(&d.Downloads.Queue)
if err != nil {
log.Println(err)
return
}
f.Close()
}
func mount() error { func mount() error {
var ( var (
partUUIDb []byte partUUIDb []byte
partUUID string partUUID string
dev string dev part
err error err error
dataStat os.FileInfo dataStat os.FileInfo
dataDir = filepath.Join(gloaderHome, "data") dataDir = filepath.Join(gloaderHome, "data")
@ -70,14 +138,15 @@ func mount() error {
} }
_, err = folder.Readdir(1) _, err = folder.Readdir(1)
if errors.Is(err, io.EOF) { if errors.Is(err, io.EOF) {
fmt.Printf("mount %s %s", partUUID, dataDir) log.Printf("mount %s %s\n", partUUID, dataDir)
dev, err = findPartUUID(partUUID) dev = findPartUUID(partUUID)
err = syscall.Mount(dev.Path, dataDir, "ext4", 0, "")
if err != nil { if err != nil {
return fmt.Errorf("error mounting datadir: %w", err) return fmt.Errorf("error mounting datadir: %w", err)
} }
err = syscall.Mount(dev, dataDir, "ext4", 0, "") err = syscall.Mount(dev.Path, dataDir, "ext4", syscall.MS_SHARED|syscall.MS_REMOUNT, "")
if err != nil { if err != nil {
return fmt.Errorf("error mounting datadir: %w", err) return fmt.Errorf("error remounting datadir: %w", err)
} }
return nil return nil
} }
@ -86,11 +155,16 @@ func mount() error {
return fmt.Errorf("error mounting datadir: data dir %s is not a directory", dataDir) return fmt.Errorf("error mounting datadir: data dir %s is not a directory", dataDir)
} }
func findPartUUID(uuid string) (string, error) { type part struct {
var dev string UUID string
Path string
}
func findPartUUID(UUID string) part {
var dev part
err := filepath.Walk("/sys/block", func(path string, info os.FileInfo, err error) error { err := filepath.Walk("/sys/block", func(path string, info os.FileInfo, err error) error {
if err != nil { if err != nil {
log.Printf("findPartUUID: %v", err) log.Printf("listPartUUID: %v", err)
return nil return nil
} }
if info.Mode()&os.ModeSymlink == 0 { if info.Mode()&os.ModeSymlink == 0 {
@ -99,39 +173,23 @@ func findPartUUID(uuid string) (string, error) {
devname := "/dev/" + filepath.Base(path) devname := "/dev/" + filepath.Base(path)
f, err := os.Open(devname) f, err := os.Open(devname)
if err != nil { if err != nil {
log.Printf("findPartUUID: %v", err) log.Printf("listPartUUID: %v", err)
return nil return nil
} }
defer f.Close() defer f.Close()
if _, err := f.Seek(440, io.SeekStart); err != nil { parttable, _ := gpt.New(f)
var se syscall.Errno if parttable.Primary != nil {
if errors.As(err, &se) && se == syscall.EINVAL { for i, partition := range parttable.Primary.Parts {
// Seek()ing empty loop devices results in EINVAL. if partition.UniqueGUID.String() == UUID {
return nil dev = part{UUID: partition.UniqueGUID.String(), Path: devname + strconv.Itoa(i+1)}
}
} }
log.Printf("findPartUUID: %v(%T)", err, err.(*os.PathError).Err)
return nil
}
var diskSig struct {
ID uint32
Trailer uint16
}
if err := binary.Read(f, binary.LittleEndian, &diskSig); err != nil {
log.Printf("findPartUUID: %v", err)
return nil
}
if fmt.Sprintf("%08x", diskSig.ID) == uuid && diskSig.Trailer == 0 {
dev = devname
// TODO: abort early with sentinel error code
return nil
} }
return nil return nil
}) })
if err != nil { if err != nil {
return "", err return part{}
} }
if dev == "" { return dev
return "", fmt.Errorf("PARTUUID=%s not found", uuid)
}
return dev, nil
} }