package grab import ( "bytes" "context" "crypto/md5" "crypto/sha1" "crypto/sha256" "crypto/sha512" "errors" "fmt" "hash" "io/ioutil" "math/rand" "net/http" "os" "path/filepath" "strings" "testing" "time" "github.com/cavaliercoder/grab/grabtest" ) // TestFilenameResolutions tests that the destination filename for Requests can // be determined correctly, using an explicitly requested path, // Content-Disposition headers or a URL path - with or without an existing // target directory. func TestFilenameResolution(t *testing.T) { tests := []struct { Name string Filename string URL string AttachmentFilename string Expect string }{ {"Using Request.Filename", ".testWithFilename", "/url-filename", "header-filename", ".testWithFilename"}, {"Using Content-Disposition Header", "", "/url-filename", ".testWithHeaderFilename", ".testWithHeaderFilename"}, {"Using Content-Disposition Header with target directory", ".test", "/url-filename", "header-filename", ".test/header-filename"}, {"Using URL Path", "", "/.testWithURLFilename?params-filename", "", ".testWithURLFilename"}, {"Using URL Path with target directory", ".test", "/url-filename?garbage", "", ".test/url-filename"}, {"Failure", "", "", "", ""}, } err := os.Mkdir(".test", 0777) if err != nil { panic(err) } defer os.RemoveAll(".test") for _, test := range tests { t.Run(test.Name, func(t *testing.T) { opts := []grabtest.HandlerOption{} if test.AttachmentFilename != "" { opts = append(opts, grabtest.AttachmentFilename(test.AttachmentFilename)) } grabtest.WithTestServer(t, func(url string) { req := mustNewRequest(test.Filename, url+test.URL) resp := DefaultClient.Do(req) defer os.Remove(resp.Filename) if err := resp.Err(); err != nil { if test.Expect != "" || err != ErrNoFilename { panic(err) } } else { if test.Expect == "" { t.Errorf("expected: %v, got: %v", ErrNoFilename, err) } } if resp.Filename != test.Expect { t.Errorf("Filename mismatch. Expected '%s', got '%s'.", test.Expect, resp.Filename) } testComplete(t, resp) }, opts...) }) } } // TestChecksums checks that checksum validation behaves as expected for valid // and corrupted downloads. func TestChecksums(t *testing.T) { tests := []struct { size int hash hash.Hash sum string match bool }{ {128, md5.New(), "37eff01866ba3f538421b30b7cbefcac", true}, {128, md5.New(), "37eff01866ba3f538421b30b7cbefcad", false}, {1024, md5.New(), "b2ea9f7fcea831a4a63b213f41a8855b", true}, {1024, md5.New(), "b2ea9f7fcea831a4a63b213f41a8855c", false}, {1048576, md5.New(), "c35cc7d8d91728a0cb052831bc4ef372", true}, {1048576, md5.New(), "c35cc7d8d91728a0cb052831bc4ef373", false}, {128, sha1.New(), "e6434bc401f98603d7eda504790c98c67385d535", true}, {128, sha1.New(), "e6434bc401f98603d7eda504790c98c67385d536", false}, {1024, sha1.New(), "5b00669c480d5cffbdfa8bdba99561160f2d1b77", true}, {1024, sha1.New(), "5b00669c480d5cffbdfa8bdba99561160f2d1b78", false}, {1048576, sha1.New(), "ecfc8e86fdd83811f9cc9bf500993b63069923be", true}, {1048576, sha1.New(), "ecfc8e86fdd83811f9cc9bf500993b63069923bf", false}, {128, sha256.New(), "471fb943aa23c511f6f72f8d1652d9c880cfa392ad80503120547703e56a2be5", true}, {128, sha256.New(), "471fb943aa23c511f6f72f8d1652d9c880cfa392ad80503120547703e56a2be4", false}, {1024, sha256.New(), "785b0751fc2c53dc14a4ce3d800e69ef9ce1009eb327ccf458afe09c242c26c9", true}, {1024, sha256.New(), "785b0751fc2c53dc14a4ce3d800e69ef9ce1009eb327ccf458afe09c242c26c8", false}, {1048576, sha256.New(), "fbbab289f7f94b25736c58be46a994c441fd02552cc6022352e3d86d2fab7c83", true}, {1048576, sha256.New(), "fbbab289f7f94b25736c58be46a994c441fd02552cc6022352e3d86d2fab7c82", false}, {128, sha512.New(), "1dffd5e3adb71d45d2245939665521ae001a317a03720a45732ba1900ca3b8351fc5c9b4ca513eba6f80bc7b1d1fdad4abd13491cb824d61b08d8c0e1561b3f7", true}, {128, sha512.New(), "1dffd5e3adb71d45d2245939665521ae001a317a03720a45732ba1900ca3b8351fc5c9b4ca513eba6f80bc7b1d1fdad4abd13491cb824d61b08d8c0e1561b3f8", false}, {1024, sha512.New(), "37f652be867f28ed033269cbba201af2112c2b3fd334a89fd2f757938ddee815787cc61d6e24a8a33340d0f7e86ffc058816b88530766ba6e231620a130b566c", true}, {1024, sha512.New(), "37f652bf867f28ed033269cbba201af2112c2b3fd334a89fd2f757938ddee815787cc61d6e24a8a33340d0f7e86ffc058816b88530766ba6e231620a130b566d", false}, {1048576, sha512.New(), "ac1d097b4ea6f6ad7ba640275b9ac290e4828cd760a0ebf76d555463a4f505f95df4f611629539a2dd1848e7c1304633baa1826462b3c87521c0c6e3469b67af", true}, {1048576, sha512.New(), "ac1d097c4ea6f6ad7ba640275b9ac290e4828cd760a0ebf76d555463a4f505f95df4f611629539a2dd1848e7c1304633baa1826462b3c87521c0c6e3469b67af", false}, } for _, test := range tests { var expect error comparison := "Match" if !test.match { comparison = "Mismatch" expect = ErrBadChecksum } t.Run(fmt.Sprintf("With%s%s", comparison, test.sum[:8]), func(t *testing.T) { filename := fmt.Sprintf(".testChecksum-%s-%s", comparison, test.sum[:8]) defer os.Remove(filename) grabtest.WithTestServer(t, func(url string) { req := mustNewRequest(filename, url) req.SetChecksum(test.hash, grabtest.MustHexDecodeString(test.sum), true) resp := DefaultClient.Do(req) err := resp.Err() if err != expect { t.Errorf("expected error: %v, got: %v", expect, err) } // ensure mismatch file was deleted if !test.match { if _, err := os.Stat(filename); err == nil { t.Errorf("checksum failure not cleaned up: %s", filename) } else if !os.IsNotExist(err) { panic(err) } } testComplete(t, resp) }, grabtest.ContentLength(test.size)) }) } } // TestContentLength ensures that ErrBadLength is returned if a server response // does not match the requested length. func TestContentLength(t *testing.T) { size := int64(32768) testCases := []struct { Name string NoHead bool Size int64 Expect int64 Match bool }{ {"Good size in HEAD request", false, size, size, true}, {"Good size in GET request", true, size, size, true}, {"Bad size in HEAD request", false, size - 1, size, false}, {"Bad size in GET request", true, size - 1, size, false}, } for _, test := range testCases { t.Run(test.Name, func(t *testing.T) { opts := []grabtest.HandlerOption{ grabtest.ContentLength(int(test.Size)), } if test.NoHead { opts = append(opts, grabtest.MethodWhitelist("GET")) } grabtest.WithTestServer(t, func(url string) { req := mustNewRequest(".testSize-mismatch-head", url) req.Size = size resp := DefaultClient.Do(req) defer os.Remove(resp.Filename) err := resp.Err() if test.Match { if err == ErrBadLength { t.Errorf("error: %v", err) } else if err != nil { panic(err) } else if resp.Size() != size { t.Errorf("expected %v bytes, got %v bytes", size, resp.Size()) } } else { if err == nil { t.Errorf("expected: %v, got %v", ErrBadLength, err) } else if err != ErrBadLength { panic(err) } } testComplete(t, resp) }, opts...) }) } } // TestAutoResume tests segmented downloading of a large file. func TestAutoResume(t *testing.T) { segs := 8 size := 1048576 sum := grabtest.DefaultHandlerSHA256ChecksumBytes //grabtest.MustHexDecodeString("fbbab289f7f94b25736c58be46a994c441fd02552cc6022352e3d86d2fab7c83") filename := ".testAutoResume" defer os.Remove(filename) for i := 0; i < segs; i++ { segsize := (i + 1) * (size / segs) t.Run(fmt.Sprintf("With%vBytes", segsize), func(t *testing.T) { grabtest.WithTestServer(t, func(url string) { req := mustNewRequest(filename, url) if i == segs-1 { req.SetChecksum(sha256.New(), sum, false) } resp := mustDo(req) if i > 0 && !resp.DidResume { t.Errorf("expected Response.DidResume to be true") } testComplete(t, resp) }, grabtest.ContentLength(segsize), ) }) } t.Run("WithFailure", func(t *testing.T) { grabtest.WithTestServer(t, func(url string) { // request smaller segment req := mustNewRequest(filename, url) resp := DefaultClient.Do(req) if err := resp.Err(); err != ErrBadLength { t.Errorf("expected ErrBadLength for smaller request, got: %v", err) } }, grabtest.ContentLength(size-128), ) }) t.Run("WithNoResume", func(t *testing.T) { grabtest.WithTestServer(t, func(url string) { req := mustNewRequest(filename, url) req.NoResume = true resp := mustDo(req) if resp.DidResume { t.Errorf("expected Response.DidResume to be false") } testComplete(t, resp) }, grabtest.ContentLength(size+128), ) }) t.Run("WithNoResumeAndTruncate", func(t *testing.T) { size := size - 128 grabtest.WithTestServer(t, func(url string) { req := mustNewRequest(filename, url) req.NoResume = true resp := mustDo(req) if resp.DidResume { t.Errorf("expected Response.DidResume to be false") } if v := resp.BytesComplete(); v != int64(size) { t.Errorf("expected Response.BytesComplete: %d, got: %d", size, v) } testComplete(t, resp) }, grabtest.ContentLength(size), ) }) t.Run("WithNoContentLengthHeader", func(t *testing.T) { grabtest.WithTestServer(t, func(url string) { req := mustNewRequest(filename, url) req.SetChecksum(sha256.New(), sum, false) resp := mustDo(req) if !resp.DidResume { t.Errorf("expected Response.DidResume to be true") } if actual := resp.Size(); actual != int64(size) { t.Errorf("expected Response.Size: %d, got: %d", size, actual) } testComplete(t, resp) }, grabtest.ContentLength(size), grabtest.HeaderBlacklist("Content-Length"), ) }) t.Run("WithNoContentLengthHeaderAndChecksumFailure", func(t *testing.T) { // ref: https://github.com/cavaliercoder/grab/pull/27 size := size * 2 grabtest.WithTestServer(t, func(url string) { req := mustNewRequest(filename, url) req.SetChecksum(sha256.New(), sum, false) resp := DefaultClient.Do(req) if err := resp.Err(); err != ErrBadChecksum { t.Errorf("expected error: %v, got: %v", ErrBadChecksum, err) } if !resp.DidResume { t.Errorf("expected Response.DidResume to be true") } if actual := resp.BytesComplete(); actual != int64(size) { t.Errorf("expected Response.BytesComplete: %d, got: %d", size, actual) } if actual := resp.Size(); actual != int64(size) { t.Errorf("expected Response.Size: %d, got: %d", size, actual) } testComplete(t, resp) }, grabtest.ContentLength(size), grabtest.HeaderBlacklist("Content-Length"), ) }) // TODO: test when existing file is corrupted } func TestSkipExisting(t *testing.T) { filename := ".testSkipExisting" defer os.Remove(filename) // download a file grabtest.WithTestServer(t, func(url string) { resp := mustDo(mustNewRequest(filename, url)) testComplete(t, resp) }) // redownload grabtest.WithTestServer(t, func(url string) { resp := mustDo(mustNewRequest(filename, url)) testComplete(t, resp) // ensure download was resumed if !resp.DidResume { t.Fatalf("Expected download to skip existing file, but it did not") } // ensure all bytes were resumed if resp.Size() == 0 || resp.Size() != resp.bytesResumed { t.Fatalf("Expected to skip %d bytes in redownload; got %d", resp.Size(), resp.bytesResumed) } }) // ensure checksum is performed on pre-existing file grabtest.WithTestServer(t, func(url string) { req := mustNewRequest(filename, url) req.SetChecksum(sha256.New(), []byte{0x01, 0x02, 0x03, 0x04}, true) resp := DefaultClient.Do(req) if err := resp.Err(); err != ErrBadChecksum { t.Fatalf("Expected checksum error, got: %v", err) } }) } // TestBatch executes multiple requests simultaneously and validates the // responses. func TestBatch(t *testing.T) { tests := 32 size := 32768 sum := grabtest.MustHexDecodeString("e11360251d1173650cdcd20f111d8f1ca2e412f572e8b36a4dc067121c1799b8") // test with 4 workers and with one per request grabtest.WithTestServer(t, func(url string) { for _, workerCount := range []int{4, 0} { // create requests reqs := make([]*Request, tests) for i := 0; i < len(reqs); i++ { filename := fmt.Sprintf(".testBatch.%d", i+1) reqs[i] = mustNewRequest(filename, url+fmt.Sprintf("/request_%d?", i+1)) reqs[i].Label = fmt.Sprintf("Test %d", i+1) reqs[i].SetChecksum(sha256.New(), sum, false) } // batch run responses := DefaultClient.DoBatch(workerCount, reqs...) // listen for responses Loop: for i := 0; i < len(reqs); { select { case resp := <-responses: if resp == nil { break Loop } testComplete(t, resp) if err := resp.Err(); err != nil { t.Errorf("%s: %v", resp.Filename, err) } // remove test file if resp.IsComplete() { os.Remove(resp.Filename) // ignore errors } i++ } } } }, grabtest.ContentLength(size), ) } // TestCancelContext tests that a batch of requests can be cancel using a // context.Context cancellation. Requests are cancelled in multiple states: // in-progress and unstarted. func TestCancelContext(t *testing.T) { fileSize := 134217728 tests := 256 client := NewClient() ctx, cancel := context.WithCancel(context.Background()) defer cancel() grabtest.WithTestServer(t, func(url string) { reqs := make([]*Request, tests) for i := 0; i < tests; i++ { req := mustNewRequest("", fmt.Sprintf("%s/.testCancelContext%d", url, i)) reqs[i] = req.WithContext(ctx) } respch := client.DoBatch(8, reqs...) time.Sleep(time.Millisecond * 500) cancel() for resp := range respch { defer os.Remove(resp.Filename) // err should be context.Canceled or http.errRequestCanceled if resp.Err() == nil || !strings.Contains(resp.Err().Error(), "canceled") { t.Errorf("expected '%v', got '%v'", context.Canceled, resp.Err()) } if resp.BytesComplete() >= int64(fileSize) { t.Errorf("expected Response.BytesComplete: < %d, got: %d", fileSize, resp.BytesComplete()) } } }, grabtest.ContentLength(fileSize), ) } // TestCancelHangingResponse tests that a never ending request is terminated // when the response is cancelled. func TestCancelHangingResponse(t *testing.T) { fileSize := 10 client := NewClient() grabtest.WithTestServer(t, func(url string) { req := mustNewRequest("", fmt.Sprintf("%s/.testCancelHangingResponse", url)) resp := client.Do(req) defer os.Remove(resp.Filename) // Wait for some bytes to be transferred for resp.BytesComplete() == 0 { time.Sleep(50 * time.Millisecond) } done := make(chan error) go func() { done <- resp.Cancel() }() select { case err := <-done: if err != context.Canceled { t.Errorf("Expected context.Canceled error, go: %v", err) } case <-time.After(time.Second): t.Fatal("response was not cancelled within 1s") } if resp.BytesComplete() == int64(fileSize) { t.Error("download was not supposed to be complete") } fmt.Println("bye") }, grabtest.RateLimiter(1), grabtest.ContentLength(fileSize), ) } // TestNestedDirectory tests that missing subdirectories are created. func TestNestedDirectory(t *testing.T) { dir := "./.testNested/one/two/three" filename := ".testNestedFile" expect := dir + "/" + filename t.Run("Create", func(t *testing.T) { grabtest.WithTestServer(t, func(url string) { resp := mustDo(mustNewRequest(expect, url+"/"+filename)) defer os.RemoveAll("./.testNested/") if resp.Filename != expect { t.Errorf("expected nested Request.Filename to be %v, got %v", expect, resp.Filename) } }) }) t.Run("No create", func(t *testing.T) { grabtest.WithTestServer(t, func(url string) { req := mustNewRequest(expect, url+"/"+filename) req.NoCreateDirectories = true resp := DefaultClient.Do(req) err := resp.Err() if !os.IsNotExist(err) { t.Errorf("expected: %v, got: %v", os.ErrNotExist, err) } }) }) } // TestRemoteTime tests that the timestamp of the downloaded file can be set // according to the timestamp of the remote file. func TestRemoteTime(t *testing.T) { filename := "./.testRemoteTime" defer os.Remove(filename) // random time between epoch and now expect := time.Unix(rand.Int63n(time.Now().Unix()), 0) grabtest.WithTestServer(t, func(url string) { resp := mustDo(mustNewRequest(filename, url)) fi, err := os.Stat(resp.Filename) if err != nil { panic(err) } actual := fi.ModTime() if !actual.Equal(expect) { t.Errorf("expected %v, got %v", expect, actual) } }, grabtest.LastModified(expect), ) } func TestResponseCode(t *testing.T) { filename := "./.testResponseCode" t.Run("With404", func(t *testing.T) { defer os.Remove(filename) grabtest.WithTestServer(t, func(url string) { req := mustNewRequest(filename, url) resp := DefaultClient.Do(req) expect := StatusCodeError(http.StatusNotFound) err := resp.Err() if err != expect { t.Errorf("expected %v, got '%v'", expect, err) } if !IsStatusCodeError(err) { t.Errorf("expected IsStatusCodeError to return true for %T: %v", err, err) } }, grabtest.StatusCodeStatic(http.StatusNotFound), ) }) t.Run("WithIgnoreNon2XX", func(t *testing.T) { defer os.Remove(filename) grabtest.WithTestServer(t, func(url string) { req := mustNewRequest(filename, url) req.IgnoreBadStatusCodes = true resp := DefaultClient.Do(req) if err := resp.Err(); err != nil { t.Errorf("expected nil, got '%v'", err) } }, grabtest.StatusCodeStatic(http.StatusNotFound), ) }) } func TestBeforeCopyHook(t *testing.T) { filename := "./.testBeforeCopy" t.Run("Noop", func(t *testing.T) { defer os.RemoveAll(filename) grabtest.WithTestServer(t, func(url string) { called := false req := mustNewRequest(filename, url) req.BeforeCopy = func(resp *Response) error { called = true if resp.IsComplete() { t.Error("Response object passed to BeforeCopy hook has already been closed") } if resp.Progress() != 0 { t.Error("Download progress already > 0 when BeforeCopy hook was called") } if resp.Duration() == 0 { t.Error("Duration was zero when BeforeCopy was called") } if resp.BytesComplete() != 0 { t.Error("BytesComplete already > 0 when BeforeCopy hook was called") } return nil } resp := DefaultClient.Do(req) if err := resp.Err(); err != nil { t.Errorf("unexpected error using BeforeCopy hook: %v", err) } testComplete(t, resp) if !called { t.Error("BeforeCopy hook was never called") } }) }) t.Run("WithError", func(t *testing.T) { defer os.RemoveAll(filename) grabtest.WithTestServer(t, func(url string) { testError := errors.New("test") req := mustNewRequest(filename, url) req.BeforeCopy = func(resp *Response) error { return testError } resp := DefaultClient.Do(req) if err := resp.Err(); err != testError { t.Errorf("expected error '%v', got '%v'", testError, err) } if resp.BytesComplete() != 0 { t.Errorf("expected 0 bytes completed for canceled BeforeCopy hook, got %d", resp.BytesComplete()) } testComplete(t, resp) }) }) // Assert that an existing local file will not be truncated prior to the // BeforeCopy hook has a chance to cancel the request t.Run("NoTruncate", func(t *testing.T) { tfile, err := ioutil.TempFile("", "grab_client_test.*.file") if err != nil { t.Fatal(err) } defer os.Remove(tfile.Name()) const size = 128 _, err = tfile.Write(bytes.Repeat([]byte("x"), size)) if err != nil { t.Fatal(err) } grabtest.WithTestServer(t, func(url string) { called := false req := mustNewRequest(tfile.Name(), url) req.NoResume = true req.BeforeCopy = func(resp *Response) error { called = true fi, err := tfile.Stat() if err != nil { t.Errorf("failed to stat temp file: %v", err) return nil } if fi.Size() != size { t.Errorf("expected existing file size of %d bytes "+ "prior to BeforeCopy hook, got %d", size, fi.Size()) } return nil } resp := DefaultClient.Do(req) if err := resp.Err(); err != nil { t.Errorf("unexpected error using BeforeCopy hook: %v", err) } testComplete(t, resp) if !called { t.Error("BeforeCopy hook was never called") } }) }) } func TestAfterCopyHook(t *testing.T) { filename := "./.testAfterCopy" t.Run("Noop", func(t *testing.T) { defer os.RemoveAll(filename) grabtest.WithTestServer(t, func(url string) { called := false req := mustNewRequest(filename, url) req.AfterCopy = func(resp *Response) error { called = true if resp.IsComplete() { t.Error("Response object passed to AfterCopy hook has already been closed") } if resp.Progress() <= 0 { t.Error("Download progress was 0 when AfterCopy hook was called") } if resp.Duration() == 0 { t.Error("Duration was zero when AfterCopy was called") } if resp.BytesComplete() <= 0 { t.Error("BytesComplete was 0 when AfterCopy hook was called") } return nil } resp := DefaultClient.Do(req) if err := resp.Err(); err != nil { t.Errorf("unexpected error using AfterCopy hook: %v", err) } testComplete(t, resp) if !called { t.Error("AfterCopy hook was never called") } }) }) t.Run("WithError", func(t *testing.T) { defer os.RemoveAll(filename) grabtest.WithTestServer(t, func(url string) { testError := errors.New("test") req := mustNewRequest(filename, url) req.AfterCopy = func(resp *Response) error { return testError } resp := DefaultClient.Do(req) if err := resp.Err(); err != testError { t.Errorf("expected error '%v', got '%v'", testError, err) } if resp.BytesComplete() <= 0 { t.Errorf("ByteCompleted was %d after AfterCopy hook was called", resp.BytesComplete()) } testComplete(t, resp) }) }) } func TestIssue37(t *testing.T) { // ref: https://github.com/cavaliercoder/grab/issues/37 filename := "./.testIssue37" largeSize := int64(2097152) smallSize := int64(1048576) defer os.RemoveAll(filename) // download large file grabtest.WithTestServer(t, func(url string) { resp := mustDo(mustNewRequest(filename, url)) if resp.Size() != largeSize { t.Errorf("expected response size: %d, got: %d", largeSize, resp.Size()) } }, grabtest.ContentLength(int(largeSize))) // download new, smaller version of same file grabtest.WithTestServer(t, func(url string) { req := mustNewRequest(filename, url) req.NoResume = true resp := mustDo(req) if resp.Size() != smallSize { t.Errorf("expected response size: %d, got: %d", smallSize, resp.Size()) } // local file should have truncated and not resumed if resp.DidResume { t.Errorf("expected download to truncate, resumed instead") } }, grabtest.ContentLength(int(smallSize))) fi, err := os.Stat(filename) if err != nil { t.Fatal(err) } if fi.Size() != int64(smallSize) { t.Errorf("expected file size %d, got %d", smallSize, fi.Size()) } } // TestHeadBadStatus validates that HEAD requests that return non-200 can be // ignored and succeed if the GET requests succeeeds. // // Fixes: https://github.com/cavaliercoder/grab/issues/43 func TestHeadBadStatus(t *testing.T) { expect := http.StatusOK filename := ".testIssue43" statusFunc := func(r *http.Request) int { if r.Method == "HEAD" { return http.StatusForbidden } return http.StatusOK } grabtest.WithTestServer(t, func(url string) { testURL := fmt.Sprintf("%s/%s", url, filename) resp := mustDo(mustNewRequest("", testURL)) if resp.HTTPResponse.StatusCode != expect { t.Errorf( "expected status code: %d, got:% d", expect, resp.HTTPResponse.StatusCode) } }, grabtest.StatusCode(statusFunc), ) } // TestMissingContentLength ensures that the Response.Size is correct for // transfers where the remote server does not send a Content-Length header. // // TestAutoResume also covers cases with checksum validation. // // Kudos to Setnička Jiří for identifying and raising // a solution to this issue. Ref: https://github.com/cavaliercoder/grab/pull/27 func TestMissingContentLength(t *testing.T) { // expectSize must be sufficiently large that DefaultClient.Do won't prefetch // the entire body and compute ContentLength before returning a Response. expectSize := 1048576 opts := []grabtest.HandlerOption{ grabtest.ContentLength(expectSize), grabtest.HeaderBlacklist("Content-Length"), grabtest.TimeToFirstByte(time.Millisecond * 100), // delay for initial read } grabtest.WithTestServer(t, func(url string) { req := mustNewRequest(".testMissingContentLength", url) req.SetChecksum( md5.New(), grabtest.DefaultHandlerMD5ChecksumBytes, false) resp := DefaultClient.Do(req) // ensure remote server is not sending content-length header if v := resp.HTTPResponse.Header.Get("Content-Length"); v != "" { panic(fmt.Sprintf("http header content length must be empty, got: %s", v)) } if v := resp.HTTPResponse.ContentLength; v != -1 { panic(fmt.Sprintf("http response content length must be -1, got: %d", v)) } // before completion, response size should be -1 if resp.Size() != -1 { t.Errorf("expected response size: -1, got: %d", resp.Size()) } // block for completion if err := resp.Err(); err != nil { panic(err) } // on completion, response size should be actual transfer size if resp.Size() != int64(expectSize) { t.Errorf("expected response size: %d, got: %d", expectSize, resp.Size()) } }, opts...) } func TestNoStore(t *testing.T) { filename := ".testSubdir/testNoStore" t.Run("DefaultCase", func(t *testing.T) { grabtest.WithTestServer(t, func(url string) { req := mustNewRequest(filename, url) req.NoStore = true req.SetChecksum(md5.New(), grabtest.DefaultHandlerMD5ChecksumBytes, true) resp := mustDo(req) // ensure Response.Bytes is correct and can be reread b, err := resp.Bytes() if err != nil { panic(err) } grabtest.AssertSHA256Sum( t, grabtest.DefaultHandlerSHA256ChecksumBytes, bytes.NewReader(b), ) // ensure Response.Open stream is correct and can be reread r, err := resp.Open() if err != nil { panic(err) } defer r.Close() grabtest.AssertSHA256Sum( t, grabtest.DefaultHandlerSHA256ChecksumBytes, r, ) // Response.Filename should still be set if resp.Filename != filename { t.Errorf("expected Response.Filename: %s, got: %s", filename, resp.Filename) } // ensure no files were written paths := []string{ filename, filepath.Base(filename), filepath.Dir(filename), resp.Filename, filepath.Base(resp.Filename), filepath.Dir(resp.Filename), } for _, path := range paths { _, err := os.Stat(path) if !os.IsNotExist(err) { t.Errorf( "expect error: %v, got: %v, for path: %s", os.ErrNotExist, err, path) } } }) }) t.Run("ChecksumValidation", func(t *testing.T) { grabtest.WithTestServer(t, func(url string) { req := mustNewRequest("", url) req.NoStore = true req.SetChecksum( md5.New(), grabtest.MustHexDecodeString("deadbeefcafebabe"), true) resp := DefaultClient.Do(req) if err := resp.Err(); err != ErrBadChecksum { t.Errorf("expected error: %v, got: %v", ErrBadChecksum, err) } }) }) }