"./operations.test -test.v -test.timeout 1h0m0s -remote TestIA:rclone-integration-test -verbose -fast-list" - Starting (try 5/5) 2022/05/21 09:27:46 DEBUG : Creating backend with remote "TestIA:rclone-integration-test/rclone-test-bonoyum0sakolof8zuvikug7" 2022/05/21 09:27:46 DEBUG : Using config file from "/home/rclone/.rclone.conf" 2022/05/21 09:27:47 DEBUG : Creating backend with remote "/tmp/rclone3573643079" === RUN TestDoMultiThreadCopy --- PASS: TestDoMultiThreadCopy (0.00s) === RUN TestMultithreadCalculateChunks === RUN TestMultithreadCalculateChunks/{size:1_streams:10_wantPartSize:65536_wantStreams:1} === RUN TestMultithreadCalculateChunks/{size:1048576_streams:1_wantPartSize:1048576_wantStreams:1} === RUN TestMultithreadCalculateChunks/{size:1048576_streams:2_wantPartSize:524288_wantStreams:2} === RUN TestMultithreadCalculateChunks/{size:1048577_streams:2_wantPartSize:589824_wantStreams:2} === RUN TestMultithreadCalculateChunks/{size:1048575_streams:2_wantPartSize:524288_wantStreams:2} --- PASS: TestMultithreadCalculateChunks (0.00s) --- PASS: TestMultithreadCalculateChunks/{size:1_streams:10_wantPartSize:65536_wantStreams:1} (0.00s) --- PASS: TestMultithreadCalculateChunks/{size:1048576_streams:1_wantPartSize:1048576_wantStreams:1} (0.00s) --- PASS: TestMultithreadCalculateChunks/{size:1048576_streams:2_wantPartSize:524288_wantStreams:2} (0.00s) --- PASS: TestMultithreadCalculateChunks/{size:1048577_streams:2_wantPartSize:589824_wantStreams:2} (0.00s) --- PASS: TestMultithreadCalculateChunks/{size:1048575_streams:2_wantPartSize:524288_wantStreams:2} (0.00s) === RUN TestMultithreadCopy run.go:181: Remote "Internet Archive item rclone-integration-test path rclone-test-bonoyum0sakolof8zuvikug7", Local "Local file system at /tmp/rclone3573643079", Modify Window "1ns" === RUN TestMultithreadCopy/{size:131071_streams:2} 2022/05/21 09:45:22 DEBUG : file1: Starting multi-thread copy with 2 parts of size 64Ki 2022/05/21 09:45:22 DEBUG : file1: multi-thread copy: stream 2/2 (65536-131071) size 63.999Ki starting 2022/05/21 09:45:22 DEBUG : file1: multi-thread copy: stream 1/2 (0-65536) size 64Ki starting 2022/05/21 09:45:23 DEBUG : file1: multi-thread copy: stream 2/2 (65536-131071) size 63.999Ki finished 2022/05/21 09:45:24 DEBUG : file1: multi-thread copy: stream 1/2 (0-65536) size 64Ki finished 2022/05/21 09:45:24 DEBUG : file1: Finished multi-thread copy with 2 parts of size 64Ki === RUN TestMultithreadCopy/{size:131072_streams:2} panic: test timed out after 1h0m0s goroutine 783 [running]: testing.(*M).startAlarm.func1() /usr/local/go/src/testing/testing.go:1788 +0x8e created by time.goFunc /usr/local/go/src/time/sleep.go:180 +0x31 goroutine 1 [chan receive, 60 minutes]: testing.(*T).Run(0xc000102b60, {0x198be8f, 0x46f693}, 0x1a233d0) /usr/local/go/src/testing/testing.go:1307 +0x375 testing.runTests.func1(0xc0006ea780) /usr/local/go/src/testing/testing.go:1598 +0x6e testing.tRunner(0xc000102b60, 0xc000951ca0) /usr/local/go/src/testing/testing.go:1259 +0x102 testing.runTests(0xc000518400, {0x2a57300, 0x52, 0x52}, {0xf0efe0, 0xc0009a1398, 0x2a65da0}) /usr/local/go/src/testing/testing.go:1596 +0x43f testing.(*M).Run(0xc000518400) /usr/local/go/src/testing/testing.go:1504 +0x51d github.com/rclone/rclone/fstest.TestMain(0xc0000001a0) /home/rclone/go/src/github.com/rclone/rclone/fstest/run.go:70 +0x9e github.com/rclone/rclone/fs/operations_test.TestMain(...) /home/rclone/go/src/github.com/rclone/rclone/fs/operations/operations_test.go:61 main.main() _testmain.go:209 +0x166 goroutine 7 [select]: go.opencensus.io/stats/view.(*worker).start(0xc00003da80) /home/rclone/go/pkg/mod/go.opencensus.io@v0.23.0/stats/view/worker.go:276 +0xb9 created by go.opencensus.io/stats/view.init.0 /home/rclone/go/pkg/mod/go.opencensus.io@v0.23.0/stats/view/worker.go:34 +0x92 goroutine 32 [chan receive, 42 minutes]: testing.(*T).Run(0xc000103a00, {0xc0009cc138, 0xc000964f48}, 0xc0009d8bd0) /usr/local/go/src/testing/testing.go:1307 +0x375 github.com/rclone/rclone/fs/operations.TestMultithreadCopy(0xc000102d00) /home/rclone/go/src/github.com/rclone/rclone/fs/operations/multithread_test.go:120 +0x246 testing.tRunner(0xc000103a00, 0x1a233d0) /usr/local/go/src/testing/testing.go:1259 +0x102 created by testing.(*T).Run /usr/local/go/src/testing/testing.go:1306 +0x35a goroutine 11 [IO wait]: internal/poll.runtime_pollWait(0x7f138c7fff30, 0x72) /usr/local/go/src/runtime/netpoll.go:234 +0x89 internal/poll.(*pollDesc).wait(0xc00014e900, 0xc000968000, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc00014e900, {0xc000968000, 0x5a74, 0x5a74}) /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc00014e900, {0xc000968000, 0xc000512c80, 0xc00096c95a}) /usr/local/go/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc000209208, {0xc000968000, 0xc00096c95a, 0x19}) /usr/local/go/src/net/net.go:183 +0x45 github.com/rclone/rclone/fs/fshttp.(*timeoutConn).Read(0xc000110000, {0xc000968000, 0xd, 0x10000c000988c40}) /home/rclone/go/src/github.com/rclone/rclone/fs/fshttp/dialer.go:104 +0x32 crypto/tls.(*atLeastReader).Read(0xc0009ee0a8, {0xc000968000, 0x0, 0x40bb8d}) /usr/local/go/src/crypto/tls/conn.go:777 +0x3d bytes.(*Buffer).ReadFrom(0xc0001693f8, {0x1d23660, 0xc0009ee0a8}) /usr/local/go/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc000169180, {0x7f138c674918, 0xc000110000}, 0x1127) /usr/local/go/src/crypto/tls/conn.go:799 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc000169180, 0x0) /usr/local/go/src/crypto/tls/conn.go:606 +0x112 crypto/tls.(*Conn).readRecord(...) /usr/local/go/src/crypto/tls/conn.go:574 crypto/tls.(*Conn).Read(0xc000169180, {0xc0008da000, 0x1000, 0x6e32d7}) /usr/local/go/src/crypto/tls/conn.go:1277 +0x16f bufio.(*Reader).Read(0xc000883800, {0xc00036d1b8, 0x9, 0xc0005321b0}) /usr/local/go/src/bufio/bufio.go:227 +0x1b4 io.ReadAtLeast({0x1d23460, 0xc000883800}, {0xc00036d1b8, 0x9, 0x9}, 0x9) /usr/local/go/src/io/io.go:328 +0x9a io.ReadFull(...) /usr/local/go/src/io/io.go:347 net/http.http2readFrameHeader({0xc00036d1b8, 0x9, 0xc0009c62a0}, {0x1d23460, 0xc000883800}) /usr/local/go/src/net/http/h2_bundle.go:1555 +0x6e net/http.(*http2Framer).ReadFrame(0xc00036d180) /usr/local/go/src/net/http/h2_bundle.go:1813 +0x95 net/http.(*http2clientConnReadLoop).run(0xc0004a1f98) /usr/local/go/src/net/http/h2_bundle.go:8583 +0x130 net/http.(*http2ClientConn).readLoop(0xc000532600) /usr/local/go/src/net/http/h2_bundle.go:8506 +0x6f created by net/http.(*http2Transport).newClientConn /usr/local/go/src/net/http/h2_bundle.go:7317 +0xb85 goroutine 308 [select, 42 minutes]: github.com/rclone/rclone/backend/internetarchive.(*Fs).waitFileUpload(0xc0000b94a0, {0x1d5bf70, 0xc000040058}, {0x1972f75, 0x5}, {0xc0008bc520, 0x20}, 0x20000) /home/rclone/go/src/github.com/rclone/rclone/backend/internetarchive/internetarchive.go:943 +0x506 github.com/rclone/rclone/backend/internetarchive.(*Object).Update(0xc0002440e0, {0x1d5bf70, 0xc000040058}, {0x1d234a0, 0xc0009d8d20}, {0x1d743b0, 0xc0008ce0f0}, {0x0, 0x0, 0x0}) /home/rclone/go/src/github.com/rclone/rclone/backend/internetarchive/internetarchive.go:721 +0x9c5 github.com/rclone/rclone/backend/internetarchive.(*Fs).Put(0xc0000b94a0, {0x1d5bf70, 0xc000040058}, {0x1d234a0, 0xc0009d8d20}, {0x1d743b0, 0xc0008ce0f0}, {0x0, 0x0, 0x0}) /home/rclone/go/src/github.com/rclone/rclone/backend/internetarchive/internetarchive.go:444 +0x1d6 github.com/rclone/rclone/fstest.(*Run).WriteObjectTo(0xc00016caf0, {0x1d5bf70, 0xc000040058}, {0x1d8c868, 0xc0000b94a0}, {0x1972f75, 0x5}, {0xc000772000, 0x20000}, {0x1dcd64ff, ...}, ...) /home/rclone/go/src/github.com/rclone/rclone/fstest/run.go:277 +0x432 github.com/rclone/rclone/fstest.(*Run).WriteObject(...) /home/rclone/go/src/github.com/rclone/rclone/fstest/run.go:294 github.com/rclone/rclone/fs/operations.TestMultithreadCopy.func1(0xc000102ea0) /home/rclone/go/src/github.com/rclone/rclone/fs/operations/multithread_test.go:127 +0x1ba testing.tRunner(0xc000102ea0, 0xc0009d8bd0) /usr/local/go/src/testing/testing.go:1259 +0x102 created by testing.(*T).Run /usr/local/go/src/testing/testing.go:1306 +0x35a goroutine 314 [select]: net/http.(*http2ClientConn).RoundTrip(0xc000532600, 0xc00017e800) /usr/local/go/src/net/http/h2_bundle.go:7662 +0x41d net/http.(*http2Transport).RoundTripOpt(0xc0002d55e0, 0xc00017e800, {0x20}) /usr/local/go/src/net/http/h2_bundle.go:7098 +0x1b7 net/http.(*http2Transport).RoundTrip(...) /usr/local/go/src/net/http/h2_bundle.go:7059 net/http.http2noDialH2RoundTripper.RoundTrip({0xc0000d6a00}, 0xc00017e800) /usr/local/go/src/net/http/h2_bundle.go:9428 +0x1b net/http.(*Transport).roundTrip(0xc0000d6a00, 0xc00017e800) /usr/local/go/src/net/http/transport.go:539 +0x390 net/http.(*Transport).RoundTrip(...) /usr/local/go/src/net/http/roundtrip.go:18 github.com/rclone/rclone/fs/fshttp.(*Transport).RoundTrip(0xc0001161e0, 0xc00017e800) /home/rclone/go/src/github.com/rclone/rclone/fs/fshttp/http.go:274 +0x4c8 net/http.send(0xc00017e800, {0x1d2a100, 0xc0001161e0}, {0x1934460, 0x1, 0x0}) /usr/local/go/src/net/http/client.go:252 +0x5d8 net/http.(*Client).send(0xc0002926c0, 0xc00017e800, {0x0, 0x8c6b2b24, 0x0}) /usr/local/go/src/net/http/client.go:176 +0x9b net/http.(*Client).do(0xc0002926c0, 0xc00017e800) /usr/local/go/src/net/http/client.go:725 +0x908 net/http.(*Client).Do(...) /usr/local/go/src/net/http/client.go:593 github.com/rclone/rclone/lib/rest.(*Client).Call(0xc000116280, {0x1d5bf70, 0xc000040058}, 0xc0004986c0) /home/rclone/go/src/github.com/rclone/rclone/lib/rest/rest.go:288 +0xc45 github.com/rclone/rclone/lib/rest.(*Client).callCodec(0x415906, {0x1d5bf70, 0xc000040058}, 0x0, {0x0, 0x0}, {0x15a47c0, 0xc0007f2000}, 0x7f13b3de6a68, 0x1a237c8, ...) /home/rclone/go/src/github.com/rclone/rclone/lib/rest/rest.go:485 +0x465 github.com/rclone/rclone/lib/rest.(*Client).CallJSON(...) /home/rclone/go/src/github.com/rclone/rclone/lib/rest/rest.go:428 github.com/rclone/rclone/backend/internetarchive.(*Fs).requestMetadata.func1() /home/rclone/go/src/github.com/rclone/rclone/backend/internetarchive/internetarchive.go:800 +0x85 github.com/rclone/rclone/fs.pacerInvoker(0xc0004e6480, 0x22, 0x100010000000022) /home/rclone/go/src/github.com/rclone/rclone/fs/pacer.go:88 +0x38 github.com/rclone/rclone/lib/pacer.(*Pacer).call(0xc0004e6480, 0xc00049dc38, 0xa) /home/rclone/go/src/github.com/rclone/rclone/lib/pacer/pacer.go:197 +0x85 github.com/rclone/rclone/lib/pacer.(*Pacer).Call(0xc0004e6480, 0x2) /home/rclone/go/src/github.com/rclone/rclone/lib/pacer/pacer.go:216 +0x90 github.com/rclone/rclone/backend/internetarchive.(*Fs).requestMetadata(0xc0000b94a0, {0x1d5bf70, 0xc000040058}, {0xc0009c8460, 0x17}) /home/rclone/go/src/github.com/rclone/rclone/backend/internetarchive/internetarchive.go:799 +0x268 github.com/rclone/rclone/backend/internetarchive.(*Fs).waitFileUpload.func1() /home/rclone/go/src/github.com/rclone/rclone/backend/internetarchive/internetarchive.go:887 +0x185 created by github.com/rclone/rclone/backend/internetarchive.(*Fs).waitFileUpload /home/rclone/go/src/github.com/rclone/rclone/backend/internetarchive/internetarchive.go:879 +0x47b goroutine 782 [select]: net/http.(*http2clientStream).writeRequest(0xc00016e180, 0xc00017e800) /usr/local/go/src/net/http/h2_bundle.go:7841 +0xa0b net/http.(*http2clientStream).doRequest(0x742e8d, 0x66657a75626f7639) /usr/local/go/src/net/http/h2_bundle.go:7706 +0x1e created by net/http.(*http2ClientConn).RoundTrip /usr/local/go/src/net/http/h2_bundle.go:7648 +0x326 "./operations.test -test.v -test.timeout 1h0m0s -remote TestIA:rclone-integration-test -verbose -fast-list" - Finished ERROR in 1h0m0.931781714s (try 5/5): exit status 2: Failed []