From c8c40cff21513ee494125c690be5da8f0d67a737 Mon Sep 17 00:00:00 2001 From: "Yamashita, Yuu" Date: Wed, 1 Jun 2016 06:55:59 +0000 Subject: [PATCH 1/3] Write remote content on stdout with `aria2c` (fixes #619, #534) `aria2c` doesn't support writing content to stdout. As a workaround, this patch will use temporary file then write content on stdout once finished downloading. --- plugins/python-build/bin/python-build | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/plugins/python-build/bin/python-build b/plugins/python-build/bin/python-build index 962ba401..4b0deb59 100755 --- a/plugins/python-build/bin/python-build +++ b/plugins/python-build/bin/python-build @@ -340,7 +340,9 @@ http_head_aria2c() { } http_get_aria2c() { - aria2c -o "${2:--}" ${ARIA2_OPTS} "$1" + local out="${2:-$(mktemp "out.XXXXXX")}" + aria2c --allow-overwrite=true -o "${out}" ${ARIA2_OPTS} "$1" >&4 + [ -n "$2" ] || cat "${out}" } http_head_curl() { From 42cffbc82d568739555067e7c6dd1f8e928c2806 Mon Sep 17 00:00:00 2001 From: "Yamashita, Yuu" Date: Thu, 2 Jun 2016 01:38:23 +0000 Subject: [PATCH 2/3] Fix broken tests related to changes in `aria2c` arguments --- plugins/python-build/test/cache.bats | 6 +++--- plugins/python-build/test/checksum.bats | 18 +++++++++--------- plugins/python-build/test/mirror.bats | 16 ++++++++-------- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/plugins/python-build/test/cache.bats b/plugins/python-build/test/cache.bats index 1dca44b0..00445ee5 100644 --- a/plugins/python-build/test/cache.bats +++ b/plugins/python-build/test/cache.bats @@ -11,7 +11,7 @@ setup() { @test "packages are saved to download cache" { - stub aria2c "-o * http://example.com/* : cp $FIXTURE_ROOT/\${3##*/} \$2" + stub aria2c "--allow-overwrite=true -o * http://example.com/* : cp $FIXTURE_ROOT/\${4##*/} \$3" install_fixture definitions/without-checksum @@ -59,7 +59,7 @@ setup() { stub shasum true "echo invalid" "echo $checksum" stub aria2c "--dry-run * : true" \ - "-o * https://?*/$checksum : cp $FIXTURE_ROOT/package-1.0.0.tar.gz \$2" + "--allow-overwrite=true -o * https://?*/$checksum : cp $FIXTURE_ROOT/package-1.0.0.tar.gz \$3" touch "${PYTHON_BUILD_CACHE_PATH}/package-1.0.0.tar.gz" @@ -76,7 +76,7 @@ setup() { @test "nonexistent cache directory is ignored" { - stub aria2c "-o * http://example.com/* : cp $FIXTURE_ROOT/\${3##*/} \$2" + stub aria2c "--allow-overwrite=true -o * http://example.com/* : cp $FIXTURE_ROOT/\${4##*/} \$3" export PYTHON_BUILD_CACHE_PATH="${TMP}/nonexistent" diff --git a/plugins/python-build/test/checksum.bats b/plugins/python-build/test/checksum.bats index ffb7d32b..9efd8dcc 100644 --- a/plugins/python-build/test/checksum.bats +++ b/plugins/python-build/test/checksum.bats @@ -7,7 +7,7 @@ export PYTHON_BUILD_ARIA2_OPTS= @test "package URL without checksum" { - stub aria2c "-o * http://example.com/* : cp $FIXTURE_ROOT/\${3##*/} \$2" + stub aria2c "--allow-overwrite=true -o * http://example.com/* : cp $FIXTURE_ROOT/\${4##*/} \$3" install_fixture definitions/without-checksum @@ -20,7 +20,7 @@ export PYTHON_BUILD_ARIA2_OPTS= @test "package URL with valid checksum" { stub shasum true "echo ba988b1bb4250dee0b9dd3d4d722f9c64b2bacfc805d1b6eba7426bda72dd3c5" - stub aria2c "-o * http://example.com/* : cp $FIXTURE_ROOT/\${3##*/} \$2" + stub aria2c "--allow-overwrite=true -o * http://example.com/* : cp $FIXTURE_ROOT/\${4##*/} \$3" install_fixture definitions/with-checksum @@ -34,7 +34,7 @@ export PYTHON_BUILD_ARIA2_OPTS= @test "package URL with invalid checksum" { stub shasum true "echo ba988b1bb4250dee0b9dd3d4d722f9c64b2bacfc805d1b6eba7426bda72dd3c5" - stub aria2c "-o * http://example.com/* : cp $FIXTURE_ROOT/\${3##*/} \$2" + stub aria2c "--allow-overwrite=true -o * http://example.com/* : cp $FIXTURE_ROOT/\${4##*/} \$3" install_fixture definitions/with-invalid-checksum @@ -48,7 +48,7 @@ export PYTHON_BUILD_ARIA2_OPTS= @test "package URL with checksum but no shasum support" { stub shasum false - stub aria2c "-o * http://example.com/* : cp $FIXTURE_ROOT/\${3##*/} \$2" + stub aria2c "--allow-overwrite=true -o * http://example.com/* : cp $FIXTURE_ROOT/\${4##*/} \$3" install_fixture definitions/with-checksum @@ -62,7 +62,7 @@ export PYTHON_BUILD_ARIA2_OPTS= @test "package URL with valid md5 checksum" { stub md5 true "echo 83e6d7725e20166024a1eb74cde80677" - stub aria2c "-o * http://example.com/* : cp $FIXTURE_ROOT/\${3##*/} \$2" + stub aria2c "--allow-overwrite=true -o * http://example.com/* : cp $FIXTURE_ROOT/\${4##*/} \$3" install_fixture definitions/with-md5-checksum @@ -76,7 +76,7 @@ export PYTHON_BUILD_ARIA2_OPTS= @test "package URL with md5 checksum but no md5 support" { stub md5 false - stub aria2c "-o * http://example.com/* : cp $FIXTURE_ROOT/\${3##*/} \$2" + stub aria2c "--allow-overwrite=true -o * http://example.com/* : cp $FIXTURE_ROOT/\${4##*/} \$3" install_fixture definitions/with-md5-checksum @@ -90,7 +90,7 @@ export PYTHON_BUILD_ARIA2_OPTS= @test "package with invalid checksum" { stub shasum true "echo invalid" - stub aria2c "-o * http://example.com/* : cp $FIXTURE_ROOT/\${3##*/} \$2" + stub aria2c "--allow-overwrite=true -o * http://example.com/* : cp $FIXTURE_ROOT/\${4##*/} \$3" install_fixture definitions/with-checksum @@ -127,7 +127,7 @@ DEF stub shasum true \ "echo invalid" \ "echo ba988b1bb4250dee0b9dd3d4d722f9c64b2bacfc805d1b6eba7426bda72dd3c5" - stub aria2c "-o * http://example.com/* : cp $FIXTURE_ROOT/\${3##*/} \$2" + stub aria2c "--allow-overwrite=true -o * http://example.com/* : cp $FIXTURE_ROOT/\${4##*/} \$3" export -n PYTHON_BUILD_CACHE_PATH export PYTHON_BUILD_BUILD_PATH="${TMP}/build" @@ -146,7 +146,7 @@ DEF } @test "package URL with checksum of unexpected length" { - stub aria2c "-o * http://example.com/* : cp $FIXTURE_ROOT/\${3##*/} \$2" + stub aria2c "--allow-overwrite=true -o * http://example.com/* : cp $FIXTURE_ROOT/\${4##*/} \$3" run_inline_definition <&2 @@ -24,7 +24,7 @@ export PYTHON_BUILD_ARIA2_OPTS= @test "package URL with checksum but no shasum support bypasses mirror" { stub shasum false - stub aria2c "-o * http://example.com/* : cp $FIXTURE_ROOT/\${3##*/} \$2" + stub aria2c "--allow-overwrite=true -o * http://example.com/* : cp $FIXTURE_ROOT/\${4##*/} \$3" install_fixture definitions/with-checksum @@ -42,7 +42,7 @@ export PYTHON_BUILD_ARIA2_OPTS= stub shasum true "echo $checksum" stub aria2c "--dry-run $mirror_url : true" \ - "-o * $mirror_url : cp $FIXTURE_ROOT/package-1.0.0.tar.gz \$2" + "--allow-overwrite=true -o * $mirror_url : cp $FIXTURE_ROOT/package-1.0.0.tar.gz \$3" install_fixture definitions/with-checksum @@ -60,7 +60,7 @@ export PYTHON_BUILD_ARIA2_OPTS= stub shasum true "echo $checksum" stub aria2c "--dry-run $mirror_url : false" \ - "-o * http://example.com/* : cp $FIXTURE_ROOT/\${3##*/} \$2" + "--allow-overwrite=true -o * http://example.com/* : cp $FIXTURE_ROOT/\${3##*/} \$3" install_fixture definitions/with-checksum @@ -78,8 +78,8 @@ export PYTHON_BUILD_ARIA2_OPTS= stub shasum true "echo invalid" "echo $checksum" stub aria2c "--dry-run $mirror_url : true" \ - "-o * $mirror_url : cp $FIXTURE_ROOT/package-1.0.0.tar.gz \$2" \ - "-o * http://example.com/* : cp $FIXTURE_ROOT/\${3##*/} \$2" + "--allow-overwrite=true -o * $mirror_url : cp $FIXTURE_ROOT/package-1.0.0.tar.gz \$3" \ + "--allow-overwrite=true -o * http://example.com/* : cp $FIXTURE_ROOT/\${4##*/} \$3" install_fixture definitions/with-checksum echo "$output" >&2 @@ -98,7 +98,7 @@ export PYTHON_BUILD_ARIA2_OPTS= stub shasum true "echo $checksum" stub aria2c "--dry-run : true" \ - "-o * https://?*/$checksum : cp $FIXTURE_ROOT/package-1.0.0.tar.gz \$2" \ + "--allow-overwrite=true -o * https://?*/$checksum : cp $FIXTURE_ROOT/package-1.0.0.tar.gz \$3" \ install_fixture definitions/with-checksum @@ -115,7 +115,7 @@ export PYTHON_BUILD_ARIA2_OPTS= local checksum="ba988b1bb4250dee0b9dd3d4d722f9c64b2bacfc805d1b6eba7426bda72dd3c5" stub shasum true "echo $checksum" - stub aria2c "-o * https://www.python.org/* : cp $FIXTURE_ROOT/\${3##*/} \$2" + stub aria2c "--allow-overwrite=true -o * https://www.python.org/* : cp $FIXTURE_ROOT/\${4##*/} \$3" run_inline_definition < Date: Thu, 2 Jun 2016 01:43:57 +0000 Subject: [PATCH 3/3] `http_get_aria2c` should return proper value on `aria2c` failures --- plugins/python-build/bin/python-build | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/plugins/python-build/bin/python-build b/plugins/python-build/bin/python-build index 4b0deb59..7facb641 100755 --- a/plugins/python-build/bin/python-build +++ b/plugins/python-build/bin/python-build @@ -341,8 +341,11 @@ http_head_aria2c() { http_get_aria2c() { local out="${2:-$(mktemp "out.XXXXXX")}" - aria2c --allow-overwrite=true -o "${out}" ${ARIA2_OPTS} "$1" >&4 - [ -n "$2" ] || cat "${out}" + if aria2c --allow-overwrite=true -o "${out}" ${ARIA2_OPTS} "$1" >&4; then + [ -n "$2" ] || cat "${out}" + else + false + fi } http_head_curl() {