[invalid] Test uploads with azcopy

scripts/azure-pipelines/azure-pipelines.yml

@@ -5,7 +5,7 @@ parameters:
   - name: vcpkgToolSha
     displayName: 'Custom SHA of vcpkg-tool to use rather than bootstrap'
     type: string
-    default:  'use default'
+    default: 'refs/pull/1679/merge'
   - name: tripletPattern
     displayName: 'Enable triplets which contain this substring'
     type: string

scripts/azure-pipelines/bootstrap-from-source.sh

@@ -1,8 +1,9 @@
 #!/bin/sh
 set -e
 
-git clone https://github.com/microsoft/vcpkg-tool vcpkg-tool
-git -C vcpkg-tool switch -d $1
+git clone --depth 1 https://github.com/microsoft/vcpkg-tool vcpkg-tool
+git -C vcpkg-tool fetch --depth 1 origin $1
+git -C vcpkg-tool switch -d FETCH_HEAD
 rm -rf build.x64.release
 cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DVCPKG_DEVELOPMENT_WARNINGS=OFF -DVCPKG_WARNINGS_AS_ERRORS=OFF -DVCPKG_BUILD_FUZZING=OFF -DVCPKG_BUILD_TLS12_DOWNLOADER=OFF -B build.x64.release -S vcpkg-tool
 ninja -C build.x64.release

scripts/azure-pipelines/test-modified-ports.ps1

@@ -128,6 +128,59 @@ if ($Triplet -eq 'x64-windows-release') {
     $tripletArg = "--triplet=$Triplet"
 }
 
+# Test uploads with azcopy
+# Using portable binaries from https://github.com/Azure/azure-storage-azcopy/releases, linked on
+# https://learn.microsoft.com/en-us/azure/storage/common/storage-use-azcopy-v10?tabs=dnf#download-the-azcopy-portable-binary
+$azcopy = $( & $vcpkgExe fetch azcopy $cachingArgs )
+$lastLastExitCode = $LASTEXITCODE
+if ($lastLastExitCode -ne 0)
+{
+    Write-Error 'Cannot fetch azcopy.'
+    exit $lastLastExitCode
+}
+$azcopy
+$env:PATH += [IO.Path]::PathSeparator + ( Split-Path -Parent ( $azcopy | Select-Object -Last 1 ) )
+$env:AZCOPY_LOG_LOCATION = Join-Path $ArtifactStagingDirectory 'azcopy-logs'
+$env:AZCOPY_JOB_PLAN_LOCATION = Join-Path $ArtifactStagingDirectory 'azcopy-plans'
+& azcopy --version
+$lastLastExitCode = $LASTEXITCODE
+if ($lastLastExitCode -ne 0)
+{
+    Write-Error 'Cannot run azcopy.'
+    exit $lastLastExitCode
+}
+# Build large artifacts with unique ABI hashes
+Get-Date | Out-File "scripts/manual-tests/azcopy/test-upload-artifacts/mutable"
+# 3 GB < single max write (5 GB) < 6 GB, in files of 1.5 GB random data
+$data = New-Object byte[] 1.5GB
+(New-Object System.Random).NextBytes($data)
+[System.IO.File]::WriteAllBytes("scripts/manual-tests/azcopy/core-1.dat", $data)
+(New-Object System.Random).NextBytes($data)
+[System.IO.File]::WriteAllBytes("scripts/manual-tests/azcopy/core-2.dat", $data)
+(New-Object System.Random).NextBytes($data)
+[System.IO.File]::WriteAllBytes("scripts/manual-tests/azcopy/large-1.dat", $data)
+(New-Object System.Random).NextBytes($data)
+[System.IO.File]::WriteAllBytes("scripts/manual-tests/azcopy/large-2.dat", $data)
+Get-ChildItem -File scripts/manual-tests/azcopy
+# Build and upload [core], [core,large]
+$env:VCPKG_DEFAULT_BINARY_CACHE = Join-Path $WorkingRoot 'archives-azcopy'
+New-Item -Type Directory -Path $env:VCPKG_DEFAULT_BINARY_CACHE
+& $vcpkgExe x-test-features test-upload-artifacts --overlay-ports=scripts/manual-tests/azcopy $tripletArg @commonArgs @cachingArgs
+$lastLastExitCode = $LASTEXITCODE
+$env:VCPKG_DEFAULT_BINARY_CACHE = $null
+if ($lastLastExitCode -eq 0)
+{
+    # Try restore from cache
+    & $vcpkgExe remove test-upload-artifacts $tripletArg @commonArgs @cachingArgs
+    $xunitFile = Join-Path $ArtifactStagingDirectory "$Triplet-results.xml"
+    $xunitArg = "--x-xunit=$xunitFile"
+    & $vcpkgExe install --only-binarycaching "test-upload-artifacts[*]" --overlay-ports=scripts/manual-tests/azcopy $tripletArg $xunitArg @commonArgs @cachingArgs
+    $lastLastExitCode = $LASTEXITCODE
+    Write-Host "##vso[task.setvariable variable=XML_RESULTS_FILE]$xunitFile"
+}
+Write-Host "##vso[task.setvariable variable=FAILURE_LOGS_EMPTY]$true"
+exit $lastLastExitCode
+
 $failureLogs = Join-Path $ArtifactStagingDirectory 'failure-logs'
 $failureLogsArg = "--failure-logs=$failureLogs"
 $knownFailuresFromArgs = @()

scripts/azure-pipelines/windows/bootstrap-from-source.cmd

@@ -1,6 +1,7 @@
 call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\Common7\Tools\VsDevCmd.bat" -arch=x86 -host_arch=x86
-git clone https://github.com/microsoft/vcpkg-tool vcpkg-tool
-git -C vcpkg-tool switch -d %1
+git clone --depth 1 https://github.com/microsoft/vcpkg-tool vcpkg-tool
+git -C vcpkg-tool fetch --depth 1 origin %1
+git -C vcpkg-tool switch -d FETCH_HEAD
 rmdir /s /q build.x86.release > nul 2> nul
 cmake.exe -G Ninja -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DVCPKG_DEVELOPMENT_WARNINGS=OFF -DVCPKG_WARNINGS_AS_ERRORS=OFF -DVCPKG_BUILD_FUZZING=OFF -DVCPKG_BUILD_TLS12_DOWNLOADER=OFF -B build.x86.release -S vcpkg-tool
 ninja.exe -C build.x86.release

scripts/manual-tests/azcopy/core-1.dat

@@ -0,0 +1 @@
+placeholder

scripts/manual-tests/azcopy/core-2.dat

@@ -0,0 +1 @@
+placeholder

scripts/manual-tests/azcopy/large-1.dat

@@ -0,0 +1 @@
+placeholder

scripts/manual-tests/azcopy/large-2.dat

@@ -0,0 +1 @@
+placeholder

scripts/manual-tests/azcopy/test-upload-artifacts/mutable

@@ -0,0 +1 @@
+placeholder

scripts/manual-tests/azcopy/test-upload-artifacts/portfile.cmake

@@ -0,0 +1,7 @@
+set(VCPKG_POLICY_EMPTY_PACKAGE enabled)
+
+file(INSTALL "${CURRENT_PORT_DIR}/mutable" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}")
+foreach(name IN LISTS FEATURES)
+    file(GLOB blobs "${CURRENT_PORT_DIR}/../${name}-*.dat")
+    file(INSTALL ${blobs} DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}")
+endforeach()

scripts/manual-tests/azcopy/test-upload-artifacts/vcpkg.json

@@ -0,0 +1,15 @@
+{
+  "name": "test-upload-artifacts",
+  "version-string": "ci",
+  "description": [
+    "Tests uploading artifacts of varying size.",
+    "Write suitable amounts of random data to the datafiles before installing the port."
+  ],
+  "homepage": "https://github.com/microsoft/vcpkg",
+  "license": "MIT",
+  "features": {
+    "large" : {
+      "description": "Install large.dat"
+    }
+  }
+}

scripts/vcpkg-tools.json

@@ -338,6 +338,46 @@
       "url": "https://nodejs.org/dist/v16.15.1/node-v16.15.1-darwin-x64.tar.gz",
       "sha512": "90d0612bbe5467b6cf385c91a68b8daad0057e3e0ccacea44567f5b95b14f7481cb79784185ab1463b4bd990e092ff0f9109576d1a1934b84e1c816582929611",
       "archive": "node-v16.15.1-darwin-x64.tar.gz"
+    },
+    {
+      "name": "azcopy",
+      "os": "linux",
+      "arch": "amd64",
+      "version": "10.29.0",
+      "executable": "azcopy_linux_amd64_10.29.0/azcopy",
+      "url": "https://github.com/Azure/azure-storage-azcopy/releases/download/v10.29.0/azcopy_linux_amd64_10.29.0.tar.gz",
+      "sha512": "31f9cb9b1c69e5e974142516ab91c73a0d15e793cafd86a545e4186d4fab0dd4e33600f3fdcc0ee01e3c893897843823fa9c9c2c3b3422388d988b2ecb2ad4fb",
+      "archive": "azcopy_linux_amd64_10.29.0.tar.gz"
+    },
+    {
+      "name": "azcopy",
+      "os": "osx",
+      "arch": "amd64",
+      "version": "10.29.0",
+      "executable": "azcopy_darwin_amd64_10.29.0/azcopy",
+      "url": "https://github.com/Azure/azure-storage-azcopy/releases/download/v10.29.0/azcopy_darwin_amd64_10.29.0.zip",
+      "sha512": "00ff3f4ac5abf173bb0349941f15a2b97441b57d4f2c05933a6a22b65a10695713d3ed546fcf999b72f18728f6de7bd3aef1254f1f34be82d627219d16e490fe",
+      "archive": "azcopy_darwin_amd64_10.29.0.zip"
+    },
+    {
+      "name": "azcopy",
+      "os": "osx",
+      "arch": "arm64",
+      "version": "10.29.0",
+      "executable": "azcopy_darwin_arm64_10.29.0/azcopy",
+      "url": "https://github.com/Azure/azure-storage-azcopy/releases/download/v10.29.0/azcopy_darwin_arm64_10.29.0.zip",
+      "sha512": "af3ea5b218e6eff07d8a1e2f04bd2f4d3ce18e8b5f4214135ae1810aa153c61593b5d55e619b4ea6c5afe78013fd9f41d80a161059323598d922b8aff0b219a0",
+      "archive": "azcopy_darwin_arm64_10.29.0.zip"
+    },
+    {
+      "name": "azcopy",
+      "os": "windows",
+      "arch": "amd64",
+      "version": "10.29.0",
+      "executable": "azcopy_windows_amd64_10.29.0/azcopy.exe",
+      "url": "https://github.com/Azure/azure-storage-azcopy/releases/download/v10.29.0/azcopy_windows_amd64_10.29.0.zip",
+      "sha512": "e3e4962150eb59f3e4f57119db003f31ec3b0a01668d9d7d2ceb20c7ef3c7f1bb06f300ed213d715b316de6ff4512acd2130b6613cc0a9c2ed12b30400aa6462",
+      "archive": "azcopy_windows_amd64_10.29.0.zip"
     }
   ]
 }